Fix ast_(v)asprintf() malloc failure usage conditions.
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                                                   and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO &&
422                 strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
423                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
424                 return 1;
425         }
426
427         return 0;
428 }
429
430 /*!
431  * \brief Determine if a stream is a video destination stream.
432  *
433  * A source channel name can be provided to narrow this to a destination stream
434  * for a particular source channel. Further, a source stream name can be provided
435  * to narrow this to a particular source stream's destination. However, empty strings
436  * can be provided to match any destination video stream, regardless of source channel
437  * or source stream.
438  *
439  * \param stream The stream to test
440  * \param source_channel_name The name of a source video channel to match
441  * \param source_stream_name The name of the source video stream to match
442  * \retval 1 The stream is a video destination stream
443  * \retval 0 The stream is not a video destination stream
444  */
445 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
446         const char *source_stream_name)
447 {
448         char *dest_video_name;
449         size_t dest_video_name_len;
450
451         if (ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
452                 return 0;
453         }
454
455         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
456
457         if (!ast_strlen_zero(source_channel_name)) {
458                 dest_video_name_len += strlen(source_channel_name) + 1;
459                 if (!ast_strlen_zero(source_stream_name)) {
460                         dest_video_name_len += strlen(source_stream_name) + 1;
461                 }
462         }
463         dest_video_name = ast_alloca(dest_video_name_len);
464
465         if (!ast_strlen_zero(source_channel_name)) {
466                 if (!ast_strlen_zero(source_stream_name)) {
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 } else {
473                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                                 source_channel_name);
476                         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
477                 }
478         } else {
479                 snprintf(dest_video_name, dest_video_name_len, "%s",
480                         SOFTBRIDGE_VIDEO_DEST_PREFIX);
481                 return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
482         }
483
484         return 0;
485 }
486
487 static int append_source_streams(struct ast_stream_topology *dest,
488         const char *channel_name,
489         const struct ast_stream_topology *source)
490 {
491         int i;
492
493         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
494                 struct ast_stream *stream;
495                 struct ast_stream *stream_clone;
496                 char *stream_clone_name = NULL;
497
498                 stream = ast_stream_topology_get_stream(source, i);
499                 if (!is_video_source(stream)) {
500                         continue;
501                 }
502
503                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
504                         channel_name, ast_stream_get_name(stream)) < 0) {
505                         return -1;
506                 }
507
508                 stream_clone = ast_stream_clone(stream, stream_clone_name);
509                 ast_free(stream_clone_name);
510                 if (!stream_clone) {
511                         return -1;
512                 }
513                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
514                         ast_stream_free(stream_clone);
515                         return -1;
516                 }
517         }
518
519         return 0;
520 }
521
522 static int append_all_streams(struct ast_stream_topology *dest,
523         const struct ast_stream_topology *source)
524 {
525         int i;
526         int dest_index = 0;
527
528         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
529                 struct ast_stream *clone;
530                 int added = 0;
531
532                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
533                 if (!clone) {
534                         return -1;
535                 }
536
537                 /* If we can reuse an existing removed stream then do so */
538                 while (dest_index < ast_stream_topology_get_count(dest)) {
539                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
540
541                         dest_index++;
542
543                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
544                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
545                                 added = 1;
546                                 break;
547                         }
548                 }
549
550                 /* If no removed stream exists that we took the place of append the stream */
551                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
552                         ast_stream_free(clone);
553                         return -1;
554                 }
555         }
556
557         return 0;
558 }
559
560 /*!
561  * \brief Issue channel stream topology change requests.
562  *
563  * When in SFU mode, each participant needs to be able to
564  * send video directly to other participants in the bridge.
565  * This means that all participants need to have their topologies
566  * updated. The joiner needs to have destination streams for
567  * all current participants, and the current participants need
568  * to have destinations streams added for the joiner's sources.
569  *
570  * \param joiner The channel that is joining the softmix bridge
571  * \param participants The current participants in the softmix bridge
572  */
573 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
574 {
575         struct ast_stream_topology *joiner_video = NULL;
576         struct ast_bridge_channel *participant;
577         int res;
578         struct softmix_channel *sc;
579
580         joiner_video = ast_stream_topology_alloc();
581         if (!joiner_video) {
582                 return;
583         }
584
585         sc = joiner->tech_pvt;
586
587         ast_channel_lock(joiner->chan);
588         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
589         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
590         ast_channel_unlock(joiner->chan);
591
592         if (res || !sc->topology) {
593                 goto cleanup;
594         }
595
596         AST_LIST_TRAVERSE(participants, participant, entry) {
597                 if (participant == joiner) {
598                         continue;
599                 }
600                 ast_channel_lock(participant->chan);
601                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
602                                 ast_channel_get_stream_topology(participant->chan));
603                 ast_channel_unlock(participant->chan);
604                 if (res) {
605                         goto cleanup;
606                 }
607         }
608
609         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
610
611         AST_LIST_TRAVERSE(participants, participant, entry) {
612                 if (participant == joiner) {
613                         continue;
614                 }
615
616                 sc = participant->tech_pvt;
617                 if (append_all_streams(sc->topology, joiner_video)) {
618                         goto cleanup;
619                 }
620                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
621         }
622
623 cleanup:
624         ast_stream_topology_free(joiner_video);
625 }
626
627 /*! \brief Function called when a channel is joined into the bridge */
628 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
629 {
630         struct softmix_channel *sc;
631         struct softmix_bridge_data *softmix_data;
632         int set_binaural = 0;
633         /*
634          * If false, the channel will be convolved, but since it is a non stereo channel, output
635          * will be mono.
636          */
637         int skip_binaural_output = 1;
638         int pos_id;
639         int is_announcement = 0;
640         int samplerate_change;
641
642         softmix_data = bridge->tech_pvt;
643         if (!softmix_data) {
644                 return -1;
645         }
646
647         /* Create a new softmix_channel structure and allocate various things on it */
648         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
649                 return -1;
650         }
651
652         samplerate_change = softmix_data->internal_rate;
653         pos_id = -1;
654         if (bridge->softmix.binaural_active) {
655                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
656                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
657                         if (set_binaural) {
658                                 softmix_data->internal_rate = samplerate_change;
659                         }
660                         skip_binaural_output = 0;
661                 } else {
662                         is_announcement = 1;
663                 }
664                 if (set_binaural) {
665                         softmix_data->convolve.binaural_active = 1;
666                 }
667                 if (!skip_binaural_output)      {
668                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
669                         if (pos_id == -1) {
670                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
671                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
672                                                 ast_channel_name(bridge_channel->chan));
673                                 return -1;
674                         }
675                 }
676         }
677
678         /* Can't forget the lock */
679         ast_mutex_init(&sc->lock);
680
681         /* Can't forget to record our pvt structure within the bridged channel structure */
682         bridge_channel->tech_pvt = sc;
683
684         set_softmix_bridge_data(softmix_data->internal_rate,
685                 softmix_data->internal_mixing_interval
686                         ? softmix_data->internal_mixing_interval
687                         : DEFAULT_SOFTMIX_INTERVAL,
688                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
689
690         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
691                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
692         }
693
694         softmix_poke_thread(softmix_data);
695         return 0;
696 }
697
698 static void remove_destination_streams(struct ast_stream_topology *topology,
699         const char *channel_name)
700 {
701         int i;
702
703         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
704                 struct ast_stream *stream;
705
706                 stream = ast_stream_topology_get_stream(topology, i);
707
708                 if (is_video_dest(stream, channel_name, NULL)) {
709                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
710                 }
711         }
712 }
713
714 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
715 {
716         struct ast_bridge_channel *participant;
717         struct softmix_channel *sc;
718
719         AST_LIST_TRAVERSE(participants, participant, entry) {
720                 sc = participant->tech_pvt;
721                 remove_destination_streams(sc->topology, ast_channel_name(leaver->chan));
722                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
723         }
724
725         sc = leaver->tech_pvt;
726         remove_destination_streams(sc->topology, "");
727         ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
728
729         return 0;
730 }
731
732 /*! \brief Function called when a channel leaves the bridge */
733 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
734 {
735         struct softmix_channel *sc;
736         struct softmix_bridge_data *softmix_data;
737
738         softmix_data = bridge->tech_pvt;
739         sc = bridge_channel->tech_pvt;
740         if (!sc) {
741                 return;
742         }
743
744         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
745                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
746         }
747
748         if (bridge->softmix.binaural_active) {
749                 if (sc->binaural) {
750                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
751                                         softmix_data->default_sample_size);
752                 }
753         }
754
755         bridge_channel->tech_pvt = NULL;
756
757         ast_stream_topology_free(sc->topology);
758
759         /* Drop mutex lock */
760         ast_mutex_destroy(&sc->lock);
761
762         /* Drop the factory */
763         ast_slinfactory_destroy(&sc->factory);
764
765         /* Drop any formats on the frames */
766         ao2_cleanup(sc->write_frame.subclass.format);
767
768         /* Drop the DSP */
769         ast_dsp_free(sc->dsp);
770
771         /* Eep! drop ourselves */
772         ast_free(sc);
773 }
774
775 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
776 {
777         struct ast_bridge_channel *cur;
778
779         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
780                 if (cur->suspended) {
781                         continue;
782                 }
783                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
784                         ast_bridge_channel_queue_frame(cur, frame);
785                         break;
786                 }
787         }
788 }
789
790 /*!
791  * \internal
792  * \brief Determine what to do with a video frame.
793  * \since 12.0.0
794  *
795  * \param bridge Which bridge is getting the frame
796  * \param bridge_channel Which channel is writing the frame.
797  * \param frame What is being written.
798  *
799  * \return Nothing
800  */
801 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
802 {
803         struct softmix_channel *sc;
804         int video_src_priority;
805
806         /* Determine if the video frame should be distributed or not */
807         switch (bridge->softmix.video_mode.mode) {
808         case AST_BRIDGE_VIDEO_MODE_NONE:
809                 break;
810         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
811                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
812                 if (video_src_priority == 1) {
813                         /* Pass to me and everyone else. */
814                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
815                 }
816                 break;
817         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
818                 sc = bridge_channel->tech_pvt;
819                 ast_mutex_lock(&sc->lock);
820                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
821                         sc->video_talker.energy_average,
822                         frame->subclass.frame_ending);
823                 ast_mutex_unlock(&sc->lock);
824                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
825                 if (video_src_priority == 1) {
826                         int num_src = ast_bridge_number_video_src(bridge);
827                         int echo = num_src > 1 ? 0 : 1;
828
829                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
830                 } else if (video_src_priority == 2) {
831                         softmix_pass_video_top_priority(bridge, frame);
832                 }
833                 break;
834         case AST_BRIDGE_VIDEO_MODE_SFU:
835                 /* Nothing special to do here, the bridge channel stream map will ensure the
836                  * video goes everywhere it needs to
837                  */
838                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
839                 break;
840         }
841 }
842
843 /*!
844  * \internal
845  * \brief Determine what to do with a voice frame.
846  * \since 12.0.0
847  *
848  * \param bridge Which bridge is getting the frame
849  * \param bridge_channel Which channel is writing the frame.
850  * \param frame What is being written.
851  *
852  * \return Nothing
853  */
854 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
855 {
856         struct softmix_channel *sc = bridge_channel->tech_pvt;
857         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
858         int totalsilence = 0;
859         int cur_energy = 0;
860         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
861                 bridge_channel->tech_args.silence_threshold :
862                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
863         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
864
865         /* Write the frame into the conference */
866         ast_mutex_lock(&sc->lock);
867
868         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
869                 /*
870                  * The incoming frame is not the expected format.  Update
871                  * the channel's translation path to get us slinear from
872                  * the new format for the next frame.
873                  *
874                  * There is the possibility that this frame is an old slinear
875                  * rate frame that was in flight when the softmix bridge
876                  * changed rates.  If so it will self correct on subsequent
877                  * frames.
878                  */
879                 ast_channel_lock(bridge_channel->chan);
880                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
881                         ast_channel_name(bridge_channel->chan),
882                         ast_format_get_name(frame->subclass.format),
883                         ast_format_get_name(sc->read_slin_format));
884                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
885                         sc->read_slin_format);
886                 ast_channel_unlock(bridge_channel->chan);
887         }
888
889         /* The channel will be leaving soon if there is no dsp. */
890         if (sc->dsp) {
891                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
892         }
893
894         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
895                 int cur_slot = sc->video_talker.energy_history_cur_slot;
896
897                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
898                 sc->video_talker.energy_accum += cur_energy;
899                 sc->video_talker.energy_history[cur_slot] = cur_energy;
900                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
901                 sc->video_talker.energy_history_cur_slot++;
902                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
903                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
904                 }
905         }
906
907         if (totalsilence < silence_threshold) {
908                 if (!sc->talking) {
909                         update_talking = 1;
910                 }
911                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
912         } else {
913                 if (sc->talking) {
914                         update_talking = 0;
915                 }
916                 sc->talking = 0;
917         }
918
919         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
920          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
921          * the audio by flushing the buffer before adding new audio in. */
922         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
923                 ast_slinfactory_flush(&sc->factory);
924         }
925
926         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
927          * is not determined to be talking. */
928         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
929                 ast_slinfactory_feed(&sc->factory, frame);
930         }
931
932         /* Alllll done */
933         ast_mutex_unlock(&sc->lock);
934
935         if (update_talking != -1) {
936                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
937         }
938 }
939
940 static int remove_all_original_streams(struct ast_stream_topology *dest,
941         const struct ast_stream_topology *source,
942         const struct ast_stream_topology *original)
943 {
944         int i;
945
946         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
947                 struct ast_stream *stream;
948                 int original_index;
949
950                 stream = ast_stream_topology_get_stream(source, i);
951
952                 /* Mark the existing stream as removed so we get a new one, this will get
953                  * reused on a subsequent renegotiation.
954                  */
955                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
956                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
957
958                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
959                                 struct ast_stream *removed;
960
961                                 /* Since the participant is still going to be in the bridge we
962                                  * change the name so that routing does not attempt to route video
963                                  * to this stream.
964                                  */
965                                 removed = ast_stream_clone(stream, "removed");
966                                 if (!removed) {
967                                         return -1;
968                                 }
969
970                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
971
972                                 /* The destination topology can only ever contain the same, or more,
973                                  * streams than the original so this is safe.
974                                  */
975                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
976                                         ast_stream_free(removed);
977                                         return -1;
978                                 }
979
980                                 break;
981                         }
982                 }
983         }
984
985         return 0;
986 }
987
988 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
989 {
990         struct ast_stream_topology *source_video = NULL;
991         struct ast_bridge_channel *participant;
992         int res;
993
994         source_video = ast_stream_topology_alloc();
995         if (!source_video) {
996                 return;
997         }
998
999         ast_channel_lock(source->chan);
1000         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
1001         ast_channel_unlock(source->chan);
1002         if (res) {
1003                 goto cleanup;
1004         }
1005
1006         AST_LIST_TRAVERSE(participants, participant, entry) {
1007                 struct ast_stream_topology *original_topology;
1008                 struct softmix_channel *sc;
1009
1010                 if (participant == source) {
1011                         continue;
1012                 }
1013
1014                 sc = participant->tech_pvt;
1015
1016                 original_topology = ast_stream_topology_clone(sc->topology);
1017                 if (!original_topology) {
1018                         goto cleanup;
1019                 }
1020
1021                 /* We add all the source streams back in, if any removed streams are already present they will
1022                  * get used first followed by appending new ones.
1023                  */
1024                 if (append_all_streams(sc->topology, source_video)) {
1025                         ast_stream_topology_free(original_topology);
1026                         goto cleanup;
1027                 }
1028
1029                 /* And the original existing streams get marked as removed. This causes the remote side to see
1030                  * a new stream for the source streams.
1031                  */
1032                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1033                         ast_stream_topology_free(original_topology);
1034                         goto cleanup;
1035                 }
1036
1037                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1038                 ast_stream_topology_free(original_topology);
1039         }
1040
1041 cleanup:
1042         ast_stream_topology_free(source_video);
1043 }
1044
1045 /*!
1046  * \internal
1047  * \brief Determine what to do with a control frame.
1048  * \since 12.0.0
1049  *
1050  * \param bridge Which bridge is getting the frame
1051  * \param bridge_channel Which channel is writing the frame.
1052  * \param frame What is being written.
1053  *
1054  * \retval 0 Frame accepted into the bridge.
1055  * \retval -1 Frame needs to be deferred.
1056  */
1057 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1058 {
1059         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1060
1061         /*
1062          * XXX Softmix needs to use channel roles to determine what to
1063          * do with control frames.
1064          */
1065
1066         switch (frame->subclass.integer) {
1067         case AST_CONTROL_VIDUPDATE:
1068                 if (!bridge->softmix.video_mode.video_update_discard ||
1069                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1070                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1071                         softmix_data->last_video_update = ast_tvnow();
1072                 }
1073                 break;
1074         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1075                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1076                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1077                 }
1078                 break;
1079         default:
1080                 break;
1081         }
1082
1083         return 0;
1084 }
1085
1086 /*!
1087  * \internal
1088  * \brief Determine what to do with a frame written into the bridge.
1089  * \since 12.0.0
1090  *
1091  * \param bridge Which bridge is getting the frame
1092  * \param bridge_channel Which channel is writing the frame.
1093  * \param frame What is being written.
1094  *
1095  * \retval 0 Frame accepted into the bridge.
1096  * \retval -1 Frame needs to be deferred.
1097  *
1098  * \note On entry, bridge is already locked.
1099  */
1100 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1101 {
1102         int res = 0;
1103
1104         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1105                 /* "Accept" the frame and discard it. */
1106                 return 0;
1107         }
1108
1109         /*
1110          * XXX Softmix needs to use channel roles to determine who gets
1111          * what frame.  Possible roles: announcer, recorder, agent,
1112          * supervisor.
1113          */
1114         switch (frame->frametype) {
1115         case AST_FRAME_NULL:
1116                 /* "Accept" the frame and discard it. */
1117                 break;
1118         case AST_FRAME_DTMF_BEGIN:
1119         case AST_FRAME_DTMF_END:
1120                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1121                 break;
1122         case AST_FRAME_VOICE:
1123                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1124                 break;
1125         case AST_FRAME_VIDEO:
1126                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1127                 break;
1128         case AST_FRAME_CONTROL:
1129                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1130                 break;
1131         case AST_FRAME_BRIDGE_ACTION:
1132                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1133                 break;
1134         case AST_FRAME_BRIDGE_ACTION_SYNC:
1135                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1136                 ast_assert(0);
1137         default:
1138                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1139                 /* "Accept" the frame and discard it. */
1140                 break;
1141         }
1142
1143         return res;
1144 }
1145
1146 static void gather_softmix_stats(struct softmix_stats *stats,
1147         const struct softmix_bridge_data *softmix_data,
1148         struct ast_bridge_channel *bridge_channel)
1149 {
1150         int channel_native_rate;
1151
1152         /* Gather stats about channel sample rates. */
1153         ast_channel_lock(bridge_channel->chan);
1154         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1155                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1156         ast_channel_unlock(bridge_channel->chan);
1157
1158         if (stats->highest_supported_rate < channel_native_rate) {
1159                 stats->highest_supported_rate = channel_native_rate;
1160         }
1161         if (softmix_data->internal_rate < channel_native_rate) {
1162                 int i;
1163
1164                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1165                         if (stats->sample_rates[i] == channel_native_rate) {
1166                                 stats->num_channels[i]++;
1167                                 break;
1168                         } else if (!stats->sample_rates[i]) {
1169                                 stats->sample_rates[i] = channel_native_rate;
1170                                 stats->num_channels[i]++;
1171                                 break;
1172                         }
1173                 }
1174                 stats->num_above_internal_rate++;
1175         } else if (softmix_data->internal_rate == channel_native_rate) {
1176                 stats->num_at_internal_rate++;
1177         }
1178 }
1179
1180 /*!
1181  * \internal
1182  * \brief Analyse mixing statistics and change bridges internal rate
1183  * if necessary.
1184  *
1185  * \retval 0, no changes to internal rate
1186  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1187  */
1188 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1189                 struct softmix_bridge_data *softmix_data, int binaural_active)
1190 {
1191         int i;
1192
1193         if (binaural_active) {
1194                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1195         }
1196
1197         /*
1198          * Re-adjust the internal bridge sample rate if
1199          * 1. The bridge's internal sample rate is locked in at a sample
1200          *    rate other than the current sample rate being used.
1201          * 2. two or more channels support a higher sample rate
1202          * 3. no channels support the current sample rate or a higher rate
1203          */
1204         if (stats->locked_rate) {
1205                 /* if the rate is locked by the bridge, only update it if it differs
1206                  * from the current rate we are using. */
1207                 if (softmix_data->internal_rate != stats->locked_rate) {
1208                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1209                                 softmix_data->internal_rate, stats->locked_rate);
1210                         softmix_data->internal_rate = stats->locked_rate;
1211                         return 1;
1212                 }
1213         } else if (stats->num_above_internal_rate >= 2) {
1214                 /* the highest rate is just used as a starting point */
1215                 unsigned int best_rate = stats->highest_supported_rate;
1216                 int best_index = -1;
1217
1218                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1219                         if (stats->num_channels[i]) {
1220                                 break;
1221                         }
1222                         if (2 <= stats->num_channels[i]) {
1223                                 /* Two or more channels support this rate. */
1224                                 if (best_index == -1
1225                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1226                                         /*
1227                                          * best_rate starts out being the first sample rate
1228                                          * greater than the internal sample rate that two or
1229                                          * more channels support.
1230                                          *
1231                                          * or
1232                                          *
1233                                          * There are multiple rates above the internal rate
1234                                          * and this rate is higher than the previous rate two
1235                                          * or more channels support.
1236                                          */
1237                                         best_rate = stats->sample_rates[i];
1238                                         best_index = i;
1239                                 }
1240                         } else if (best_index == -1) {
1241                                 /*
1242                                  * It is possible that multiple channels exist with native sample
1243                                  * rates above the internal sample rate, but none of those channels
1244                                  * have the same rate in common.  In this case, the lowest sample
1245                                  * rate among those channels is picked. Over time as additional
1246                                  * statistic runs are made the internal sample rate number will
1247                                  * adjust to the most optimal sample rate, but it may take multiple
1248                                  * iterations.
1249                                  */
1250                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1251                         }
1252                 }
1253
1254                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1255                         softmix_data->internal_rate, best_rate);
1256                 softmix_data->internal_rate = best_rate;
1257                 return 1;
1258         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1259                 /* In this case, the highest supported rate is actually lower than the internal rate */
1260                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1261                         softmix_data->internal_rate, stats->highest_supported_rate);
1262                 softmix_data->internal_rate = stats->highest_supported_rate;
1263                 return 1;
1264         }
1265         return 0;
1266 }
1267
1268 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1269                 unsigned int starting_num_entries, unsigned int binaural_active)
1270 {
1271         memset(mixing_array, 0, sizeof(*mixing_array));
1272         mixing_array->max_num_entries = starting_num_entries;
1273         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1274                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1275                 return -1;
1276         }
1277         if (binaural_active) {
1278                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1279                                 sizeof(struct convolve_channel_pair *)))) {
1280                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1281                         return -1;
1282                 }
1283         }
1284         return 0;
1285 }
1286
1287 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1288                 unsigned int binaural_active)
1289 {
1290         ast_free(mixing_array->buffers);
1291         if (binaural_active) {
1292                 ast_free(mixing_array->chan_pairs);
1293         }
1294 }
1295
1296 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1297                 unsigned int num_entries, unsigned int binaural_active)
1298 {
1299         int16_t **tmp;
1300
1301         /* give it some room to grow since memory is cheap but allocations can be expensive */
1302         mixing_array->max_num_entries = num_entries;
1303         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1304                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1305                 return -1;
1306         }
1307         if (binaural_active) {
1308                 struct convolve_channel_pair **tmp2;
1309                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1310                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1311                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1312                         return -1;
1313                 }
1314                 mixing_array->chan_pairs = tmp2;
1315         }
1316         mixing_array->buffers = tmp;
1317         return 0;
1318 }
1319
1320 /*!
1321  * \brief Mixing loop.
1322  *
1323  * \retval 0 on success
1324  * \retval -1 on failure
1325  */
1326 static int softmix_mixing_loop(struct ast_bridge *bridge)
1327 {
1328         struct softmix_stats stats = { { 0 }, };
1329         struct softmix_mixing_array mixing_array;
1330         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1331         struct ast_timer *timer;
1332         struct softmix_translate_helper trans_helper;
1333         int16_t buf[MAX_DATALEN];
1334 #ifdef BINAURAL_RENDERING
1335         int16_t bin_buf[MAX_DATALEN];
1336         int16_t ann_buf[MAX_DATALEN];
1337 #endif
1338         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1339         int timingfd;
1340         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1341         unsigned int idx;
1342         unsigned int x;
1343         int res = -1;
1344
1345         timer = softmix_data->timer;
1346         timingfd = ast_timer_fd(timer);
1347         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1348         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1349
1350         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1351         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1352                         bridge->softmix.binaural_active)) {
1353                 goto softmix_cleanup;
1354         }
1355
1356         /*
1357          * XXX Softmix needs to use channel roles to determine who gets
1358          * what audio mixed.
1359          */
1360         while (!softmix_data->stop && bridge->num_active) {
1361                 struct ast_bridge_channel *bridge_channel;
1362                 int timeout = -1;
1363                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1364                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1365                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1366
1367                 if (softmix_datalen > MAX_DATALEN) {
1368                         /* This should NEVER happen, but if it does we need to know about it. Almost
1369                          * all the memcpys used during this process depend on this assumption.  Rather
1370                          * than checking this over and over again through out the code, this single
1371                          * verification is done on each iteration. */
1372                         ast_log(LOG_WARNING,
1373                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1374                                 bridge->uniqueid);
1375                         goto softmix_cleanup;
1376                 }
1377
1378                 /* Grow the mixing array buffer as participants are added. */
1379                 if (mixing_array.max_num_entries < bridge->num_channels
1380                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1381                                         bridge->softmix.binaural_active)) {
1382                         goto softmix_cleanup;
1383                 }
1384
1385                 /* init the number of buffers stored in the mixing array to 0.
1386                  * As buffers are added for mixing, this number is incremented. */
1387                 mixing_array.used_entries = 0;
1388
1389                 /* These variables help determine if a rate change is required */
1390                 if (!stat_iteration_counter) {
1391                         memset(&stats, 0, sizeof(stats));
1392                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1393                 }
1394
1395                 /* If the sample rate has changed, update the translator helper */
1396                 if (update_all_rates) {
1397                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1398                 }
1399
1400 #ifdef BINAURAL_RENDERING
1401                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1402 #endif
1403
1404                 /* Go through pulling audio from each factory that has it available */
1405                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1406                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1407
1408                         if (!sc) {
1409                                 /* This channel failed to join successfully. */
1410                                 continue;
1411                         }
1412
1413                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1414                         if (update_all_rates) {
1415                                 set_softmix_bridge_data(softmix_data->internal_rate,
1416                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1417                         }
1418
1419                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1420                         if (!stat_iteration_counter) {
1421                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1422                         }
1423
1424                         /* if the channel is suspended, don't check for audio, but still gather stats */
1425                         if (bridge_channel->suspended) {
1426                                 continue;
1427                         }
1428
1429                         /* Try to get audio from the factory if available */
1430                         ast_mutex_lock(&sc->lock);
1431                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1432 #ifdef BINAURAL_RENDERING
1433                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1434                                                 ast_channel_name(bridge_channel->chan));
1435 #endif
1436                                 mixing_array.used_entries++;
1437                         }
1438                         ast_mutex_unlock(&sc->lock);
1439                 }
1440
1441                 /* mix it like crazy (non binaural channels)*/
1442                 memset(buf, 0, softmix_datalen);
1443                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1444                         for (x = 0; x < softmix_samples; ++x) {
1445                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1446                         }
1447                 }
1448
1449 #ifdef BINAURAL_RENDERING
1450                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1451 #endif
1452
1453                 /* Next step go through removing the channel's own audio and creating a good frame... */
1454                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1455                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1456
1457                         if (!sc || bridge_channel->suspended) {
1458                                 /* This channel failed to join successfully or is suspended. */
1459                                 continue;
1460                         }
1461
1462                         ast_mutex_lock(&sc->lock);
1463
1464                         /* Make SLINEAR write frame from local buffer */
1465                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1466                                 "Replace softmix channel slin format");
1467 #ifdef BINAURAL_RENDERING
1468                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1469                                         && sc->binaural) {
1470                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1471                                                 softmix_samples, buf);
1472                         } else
1473 #endif
1474                         {
1475                                 sc->write_frame.datalen = softmix_datalen;
1476                                 sc->write_frame.samples = softmix_samples;
1477                                 memcpy(sc->final_buf, buf, softmix_datalen);
1478                         }
1479                         /* process the softmix channel's new write audio */
1480                         softmix_process_write_audio(&trans_helper,
1481                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1482                                         softmix_data->default_sample_size);
1483
1484                         ast_mutex_unlock(&sc->lock);
1485
1486                         /* A frame is now ready for the channel. */
1487                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1488                 }
1489
1490                 update_all_rates = 0;
1491                 if (!stat_iteration_counter) {
1492                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1493                                         bridge->softmix.binaural_active);
1494                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1495                 }
1496                 stat_iteration_counter--;
1497
1498                 ast_bridge_unlock(bridge);
1499                 /* cleanup any translation frame data from the previous mixing iteration. */
1500                 softmix_translate_helper_cleanup(&trans_helper);
1501                 /* Wait for the timing source to tell us to wake up and get things done */
1502                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1503                 if (ast_timer_ack(timer, 1) < 0) {
1504                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1505                                 bridge->uniqueid);
1506                         ast_bridge_lock(bridge);
1507                         goto softmix_cleanup;
1508                 }
1509                 ast_bridge_lock(bridge);
1510
1511                 /* make sure to detect mixing interval changes if they occur. */
1512                 if (bridge->softmix.internal_mixing_interval
1513                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1514                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1515                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1516                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1517                 }
1518         }
1519
1520         res = 0;
1521
1522 softmix_cleanup:
1523         softmix_translate_helper_destroy(&trans_helper);
1524         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1525         return res;
1526 }
1527
1528 /*!
1529  * \internal
1530  * \brief Mixing thread.
1531  * \since 12.0.0
1532  *
1533  * \note The thread does not have its own reference to the
1534  * bridge.  The lifetime of the thread is tied to the lifetime
1535  * of the mixing technology association with the bridge.
1536  */
1537 static void *softmix_mixing_thread(void *data)
1538 {
1539         struct softmix_bridge_data *softmix_data = data;
1540         struct ast_bridge *bridge = softmix_data->bridge;
1541
1542         ast_bridge_lock(bridge);
1543         if (bridge->callid) {
1544                 ast_callid_threadassoc_add(bridge->callid);
1545         }
1546
1547         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1548
1549         while (!softmix_data->stop) {
1550                 if (!bridge->num_active) {
1551                         /* Wait for something to happen to the bridge. */
1552                         ast_bridge_unlock(bridge);
1553                         ast_mutex_lock(&softmix_data->lock);
1554                         if (!softmix_data->stop) {
1555                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1556                         }
1557                         ast_mutex_unlock(&softmix_data->lock);
1558                         ast_bridge_lock(bridge);
1559                         continue;
1560                 }
1561
1562                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1563 #ifndef BINAURAL_RENDERING
1564                         ast_bridge_lock(bridge);
1565                         bridge->softmix.binaural_active = 0;
1566                         ast_bridge_unlock(bridge);
1567                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1568                                         "compiled.\n", bridge->uniqueid);
1569 #else
1570                         /* Set and init binaural data if binaural is activated in the configuration. */
1571                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1572                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1573                                         softmix_data->internal_mixing_interval);
1574                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1575                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1576                                         == -1) {
1577                                 ast_bridge_lock(bridge);
1578                                 bridge->softmix.binaural_active = 0;
1579                                 ast_bridge_unlock(bridge);
1580                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1581                                                 "binaural processing,  Will only process mono audio.\n",
1582                                                 bridge->uniqueid);
1583                         }
1584                         softmix_data->binaural_init = 1;
1585 #endif
1586                 }
1587
1588                 if (softmix_mixing_loop(bridge)) {
1589                         /*
1590                          * A mixing error occurred.  Sleep and try again later so we
1591                          * won't flood the logs.
1592                          */
1593                         ast_bridge_unlock(bridge);
1594                         sleep(1);
1595                         ast_bridge_lock(bridge);
1596                 }
1597         }
1598
1599         ast_bridge_unlock(bridge);
1600
1601         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1602
1603         return NULL;
1604 }
1605
1606 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1607 {
1608         if (softmix_data->timer) {
1609                 ast_timer_close(softmix_data->timer);
1610                 softmix_data->timer = NULL;
1611         }
1612         ast_mutex_destroy(&softmix_data->lock);
1613         ast_cond_destroy(&softmix_data->cond);
1614         ast_free(softmix_data);
1615 }
1616
1617 /*! \brief Function called when a bridge is created */
1618 static int softmix_bridge_create(struct ast_bridge *bridge)
1619 {
1620         struct softmix_bridge_data *softmix_data;
1621
1622         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1623         if (!softmix_data) {
1624                 return -1;
1625         }
1626         softmix_data->bridge = bridge;
1627         ast_mutex_init(&softmix_data->lock);
1628         ast_cond_init(&softmix_data->cond, NULL);
1629         softmix_data->timer = ast_timer_open();
1630         if (!softmix_data->timer) {
1631                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1632                 softmix_bridge_data_destroy(softmix_data);
1633                 return -1;
1634         }
1635         /* start at minimum rate, let it grow from there */
1636         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1637         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1638
1639 #ifdef BINAURAL_RENDERING
1640         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1641                         softmix_data->internal_mixing_interval);
1642 #endif
1643
1644         bridge->tech_pvt = softmix_data;
1645
1646         /* Start the mixing thread. */
1647         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1648                 softmix_data)) {
1649                 softmix_data->thread = AST_PTHREADT_NULL;
1650                 softmix_bridge_data_destroy(softmix_data);
1651                 bridge->tech_pvt = NULL;
1652                 return -1;
1653         }
1654
1655         return 0;
1656 }
1657
1658 /*!
1659  * \internal
1660  * \brief Request the softmix mixing thread stop.
1661  * \since 12.0.0
1662  *
1663  * \param bridge Which bridge is being stopped.
1664  *
1665  * \return Nothing
1666  */
1667 static void softmix_bridge_stop(struct ast_bridge *bridge)
1668 {
1669         struct softmix_bridge_data *softmix_data;
1670
1671         softmix_data = bridge->tech_pvt;
1672         if (!softmix_data) {
1673                 return;
1674         }
1675
1676         ast_mutex_lock(&softmix_data->lock);
1677         softmix_data->stop = 1;
1678         ast_mutex_unlock(&softmix_data->lock);
1679 }
1680
1681 /*! \brief Function called when a bridge is destroyed */
1682 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1683 {
1684         struct softmix_bridge_data *softmix_data;
1685         pthread_t thread;
1686
1687         softmix_data = bridge->tech_pvt;
1688         if (!softmix_data) {
1689                 return;
1690         }
1691
1692         /* Stop the mixing thread. */
1693         ast_mutex_lock(&softmix_data->lock);
1694         softmix_data->stop = 1;
1695         ast_cond_signal(&softmix_data->cond);
1696         thread = softmix_data->thread;
1697         softmix_data->thread = AST_PTHREADT_NULL;
1698         ast_mutex_unlock(&softmix_data->lock);
1699         if (thread != AST_PTHREADT_NULL) {
1700                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1701                 pthread_join(thread, NULL);
1702         }
1703 #ifdef BINAURAL_RENDERING
1704         free_convolve_data(&softmix_data->convolve);
1705 #endif
1706         softmix_bridge_data_destroy(softmix_data);
1707         bridge->tech_pvt = NULL;
1708 }
1709
1710 /*!
1711  * \brief Map a source stream to all of its destination streams.
1712  *
1713  * \param source_stream_name Name of the source stream
1714  * \param source_channel_name Name of channel where the source stream originates
1715  * \param bridge_stream_position The slot in the bridge where source video will come from
1716  * \param participants The bridge_channels in the bridge
1717  */
1718 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1719         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1720 {
1721         struct ast_bridge_channel *participant;
1722
1723         AST_LIST_TRAVERSE(participants, participant, entry) {
1724                 int i;
1725                 struct ast_stream_topology *topology;
1726
1727                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1728                         continue;
1729                 }
1730
1731                 ast_bridge_channel_lock(participant);
1732                 ast_channel_lock(participant->chan);
1733                 topology = ast_channel_get_stream_topology(participant->chan);
1734
1735                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1736                         struct ast_stream *stream;
1737
1738                         stream = ast_stream_topology_get_stream(topology, i);
1739                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1740                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1741                                 break;
1742                         }
1743                 }
1744                 ast_channel_unlock(participant->chan);
1745                 ast_bridge_channel_unlock(participant);
1746         }
1747 }
1748
1749 /*!
1750  * \brief stream_topology_changed callback
1751  *
1752  * For most video modes, nothing beyond the ordinary is required.
1753  * For the SFU case, though, we need to completely remap the streams
1754  * in order to ensure video gets directed where it is expected to go.
1755  *
1756  * \param bridge The bridge
1757  * \param bridge_channel Channel whose topology has changed
1758  */
1759 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1760 {
1761         struct ast_bridge_channel *participant;
1762         struct ast_vector_int media_types;
1763         int nths[AST_MEDIA_TYPE_END] = {0};
1764
1765         switch (bridge->softmix.video_mode.mode) {
1766         case AST_BRIDGE_VIDEO_MODE_NONE:
1767         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1768         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1769         default:
1770                 ast_bridge_channel_stream_map(bridge_channel);
1771                 return;
1772         case AST_BRIDGE_VIDEO_MODE_SFU:
1773                 break;
1774         }
1775
1776         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1777
1778         /* First traversal: re-initialize all of the participants' stream maps */
1779         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1780                 ast_bridge_channel_lock(participant);
1781                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1782                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1783                 ast_bridge_channel_unlock(participant);
1784         }
1785
1786         /* Second traversal: Map specific video channels from their source to their destinations.
1787          *
1788          * This is similar to what is done in ast_stream_topology_map(),
1789          * except that video channels are handled differently.  Each video
1790          * source has it's own unique index on the bridge.  This way, a
1791          * particular channel's source video can be distributed to the
1792          * appropriate destination streams on the other channels.
1793          */
1794         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1795                 int i;
1796                 struct ast_stream_topology *topology;
1797
1798                 ast_bridge_channel_lock(participant);
1799                 ast_channel_lock(participant->chan);
1800
1801                 topology = ast_channel_get_stream_topology(participant->chan);
1802                 if (topology) {
1803                         /*
1804                          * Sigh.  We have to clone to avoid deadlock in
1805                          * map_source_to_destinations() because topology
1806                          * is not an ao2 object.
1807                          */
1808                         topology = ast_stream_topology_clone(topology);
1809                 }
1810                 if (!topology) {
1811                         /* Oh, my, we are in trouble. */
1812                         ast_channel_unlock(participant->chan);
1813                         ast_bridge_channel_unlock(participant);
1814                         continue;
1815                 }
1816
1817                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1818                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1819
1820                         if (is_video_source(stream)) {
1821                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1822                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1823                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1824                                 /*
1825                                  * Unlock the channel and participant to prevent
1826                                  * potential deadlock in map_source_to_destinations().
1827                                  */
1828                                 ast_channel_unlock(participant->chan);
1829                                 ast_bridge_channel_unlock(participant);
1830                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1831                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1832                                 ast_bridge_channel_lock(participant);
1833                                 ast_channel_lock(participant->chan);
1834                         } else if (is_video_dest(stream, NULL, NULL)) {
1835                                 /* We expect to never read media from video destination channels, but just
1836                                  * in case, we should set their to_bridge value to -1.
1837                                  */
1838                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1839                         } else {
1840                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1841                                  * be factored out in some way
1842                                  */
1843                                 enum ast_media_type type = ast_stream_get_type(stream);
1844                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1845                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1846
1847                                 if (index == -1) {
1848                                         AST_VECTOR_APPEND(&media_types, type);
1849                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1850                                 }
1851
1852                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1853                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1854                         }
1855                 }
1856
1857                 ast_stream_topology_free(topology);
1858
1859                 ast_channel_unlock(participant->chan);
1860                 ast_bridge_channel_unlock(participant);
1861         }
1862 }
1863
1864 static struct ast_bridge_technology softmix_bridge = {
1865         .name = "softmix",
1866         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1867         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1868         .create = softmix_bridge_create,
1869         .stop = softmix_bridge_stop,
1870         .destroy = softmix_bridge_destroy,
1871         .join = softmix_bridge_join,
1872         .leave = softmix_bridge_leave,
1873         .unsuspend = softmix_bridge_unsuspend,
1874         .write = softmix_bridge_write,
1875         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1876 };
1877
1878 #ifdef TEST_FRAMEWORK
1879 struct stream_parameters {
1880         const char *name;
1881         const char *formats;
1882         enum ast_media_type type;
1883 };
1884
1885 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1886 {
1887         struct ast_stream_topology *topology;
1888         size_t i;
1889
1890         topology = ast_stream_topology_alloc();
1891         if (!topology) {
1892                 return NULL;
1893         }
1894
1895         for (i = 0; i < num_streams; ++i) {
1896                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1897                 struct ast_stream *stream;
1898
1899                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1900                 if (!caps) {
1901                         goto fail;
1902                 }
1903                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1904                         goto fail;
1905                 }
1906                 stream = ast_stream_alloc(params[i].name, params[i].type);
1907                 if (!stream) {
1908                         goto fail;
1909                 }
1910                 ast_stream_set_formats(stream, caps);
1911                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1912                         ast_stream_free(stream);
1913                         goto fail;
1914                 }
1915         }
1916
1917         return topology;
1918
1919 fail:
1920         ast_stream_topology_free(topology);
1921         return NULL;
1922 }
1923
1924 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1925         const struct stream_parameters *params)
1926 {
1927         struct ast_format_cap *stream_caps;
1928         struct ast_format_cap *params_caps;
1929
1930         if (ast_stream_get_type(stream) != params->type) {
1931                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1932                         ast_codec_media_type2str(params->type),
1933                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1934                 return -1;
1935         }
1936         if (strcmp(ast_stream_get_name(stream), params->name)) {
1937                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1938                         params->name, ast_stream_get_name(stream));
1939                 return -1;
1940         }
1941
1942         stream_caps = ast_stream_get_formats(stream);
1943         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1944         if (!params_caps) {
1945                 ast_test_status_update(test, "Allocation error on capabilities\n");
1946                 return -1;
1947         }
1948         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
1949
1950         if (ast_format_cap_identical(stream_caps, params_caps)) {
1951                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
1952                         ast_stream_get_name(stream));
1953                 ao2_cleanup(params_caps);
1954                 return -1;
1955         }
1956
1957         ao2_cleanup(params_caps);
1958         return 0;
1959 }
1960
1961 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
1962         const struct stream_parameters *params, size_t num_streams)
1963 {
1964         int i;
1965
1966         if (ast_stream_topology_get_count(topology) < num_streams) {
1967                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
1968                         ast_stream_topology_get_count(topology), num_streams);
1969                 return -1;
1970         }
1971
1972         for (i = 0; i < ARRAY_LEN(params); ++i) {
1973                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
1974                         return -1;
1975                 }
1976         }
1977
1978         return 0;
1979 }
1980
1981 AST_TEST_DEFINE(sfu_append_source_streams)
1982 {
1983         enum ast_test_result_state res = AST_TEST_FAIL;
1984         static const struct stream_parameters bob_streams[] = {
1985                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1986                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1987         };
1988         static const struct stream_parameters alice_streams[] = {
1989                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
1990                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
1991         };
1992         static const struct stream_parameters alice_dest_stream = {
1993                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
1994         };
1995         static const struct stream_parameters bob_dest_stream = {
1996                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
1997         };
1998         struct ast_stream_topology *topology_alice = NULL;
1999         struct ast_stream_topology *topology_bob = NULL;
2000
2001         switch (cmd) {
2002         case TEST_INIT:
2003                 info->name = "sfu_append_source_streams";
2004                 info->category = "/bridges/bridge_softmix/";
2005                 info->summary = "Test appending of video streams";
2006                 info->description =
2007                         "This tests does stuff.";
2008                 return AST_TEST_NOT_RUN;
2009         case TEST_EXECUTE:
2010                 break;
2011         }
2012
2013         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2014         if (!topology_alice) {
2015                 goto end;
2016         }
2017
2018         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2019         if (!topology_bob) {
2020                 goto end;
2021         }
2022
2023         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2024                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2025                 goto end;
2026         }
2027
2028         if (ast_stream_topology_get_count(topology_alice) != 3) {
2029                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2030                         ast_stream_topology_get_count(topology_alice), 3);
2031                 goto end;
2032         }
2033
2034         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2035                 goto end;
2036         }
2037
2038         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2039                 goto end;
2040         }
2041
2042         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2043                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2044                 goto end;
2045         }
2046
2047         if (ast_stream_topology_get_count(topology_bob) != 3) {
2048                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2049                         ast_stream_topology_get_count(topology_bob), 3);
2050                 goto end;
2051         }
2052
2053         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2054                 goto end;
2055         }
2056
2057         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2058                 goto end;
2059         }
2060
2061         res = AST_TEST_PASS;
2062
2063 end:
2064         ast_stream_topology_free(topology_alice);
2065         ast_stream_topology_free(topology_bob);
2066         return res;
2067 }
2068
2069 AST_TEST_DEFINE(sfu_remove_destination_streams)
2070 {
2071         enum ast_test_result_state res = AST_TEST_FAIL;
2072         static const struct stream_parameters params[] = {
2073                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2074                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2075                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2076                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2077         };
2078         static const struct {
2079                 const char *channel_name;
2080                 int num_streams;
2081                 int params_index[4];
2082         } removal_results[] = {
2083                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2084                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2085                 { "", 4, { 0, 1, 2, 3 }, },
2086         };
2087         struct ast_stream_topology *orig = NULL;
2088         int i;
2089
2090         switch (cmd) {
2091         case TEST_INIT:
2092                 info->name = "sfu_remove_destination_streams";
2093                 info->category = "/bridges/bridge_softmix/";
2094                 info->summary = "Test removal of destination video streams";
2095                 info->description =
2096                         "This tests does stuff.";
2097                 return AST_TEST_NOT_RUN;
2098         case TEST_EXECUTE:
2099                 break;
2100         }
2101
2102         orig = build_topology(params, ARRAY_LEN(params));
2103         if (!orig) {
2104                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2105                 goto end;
2106         }
2107
2108         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2109                 int j;
2110
2111                 remove_destination_streams(orig, removal_results[i].channel_name);
2112
2113                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2114                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2115                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2116                         goto end;
2117                 }
2118
2119                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2120                         struct ast_stream *actual;
2121                         struct ast_stream *expected;
2122                         int orig_index;
2123
2124                         actual = ast_stream_topology_get_stream(orig, j);
2125
2126                         orig_index = removal_results[i].params_index[j];
2127                         expected = ast_stream_topology_get_stream(orig, orig_index);
2128
2129                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2130                                 ast_stream_get_formats(expected))) {
2131                                 struct ast_str *expected_str;
2132                                 struct ast_str *actual_str;
2133
2134                                 expected_str = ast_str_alloca(64);
2135                                 actual_str = ast_str_alloca(64);
2136
2137                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2138                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2139                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2140                                 goto end;
2141                         }
2142
2143                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2144                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2145                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2146                                 goto end;
2147                         }
2148                 }
2149         }
2150
2151         res = AST_TEST_PASS;
2152
2153 end:
2154         ast_stream_topology_free(orig);
2155         return res;
2156 }
2157
2158 #endif
2159
2160 static int unload_module(void)
2161 {
2162         ast_bridge_technology_unregister(&softmix_bridge);
2163         AST_TEST_UNREGISTER(sfu_append_source_streams);
2164         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2165         return 0;
2166 }
2167
2168 static int load_module(void)
2169 {
2170         if (ast_bridge_technology_register(&softmix_bridge)) {
2171                 unload_module();
2172                 return AST_MODULE_LOAD_DECLINE;
2173         }
2174         AST_TEST_REGISTER(sfu_append_source_streams);
2175         AST_TEST_REGISTER(sfu_remove_destination_streams);
2176         return AST_MODULE_LOAD_SUCCESS;
2177 }
2178
2179 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");