8de88f257ca88350108e2bcab9299255a2992646
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                                                   and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO &&
422                 strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
423                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
424                 return 1;
425         }
426
427         return 0;
428 }
429
430 /*!
431  * \brief Determine if a stream is a video destination stream.
432  *
433  * A source channel name can be provided to narrow this to a destination stream
434  * for a particular source channel. Further, a source stream name can be provided
435  * to narrow this to a particular source stream's destination. However, empty strings
436  * can be provided to match any destination video stream, regardless of source channel
437  * or source stream.
438  *
439  * \param stream The stream to test
440  * \param source_channel_name The name of a source video channel to match
441  * \param source_stream_name The name of the source video stream to match
442  * \retval 1 The stream is a video destination stream
443  * \retval 0 The stream is not a video destination stream
444  */
445 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
446         const char *source_stream_name)
447 {
448         char *dest_video_name;
449         size_t dest_video_name_len;
450
451         if (ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
452                 return 0;
453         }
454
455         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
456
457         if (!ast_strlen_zero(source_channel_name)) {
458                 dest_video_name_len += strlen(source_channel_name) + 1;
459                 if (!ast_strlen_zero(source_stream_name)) {
460                         dest_video_name_len += strlen(source_stream_name) + 1;
461                 }
462         }
463         dest_video_name = ast_alloca(dest_video_name_len);
464
465         if (!ast_strlen_zero(source_channel_name)) {
466                 if (!ast_strlen_zero(source_stream_name)) {
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 } else {
473                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                                 source_channel_name);
476                         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
477                 }
478         } else {
479                 snprintf(dest_video_name, dest_video_name_len, "%s",
480                         SOFTBRIDGE_VIDEO_DEST_PREFIX);
481                 return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
482         }
483
484         return 0;
485 }
486
487 static int append_source_streams(struct ast_stream_topology *dest,
488         const char *channel_name,
489         const struct ast_stream_topology *source)
490 {
491         int i;
492
493         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
494                 struct ast_stream *stream;
495                 struct ast_stream *stream_clone;
496                 char *stream_clone_name = NULL;
497
498                 stream = ast_stream_topology_get_stream(source, i);
499                 if (!is_video_source(stream)) {
500                         continue;
501                 }
502
503                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
504                         channel_name, ast_stream_get_name(stream)) < 0) {
505                         return -1;
506                 }
507
508                 stream_clone = ast_stream_clone(stream, stream_clone_name);
509                 ast_free(stream_clone_name);
510                 if (!stream_clone) {
511                         return -1;
512                 }
513                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
514                         ast_stream_free(stream_clone);
515                         return -1;
516                 }
517         }
518
519         return 0;
520 }
521
522 static int append_all_streams(struct ast_stream_topology *dest,
523         const struct ast_stream_topology *source)
524 {
525         int i;
526         int dest_index = 0;
527
528         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
529                 struct ast_stream *clone;
530                 int added = 0;
531
532                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
533                 if (!clone) {
534                         return -1;
535                 }
536
537                 /* If we can reuse an existing removed stream then do so */
538                 while (dest_index < ast_stream_topology_get_count(dest)) {
539                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
540
541                         dest_index++;
542
543                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
544                                 /* This cannot fail because dest_index - 1 is less than the
545                                  * current count in dest. */
546                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
547                                 added = 1;
548                                 break;
549                         }
550                 }
551
552                 /* If no removed stream exists that we took the place of append the stream */
553                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
554                         ast_stream_free(clone);
555                         return -1;
556                 }
557         }
558
559         return 0;
560 }
561
562 /*!
563  * \brief Issue channel stream topology change requests.
564  *
565  * When in SFU mode, each participant needs to be able to
566  * send video directly to other participants in the bridge.
567  * This means that all participants need to have their topologies
568  * updated. The joiner needs to have destination streams for
569  * all current participants, and the current participants need
570  * to have destinations streams added for the joiner's sources.
571  *
572  * \param joiner The channel that is joining the softmix bridge
573  * \param participants The current participants in the softmix bridge
574  */
575 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
576 {
577         struct ast_stream_topology *joiner_video = NULL;
578         struct ast_bridge_channel *participant;
579         int res;
580         struct softmix_channel *sc;
581
582         joiner_video = ast_stream_topology_alloc();
583         if (!joiner_video) {
584                 return;
585         }
586
587         sc = joiner->tech_pvt;
588
589         ast_channel_lock(joiner->chan);
590         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
591         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
592         ast_channel_unlock(joiner->chan);
593
594         if (res || !sc->topology) {
595                 goto cleanup;
596         }
597
598         AST_LIST_TRAVERSE(participants, participant, entry) {
599                 if (participant == joiner) {
600                         continue;
601                 }
602                 ast_channel_lock(participant->chan);
603                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
604                                 ast_channel_get_stream_topology(participant->chan));
605                 ast_channel_unlock(participant->chan);
606                 if (res) {
607                         goto cleanup;
608                 }
609         }
610
611         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
612
613         AST_LIST_TRAVERSE(participants, participant, entry) {
614                 if (participant == joiner) {
615                         continue;
616                 }
617
618                 sc = participant->tech_pvt;
619                 if (append_all_streams(sc->topology, joiner_video)) {
620                         goto cleanup;
621                 }
622                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
623         }
624
625 cleanup:
626         ast_stream_topology_free(joiner_video);
627 }
628
629 /*! \brief Function called when a channel is joined into the bridge */
630 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
631 {
632         struct softmix_channel *sc;
633         struct softmix_bridge_data *softmix_data;
634         int set_binaural = 0;
635         /*
636          * If false, the channel will be convolved, but since it is a non stereo channel, output
637          * will be mono.
638          */
639         int skip_binaural_output = 1;
640         int pos_id;
641         int is_announcement = 0;
642         int samplerate_change;
643
644         softmix_data = bridge->tech_pvt;
645         if (!softmix_data) {
646                 return -1;
647         }
648
649         /* Create a new softmix_channel structure and allocate various things on it */
650         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
651                 return -1;
652         }
653
654         samplerate_change = softmix_data->internal_rate;
655         pos_id = -1;
656         if (bridge->softmix.binaural_active) {
657                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
658                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
659                         if (set_binaural) {
660                                 softmix_data->internal_rate = samplerate_change;
661                         }
662                         skip_binaural_output = 0;
663                 } else {
664                         is_announcement = 1;
665                 }
666                 if (set_binaural) {
667                         softmix_data->convolve.binaural_active = 1;
668                 }
669                 if (!skip_binaural_output)      {
670                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
671                         if (pos_id == -1) {
672                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
673                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
674                                                 ast_channel_name(bridge_channel->chan));
675                                 return -1;
676                         }
677                 }
678         }
679
680         /* Can't forget the lock */
681         ast_mutex_init(&sc->lock);
682
683         /* Can't forget to record our pvt structure within the bridged channel structure */
684         bridge_channel->tech_pvt = sc;
685
686         set_softmix_bridge_data(softmix_data->internal_rate,
687                 softmix_data->internal_mixing_interval
688                         ? softmix_data->internal_mixing_interval
689                         : DEFAULT_SOFTMIX_INTERVAL,
690                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
691
692         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
693                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
694         }
695
696         softmix_poke_thread(softmix_data);
697         return 0;
698 }
699
700 static void remove_destination_streams(struct ast_stream_topology *topology,
701         const char *channel_name)
702 {
703         int i;
704
705         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
706                 struct ast_stream *stream;
707
708                 stream = ast_stream_topology_get_stream(topology, i);
709
710                 if (is_video_dest(stream, channel_name, NULL)) {
711                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
712                 }
713         }
714 }
715
716 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
717 {
718         struct ast_bridge_channel *participant;
719         struct softmix_channel *sc;
720
721         AST_LIST_TRAVERSE(participants, participant, entry) {
722                 sc = participant->tech_pvt;
723                 remove_destination_streams(sc->topology, ast_channel_name(leaver->chan));
724                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
725         }
726
727         sc = leaver->tech_pvt;
728         remove_destination_streams(sc->topology, "");
729         ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
730
731         return 0;
732 }
733
734 /*! \brief Function called when a channel leaves the bridge */
735 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
736 {
737         struct softmix_channel *sc;
738         struct softmix_bridge_data *softmix_data;
739
740         softmix_data = bridge->tech_pvt;
741         sc = bridge_channel->tech_pvt;
742         if (!sc) {
743                 return;
744         }
745
746         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
747                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
748         }
749
750         if (bridge->softmix.binaural_active) {
751                 if (sc->binaural) {
752                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
753                                         softmix_data->default_sample_size);
754                 }
755         }
756
757         bridge_channel->tech_pvt = NULL;
758
759         ast_stream_topology_free(sc->topology);
760
761         /* Drop mutex lock */
762         ast_mutex_destroy(&sc->lock);
763
764         /* Drop the factory */
765         ast_slinfactory_destroy(&sc->factory);
766
767         /* Drop any formats on the frames */
768         ao2_cleanup(sc->write_frame.subclass.format);
769
770         /* Drop the DSP */
771         ast_dsp_free(sc->dsp);
772
773         /* Eep! drop ourselves */
774         ast_free(sc);
775 }
776
777 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
778 {
779         struct ast_bridge_channel *cur;
780
781         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
782                 if (cur->suspended) {
783                         continue;
784                 }
785                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
786                         ast_bridge_channel_queue_frame(cur, frame);
787                         break;
788                 }
789         }
790 }
791
792 /*!
793  * \internal
794  * \brief Determine what to do with a video frame.
795  * \since 12.0.0
796  *
797  * \param bridge Which bridge is getting the frame
798  * \param bridge_channel Which channel is writing the frame.
799  * \param frame What is being written.
800  *
801  * \return Nothing
802  */
803 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
804 {
805         struct softmix_channel *sc;
806         int video_src_priority;
807
808         /* Determine if the video frame should be distributed or not */
809         switch (bridge->softmix.video_mode.mode) {
810         case AST_BRIDGE_VIDEO_MODE_NONE:
811                 break;
812         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
813                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
814                 if (video_src_priority == 1) {
815                         /* Pass to me and everyone else. */
816                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
817                 }
818                 break;
819         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
820                 sc = bridge_channel->tech_pvt;
821                 ast_mutex_lock(&sc->lock);
822                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
823                         sc->video_talker.energy_average,
824                         frame->subclass.frame_ending);
825                 ast_mutex_unlock(&sc->lock);
826                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
827                 if (video_src_priority == 1) {
828                         int num_src = ast_bridge_number_video_src(bridge);
829                         int echo = num_src > 1 ? 0 : 1;
830
831                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
832                 } else if (video_src_priority == 2) {
833                         softmix_pass_video_top_priority(bridge, frame);
834                 }
835                 break;
836         case AST_BRIDGE_VIDEO_MODE_SFU:
837                 /* Nothing special to do here, the bridge channel stream map will ensure the
838                  * video goes everywhere it needs to
839                  */
840                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
841                 break;
842         }
843 }
844
845 /*!
846  * \internal
847  * \brief Determine what to do with a voice frame.
848  * \since 12.0.0
849  *
850  * \param bridge Which bridge is getting the frame
851  * \param bridge_channel Which channel is writing the frame.
852  * \param frame What is being written.
853  *
854  * \return Nothing
855  */
856 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
857 {
858         struct softmix_channel *sc = bridge_channel->tech_pvt;
859         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
860         int totalsilence = 0;
861         int cur_energy = 0;
862         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
863                 bridge_channel->tech_args.silence_threshold :
864                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
865         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
866
867         /* Write the frame into the conference */
868         ast_mutex_lock(&sc->lock);
869
870         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
871                 /*
872                  * The incoming frame is not the expected format.  Update
873                  * the channel's translation path to get us slinear from
874                  * the new format for the next frame.
875                  *
876                  * There is the possibility that this frame is an old slinear
877                  * rate frame that was in flight when the softmix bridge
878                  * changed rates.  If so it will self correct on subsequent
879                  * frames.
880                  */
881                 ast_channel_lock(bridge_channel->chan);
882                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
883                         ast_channel_name(bridge_channel->chan),
884                         ast_format_get_name(frame->subclass.format),
885                         ast_format_get_name(sc->read_slin_format));
886                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
887                         sc->read_slin_format);
888                 ast_channel_unlock(bridge_channel->chan);
889         }
890
891         /* The channel will be leaving soon if there is no dsp. */
892         if (sc->dsp) {
893                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
894         }
895
896         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
897                 int cur_slot = sc->video_talker.energy_history_cur_slot;
898
899                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
900                 sc->video_talker.energy_accum += cur_energy;
901                 sc->video_talker.energy_history[cur_slot] = cur_energy;
902                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
903                 sc->video_talker.energy_history_cur_slot++;
904                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
905                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
906                 }
907         }
908
909         if (totalsilence < silence_threshold) {
910                 if (!sc->talking) {
911                         update_talking = 1;
912                 }
913                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
914         } else {
915                 if (sc->talking) {
916                         update_talking = 0;
917                 }
918                 sc->talking = 0;
919         }
920
921         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
922          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
923          * the audio by flushing the buffer before adding new audio in. */
924         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
925                 ast_slinfactory_flush(&sc->factory);
926         }
927
928         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
929          * is not determined to be talking. */
930         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
931                 ast_slinfactory_feed(&sc->factory, frame);
932         }
933
934         /* Alllll done */
935         ast_mutex_unlock(&sc->lock);
936
937         if (update_talking != -1) {
938                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
939         }
940 }
941
942 static int remove_all_original_streams(struct ast_stream_topology *dest,
943         const struct ast_stream_topology *source,
944         const struct ast_stream_topology *original)
945 {
946         int i;
947
948         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
949                 struct ast_stream *stream;
950                 int original_index;
951
952                 stream = ast_stream_topology_get_stream(source, i);
953
954                 /* Mark the existing stream as removed so we get a new one, this will get
955                  * reused on a subsequent renegotiation.
956                  */
957                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
958                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
959
960                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
961                                 struct ast_stream *removed;
962
963                                 /* Since the participant is still going to be in the bridge we
964                                  * change the name so that routing does not attempt to route video
965                                  * to this stream.
966                                  */
967                                 removed = ast_stream_clone(stream, "removed");
968                                 if (!removed) {
969                                         return -1;
970                                 }
971
972                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
973
974                                 /* The destination topology can only ever contain the same, or more,
975                                  * streams than the original so this is safe.
976                                  */
977                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
978                                         ast_stream_free(removed);
979                                         return -1;
980                                 }
981
982                                 break;
983                         }
984                 }
985         }
986
987         return 0;
988 }
989
990 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
991 {
992         struct ast_stream_topology *source_video = NULL;
993         struct ast_bridge_channel *participant;
994         int res;
995
996         source_video = ast_stream_topology_alloc();
997         if (!source_video) {
998                 return;
999         }
1000
1001         ast_channel_lock(source->chan);
1002         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
1003         ast_channel_unlock(source->chan);
1004         if (res) {
1005                 goto cleanup;
1006         }
1007
1008         AST_LIST_TRAVERSE(participants, participant, entry) {
1009                 struct ast_stream_topology *original_topology;
1010                 struct softmix_channel *sc;
1011
1012                 if (participant == source) {
1013                         continue;
1014                 }
1015
1016                 sc = participant->tech_pvt;
1017
1018                 original_topology = ast_stream_topology_clone(sc->topology);
1019                 if (!original_topology) {
1020                         goto cleanup;
1021                 }
1022
1023                 /* We add all the source streams back in, if any removed streams are already present they will
1024                  * get used first followed by appending new ones.
1025                  */
1026                 if (append_all_streams(sc->topology, source_video)) {
1027                         ast_stream_topology_free(original_topology);
1028                         goto cleanup;
1029                 }
1030
1031                 /* And the original existing streams get marked as removed. This causes the remote side to see
1032                  * a new stream for the source streams.
1033                  */
1034                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1035                         ast_stream_topology_free(original_topology);
1036                         goto cleanup;
1037                 }
1038
1039                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1040                 ast_stream_topology_free(original_topology);
1041         }
1042
1043 cleanup:
1044         ast_stream_topology_free(source_video);
1045 }
1046
1047 /*!
1048  * \internal
1049  * \brief Determine what to do with a control frame.
1050  * \since 12.0.0
1051  *
1052  * \param bridge Which bridge is getting the frame
1053  * \param bridge_channel Which channel is writing the frame.
1054  * \param frame What is being written.
1055  *
1056  * \retval 0 Frame accepted into the bridge.
1057  * \retval -1 Frame needs to be deferred.
1058  */
1059 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1060 {
1061         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1062
1063         /*
1064          * XXX Softmix needs to use channel roles to determine what to
1065          * do with control frames.
1066          */
1067
1068         switch (frame->subclass.integer) {
1069         case AST_CONTROL_VIDUPDATE:
1070                 if (!bridge->softmix.video_mode.video_update_discard ||
1071                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1072                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1073                         softmix_data->last_video_update = ast_tvnow();
1074                 }
1075                 break;
1076         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1077                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1078                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1079                 }
1080                 break;
1081         default:
1082                 break;
1083         }
1084
1085         return 0;
1086 }
1087
1088 /*!
1089  * \internal
1090  * \brief Determine what to do with a frame written into the bridge.
1091  * \since 12.0.0
1092  *
1093  * \param bridge Which bridge is getting the frame
1094  * \param bridge_channel Which channel is writing the frame.
1095  * \param frame What is being written.
1096  *
1097  * \retval 0 Frame accepted into the bridge.
1098  * \retval -1 Frame needs to be deferred.
1099  *
1100  * \note On entry, bridge is already locked.
1101  */
1102 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1103 {
1104         int res = 0;
1105
1106         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1107                 /* "Accept" the frame and discard it. */
1108                 return 0;
1109         }
1110
1111         /*
1112          * XXX Softmix needs to use channel roles to determine who gets
1113          * what frame.  Possible roles: announcer, recorder, agent,
1114          * supervisor.
1115          */
1116         switch (frame->frametype) {
1117         case AST_FRAME_NULL:
1118                 /* "Accept" the frame and discard it. */
1119                 break;
1120         case AST_FRAME_DTMF_BEGIN:
1121         case AST_FRAME_DTMF_END:
1122                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1123                 break;
1124         case AST_FRAME_VOICE:
1125                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1126                 break;
1127         case AST_FRAME_VIDEO:
1128                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1129                 break;
1130         case AST_FRAME_CONTROL:
1131                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1132                 break;
1133         case AST_FRAME_BRIDGE_ACTION:
1134                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1135                 break;
1136         case AST_FRAME_BRIDGE_ACTION_SYNC:
1137                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1138                 ast_assert(0);
1139         default:
1140                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1141                 /* "Accept" the frame and discard it. */
1142                 break;
1143         }
1144
1145         return res;
1146 }
1147
1148 static void gather_softmix_stats(struct softmix_stats *stats,
1149         const struct softmix_bridge_data *softmix_data,
1150         struct ast_bridge_channel *bridge_channel)
1151 {
1152         int channel_native_rate;
1153
1154         /* Gather stats about channel sample rates. */
1155         ast_channel_lock(bridge_channel->chan);
1156         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1157                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1158         ast_channel_unlock(bridge_channel->chan);
1159
1160         if (stats->highest_supported_rate < channel_native_rate) {
1161                 stats->highest_supported_rate = channel_native_rate;
1162         }
1163         if (softmix_data->internal_rate < channel_native_rate) {
1164                 int i;
1165
1166                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1167                         if (stats->sample_rates[i] == channel_native_rate) {
1168                                 stats->num_channels[i]++;
1169                                 break;
1170                         } else if (!stats->sample_rates[i]) {
1171                                 stats->sample_rates[i] = channel_native_rate;
1172                                 stats->num_channels[i]++;
1173                                 break;
1174                         }
1175                 }
1176                 stats->num_above_internal_rate++;
1177         } else if (softmix_data->internal_rate == channel_native_rate) {
1178                 stats->num_at_internal_rate++;
1179         }
1180 }
1181
1182 /*!
1183  * \internal
1184  * \brief Analyse mixing statistics and change bridges internal rate
1185  * if necessary.
1186  *
1187  * \retval 0, no changes to internal rate
1188  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1189  */
1190 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1191                 struct softmix_bridge_data *softmix_data, int binaural_active)
1192 {
1193         int i;
1194
1195         if (binaural_active) {
1196                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1197         }
1198
1199         /*
1200          * Re-adjust the internal bridge sample rate if
1201          * 1. The bridge's internal sample rate is locked in at a sample
1202          *    rate other than the current sample rate being used.
1203          * 2. two or more channels support a higher sample rate
1204          * 3. no channels support the current sample rate or a higher rate
1205          */
1206         if (stats->locked_rate) {
1207                 /* if the rate is locked by the bridge, only update it if it differs
1208                  * from the current rate we are using. */
1209                 if (softmix_data->internal_rate != stats->locked_rate) {
1210                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1211                                 softmix_data->internal_rate, stats->locked_rate);
1212                         softmix_data->internal_rate = stats->locked_rate;
1213                         return 1;
1214                 }
1215         } else if (stats->num_above_internal_rate >= 2) {
1216                 /* the highest rate is just used as a starting point */
1217                 unsigned int best_rate = stats->highest_supported_rate;
1218                 int best_index = -1;
1219
1220                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1221                         if (stats->num_channels[i]) {
1222                                 break;
1223                         }
1224                         if (2 <= stats->num_channels[i]) {
1225                                 /* Two or more channels support this rate. */
1226                                 if (best_index == -1
1227                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1228                                         /*
1229                                          * best_rate starts out being the first sample rate
1230                                          * greater than the internal sample rate that two or
1231                                          * more channels support.
1232                                          *
1233                                          * or
1234                                          *
1235                                          * There are multiple rates above the internal rate
1236                                          * and this rate is higher than the previous rate two
1237                                          * or more channels support.
1238                                          */
1239                                         best_rate = stats->sample_rates[i];
1240                                         best_index = i;
1241                                 }
1242                         } else if (best_index == -1) {
1243                                 /*
1244                                  * It is possible that multiple channels exist with native sample
1245                                  * rates above the internal sample rate, but none of those channels
1246                                  * have the same rate in common.  In this case, the lowest sample
1247                                  * rate among those channels is picked. Over time as additional
1248                                  * statistic runs are made the internal sample rate number will
1249                                  * adjust to the most optimal sample rate, but it may take multiple
1250                                  * iterations.
1251                                  */
1252                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1253                         }
1254                 }
1255
1256                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1257                         softmix_data->internal_rate, best_rate);
1258                 softmix_data->internal_rate = best_rate;
1259                 return 1;
1260         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1261                 /* In this case, the highest supported rate is actually lower than the internal rate */
1262                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1263                         softmix_data->internal_rate, stats->highest_supported_rate);
1264                 softmix_data->internal_rate = stats->highest_supported_rate;
1265                 return 1;
1266         }
1267         return 0;
1268 }
1269
1270 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1271                 unsigned int starting_num_entries, unsigned int binaural_active)
1272 {
1273         memset(mixing_array, 0, sizeof(*mixing_array));
1274         mixing_array->max_num_entries = starting_num_entries;
1275         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1276                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1277                 return -1;
1278         }
1279         if (binaural_active) {
1280                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1281                                 sizeof(struct convolve_channel_pair *)))) {
1282                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1283                         return -1;
1284                 }
1285         }
1286         return 0;
1287 }
1288
1289 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1290                 unsigned int binaural_active)
1291 {
1292         ast_free(mixing_array->buffers);
1293         if (binaural_active) {
1294                 ast_free(mixing_array->chan_pairs);
1295         }
1296 }
1297
1298 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1299                 unsigned int num_entries, unsigned int binaural_active)
1300 {
1301         int16_t **tmp;
1302
1303         /* give it some room to grow since memory is cheap but allocations can be expensive */
1304         mixing_array->max_num_entries = num_entries;
1305         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1306                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1307                 return -1;
1308         }
1309         if (binaural_active) {
1310                 struct convolve_channel_pair **tmp2;
1311                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1312                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1313                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1314                         return -1;
1315                 }
1316                 mixing_array->chan_pairs = tmp2;
1317         }
1318         mixing_array->buffers = tmp;
1319         return 0;
1320 }
1321
1322 /*!
1323  * \brief Mixing loop.
1324  *
1325  * \retval 0 on success
1326  * \retval -1 on failure
1327  */
1328 static int softmix_mixing_loop(struct ast_bridge *bridge)
1329 {
1330         struct softmix_stats stats = { { 0 }, };
1331         struct softmix_mixing_array mixing_array;
1332         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1333         struct ast_timer *timer;
1334         struct softmix_translate_helper trans_helper;
1335         int16_t buf[MAX_DATALEN];
1336 #ifdef BINAURAL_RENDERING
1337         int16_t bin_buf[MAX_DATALEN];
1338         int16_t ann_buf[MAX_DATALEN];
1339 #endif
1340         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1341         int timingfd;
1342         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1343         unsigned int idx;
1344         unsigned int x;
1345         int res = -1;
1346
1347         timer = softmix_data->timer;
1348         timingfd = ast_timer_fd(timer);
1349         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1350         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1351
1352         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1353         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1354                         bridge->softmix.binaural_active)) {
1355                 goto softmix_cleanup;
1356         }
1357
1358         /*
1359          * XXX Softmix needs to use channel roles to determine who gets
1360          * what audio mixed.
1361          */
1362         while (!softmix_data->stop && bridge->num_active) {
1363                 struct ast_bridge_channel *bridge_channel;
1364                 int timeout = -1;
1365                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1366                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1367                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1368
1369                 if (softmix_datalen > MAX_DATALEN) {
1370                         /* This should NEVER happen, but if it does we need to know about it. Almost
1371                          * all the memcpys used during this process depend on this assumption.  Rather
1372                          * than checking this over and over again through out the code, this single
1373                          * verification is done on each iteration. */
1374                         ast_log(LOG_WARNING,
1375                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1376                                 bridge->uniqueid);
1377                         goto softmix_cleanup;
1378                 }
1379
1380                 /* Grow the mixing array buffer as participants are added. */
1381                 if (mixing_array.max_num_entries < bridge->num_channels
1382                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1383                                         bridge->softmix.binaural_active)) {
1384                         goto softmix_cleanup;
1385                 }
1386
1387                 /* init the number of buffers stored in the mixing array to 0.
1388                  * As buffers are added for mixing, this number is incremented. */
1389                 mixing_array.used_entries = 0;
1390
1391                 /* These variables help determine if a rate change is required */
1392                 if (!stat_iteration_counter) {
1393                         memset(&stats, 0, sizeof(stats));
1394                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1395                 }
1396
1397                 /* If the sample rate has changed, update the translator helper */
1398                 if (update_all_rates) {
1399                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1400                 }
1401
1402 #ifdef BINAURAL_RENDERING
1403                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1404 #endif
1405
1406                 /* Go through pulling audio from each factory that has it available */
1407                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1408                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1409
1410                         if (!sc) {
1411                                 /* This channel failed to join successfully. */
1412                                 continue;
1413                         }
1414
1415                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1416                         if (update_all_rates) {
1417                                 set_softmix_bridge_data(softmix_data->internal_rate,
1418                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1419                         }
1420
1421                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1422                         if (!stat_iteration_counter) {
1423                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1424                         }
1425
1426                         /* if the channel is suspended, don't check for audio, but still gather stats */
1427                         if (bridge_channel->suspended) {
1428                                 continue;
1429                         }
1430
1431                         /* Try to get audio from the factory if available */
1432                         ast_mutex_lock(&sc->lock);
1433                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1434 #ifdef BINAURAL_RENDERING
1435                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1436                                                 ast_channel_name(bridge_channel->chan));
1437 #endif
1438                                 mixing_array.used_entries++;
1439                         }
1440                         ast_mutex_unlock(&sc->lock);
1441                 }
1442
1443                 /* mix it like crazy (non binaural channels)*/
1444                 memset(buf, 0, softmix_datalen);
1445                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1446                         for (x = 0; x < softmix_samples; ++x) {
1447                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1448                         }
1449                 }
1450
1451 #ifdef BINAURAL_RENDERING
1452                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1453 #endif
1454
1455                 /* Next step go through removing the channel's own audio and creating a good frame... */
1456                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1457                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1458
1459                         if (!sc || bridge_channel->suspended) {
1460                                 /* This channel failed to join successfully or is suspended. */
1461                                 continue;
1462                         }
1463
1464                         ast_mutex_lock(&sc->lock);
1465
1466                         /* Make SLINEAR write frame from local buffer */
1467                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1468                                 "Replace softmix channel slin format");
1469 #ifdef BINAURAL_RENDERING
1470                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1471                                         && sc->binaural) {
1472                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1473                                                 softmix_samples, buf);
1474                         } else
1475 #endif
1476                         {
1477                                 sc->write_frame.datalen = softmix_datalen;
1478                                 sc->write_frame.samples = softmix_samples;
1479                                 memcpy(sc->final_buf, buf, softmix_datalen);
1480                         }
1481                         /* process the softmix channel's new write audio */
1482                         softmix_process_write_audio(&trans_helper,
1483                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1484                                         softmix_data->default_sample_size);
1485
1486                         ast_mutex_unlock(&sc->lock);
1487
1488                         /* A frame is now ready for the channel. */
1489                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1490                 }
1491
1492                 update_all_rates = 0;
1493                 if (!stat_iteration_counter) {
1494                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1495                                         bridge->softmix.binaural_active);
1496                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1497                 }
1498                 stat_iteration_counter--;
1499
1500                 ast_bridge_unlock(bridge);
1501                 /* cleanup any translation frame data from the previous mixing iteration. */
1502                 softmix_translate_helper_cleanup(&trans_helper);
1503                 /* Wait for the timing source to tell us to wake up and get things done */
1504                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1505                 if (ast_timer_ack(timer, 1) < 0) {
1506                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1507                                 bridge->uniqueid);
1508                         ast_bridge_lock(bridge);
1509                         goto softmix_cleanup;
1510                 }
1511                 ast_bridge_lock(bridge);
1512
1513                 /* make sure to detect mixing interval changes if they occur. */
1514                 if (bridge->softmix.internal_mixing_interval
1515                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1516                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1517                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1518                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1519                 }
1520         }
1521
1522         res = 0;
1523
1524 softmix_cleanup:
1525         softmix_translate_helper_destroy(&trans_helper);
1526         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1527         return res;
1528 }
1529
1530 /*!
1531  * \internal
1532  * \brief Mixing thread.
1533  * \since 12.0.0
1534  *
1535  * \note The thread does not have its own reference to the
1536  * bridge.  The lifetime of the thread is tied to the lifetime
1537  * of the mixing technology association with the bridge.
1538  */
1539 static void *softmix_mixing_thread(void *data)
1540 {
1541         struct softmix_bridge_data *softmix_data = data;
1542         struct ast_bridge *bridge = softmix_data->bridge;
1543
1544         ast_bridge_lock(bridge);
1545         if (bridge->callid) {
1546                 ast_callid_threadassoc_add(bridge->callid);
1547         }
1548
1549         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1550
1551         while (!softmix_data->stop) {
1552                 if (!bridge->num_active) {
1553                         /* Wait for something to happen to the bridge. */
1554                         ast_bridge_unlock(bridge);
1555                         ast_mutex_lock(&softmix_data->lock);
1556                         if (!softmix_data->stop) {
1557                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1558                         }
1559                         ast_mutex_unlock(&softmix_data->lock);
1560                         ast_bridge_lock(bridge);
1561                         continue;
1562                 }
1563
1564                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1565 #ifndef BINAURAL_RENDERING
1566                         ast_bridge_lock(bridge);
1567                         bridge->softmix.binaural_active = 0;
1568                         ast_bridge_unlock(bridge);
1569                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1570                                         "compiled.\n", bridge->uniqueid);
1571 #else
1572                         /* Set and init binaural data if binaural is activated in the configuration. */
1573                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1574                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1575                                         softmix_data->internal_mixing_interval);
1576                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1577                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1578                                         == -1) {
1579                                 ast_bridge_lock(bridge);
1580                                 bridge->softmix.binaural_active = 0;
1581                                 ast_bridge_unlock(bridge);
1582                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1583                                                 "binaural processing,  Will only process mono audio.\n",
1584                                                 bridge->uniqueid);
1585                         }
1586                         softmix_data->binaural_init = 1;
1587 #endif
1588                 }
1589
1590                 if (softmix_mixing_loop(bridge)) {
1591                         /*
1592                          * A mixing error occurred.  Sleep and try again later so we
1593                          * won't flood the logs.
1594                          */
1595                         ast_bridge_unlock(bridge);
1596                         sleep(1);
1597                         ast_bridge_lock(bridge);
1598                 }
1599         }
1600
1601         ast_bridge_unlock(bridge);
1602
1603         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1604
1605         return NULL;
1606 }
1607
1608 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1609 {
1610         if (softmix_data->timer) {
1611                 ast_timer_close(softmix_data->timer);
1612                 softmix_data->timer = NULL;
1613         }
1614         ast_mutex_destroy(&softmix_data->lock);
1615         ast_cond_destroy(&softmix_data->cond);
1616         ast_free(softmix_data);
1617 }
1618
1619 /*! \brief Function called when a bridge is created */
1620 static int softmix_bridge_create(struct ast_bridge *bridge)
1621 {
1622         struct softmix_bridge_data *softmix_data;
1623
1624         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1625         if (!softmix_data) {
1626                 return -1;
1627         }
1628         softmix_data->bridge = bridge;
1629         ast_mutex_init(&softmix_data->lock);
1630         ast_cond_init(&softmix_data->cond, NULL);
1631         softmix_data->timer = ast_timer_open();
1632         if (!softmix_data->timer) {
1633                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1634                 softmix_bridge_data_destroy(softmix_data);
1635                 return -1;
1636         }
1637         /* start at minimum rate, let it grow from there */
1638         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1639         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1640
1641 #ifdef BINAURAL_RENDERING
1642         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1643                         softmix_data->internal_mixing_interval);
1644 #endif
1645
1646         bridge->tech_pvt = softmix_data;
1647
1648         /* Start the mixing thread. */
1649         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1650                 softmix_data)) {
1651                 softmix_data->thread = AST_PTHREADT_NULL;
1652                 softmix_bridge_data_destroy(softmix_data);
1653                 bridge->tech_pvt = NULL;
1654                 return -1;
1655         }
1656
1657         return 0;
1658 }
1659
1660 /*!
1661  * \internal
1662  * \brief Request the softmix mixing thread stop.
1663  * \since 12.0.0
1664  *
1665  * \param bridge Which bridge is being stopped.
1666  *
1667  * \return Nothing
1668  */
1669 static void softmix_bridge_stop(struct ast_bridge *bridge)
1670 {
1671         struct softmix_bridge_data *softmix_data;
1672
1673         softmix_data = bridge->tech_pvt;
1674         if (!softmix_data) {
1675                 return;
1676         }
1677
1678         ast_mutex_lock(&softmix_data->lock);
1679         softmix_data->stop = 1;
1680         ast_mutex_unlock(&softmix_data->lock);
1681 }
1682
1683 /*! \brief Function called when a bridge is destroyed */
1684 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1685 {
1686         struct softmix_bridge_data *softmix_data;
1687         pthread_t thread;
1688
1689         softmix_data = bridge->tech_pvt;
1690         if (!softmix_data) {
1691                 return;
1692         }
1693
1694         /* Stop the mixing thread. */
1695         ast_mutex_lock(&softmix_data->lock);
1696         softmix_data->stop = 1;
1697         ast_cond_signal(&softmix_data->cond);
1698         thread = softmix_data->thread;
1699         softmix_data->thread = AST_PTHREADT_NULL;
1700         ast_mutex_unlock(&softmix_data->lock);
1701         if (thread != AST_PTHREADT_NULL) {
1702                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1703                 pthread_join(thread, NULL);
1704         }
1705 #ifdef BINAURAL_RENDERING
1706         free_convolve_data(&softmix_data->convolve);
1707 #endif
1708         softmix_bridge_data_destroy(softmix_data);
1709         bridge->tech_pvt = NULL;
1710 }
1711
1712 /*!
1713  * \brief Map a source stream to all of its destination streams.
1714  *
1715  * \param source_stream_name Name of the source stream
1716  * \param source_channel_name Name of channel where the source stream originates
1717  * \param bridge_stream_position The slot in the bridge where source video will come from
1718  * \param participants The bridge_channels in the bridge
1719  */
1720 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1721         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1722 {
1723         struct ast_bridge_channel *participant;
1724
1725         AST_LIST_TRAVERSE(participants, participant, entry) {
1726                 int i;
1727                 struct ast_stream_topology *topology;
1728
1729                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1730                         continue;
1731                 }
1732
1733                 ast_bridge_channel_lock(participant);
1734                 ast_channel_lock(participant->chan);
1735                 topology = ast_channel_get_stream_topology(participant->chan);
1736
1737                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1738                         struct ast_stream *stream;
1739
1740                         stream = ast_stream_topology_get_stream(topology, i);
1741                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1742                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1743                                 break;
1744                         }
1745                 }
1746                 ast_channel_unlock(participant->chan);
1747                 ast_bridge_channel_unlock(participant);
1748         }
1749 }
1750
1751 /*!
1752  * \brief stream_topology_changed callback
1753  *
1754  * For most video modes, nothing beyond the ordinary is required.
1755  * For the SFU case, though, we need to completely remap the streams
1756  * in order to ensure video gets directed where it is expected to go.
1757  *
1758  * \param bridge The bridge
1759  * \param bridge_channel Channel whose topology has changed
1760  */
1761 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1762 {
1763         struct ast_bridge_channel *participant;
1764         struct ast_vector_int media_types;
1765         int nths[AST_MEDIA_TYPE_END] = {0};
1766
1767         switch (bridge->softmix.video_mode.mode) {
1768         case AST_BRIDGE_VIDEO_MODE_NONE:
1769         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1770         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1771         default:
1772                 ast_bridge_channel_stream_map(bridge_channel);
1773                 return;
1774         case AST_BRIDGE_VIDEO_MODE_SFU:
1775                 break;
1776         }
1777
1778         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1779
1780         /* First traversal: re-initialize all of the participants' stream maps */
1781         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1782                 ast_bridge_channel_lock(participant);
1783                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1784                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1785                 ast_bridge_channel_unlock(participant);
1786         }
1787
1788         /* Second traversal: Map specific video channels from their source to their destinations.
1789          *
1790          * This is similar to what is done in ast_stream_topology_map(),
1791          * except that video channels are handled differently.  Each video
1792          * source has it's own unique index on the bridge.  This way, a
1793          * particular channel's source video can be distributed to the
1794          * appropriate destination streams on the other channels.
1795          */
1796         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1797                 int i;
1798                 struct ast_stream_topology *topology;
1799
1800                 ast_bridge_channel_lock(participant);
1801                 ast_channel_lock(participant->chan);
1802
1803                 topology = ast_channel_get_stream_topology(participant->chan);
1804                 if (topology) {
1805                         /*
1806                          * Sigh.  We have to clone to avoid deadlock in
1807                          * map_source_to_destinations() because topology
1808                          * is not an ao2 object.
1809                          */
1810                         topology = ast_stream_topology_clone(topology);
1811                 }
1812                 if (!topology) {
1813                         /* Oh, my, we are in trouble. */
1814                         ast_channel_unlock(participant->chan);
1815                         ast_bridge_channel_unlock(participant);
1816                         continue;
1817                 }
1818
1819                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1820                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1821
1822                         if (is_video_source(stream)) {
1823                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1824                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1825                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1826                                 /*
1827                                  * Unlock the channel and participant to prevent
1828                                  * potential deadlock in map_source_to_destinations().
1829                                  */
1830                                 ast_channel_unlock(participant->chan);
1831                                 ast_bridge_channel_unlock(participant);
1832                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1833                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1834                                 ast_bridge_channel_lock(participant);
1835                                 ast_channel_lock(participant->chan);
1836                         } else if (is_video_dest(stream, NULL, NULL)) {
1837                                 /* We expect to never read media from video destination channels, but just
1838                                  * in case, we should set their to_bridge value to -1.
1839                                  */
1840                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1841                         } else {
1842                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1843                                  * be factored out in some way
1844                                  */
1845                                 enum ast_media_type type = ast_stream_get_type(stream);
1846                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1847                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1848
1849                                 if (index == -1) {
1850                                         AST_VECTOR_APPEND(&media_types, type);
1851                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1852                                 }
1853
1854                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1855                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1856                         }
1857                 }
1858
1859                 ast_stream_topology_free(topology);
1860
1861                 ast_channel_unlock(participant->chan);
1862                 ast_bridge_channel_unlock(participant);
1863         }
1864 }
1865
1866 static struct ast_bridge_technology softmix_bridge = {
1867         .name = "softmix",
1868         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1869         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1870         .create = softmix_bridge_create,
1871         .stop = softmix_bridge_stop,
1872         .destroy = softmix_bridge_destroy,
1873         .join = softmix_bridge_join,
1874         .leave = softmix_bridge_leave,
1875         .unsuspend = softmix_bridge_unsuspend,
1876         .write = softmix_bridge_write,
1877         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1878 };
1879
1880 #ifdef TEST_FRAMEWORK
1881 struct stream_parameters {
1882         const char *name;
1883         const char *formats;
1884         enum ast_media_type type;
1885 };
1886
1887 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1888 {
1889         struct ast_stream_topology *topology;
1890         size_t i;
1891
1892         topology = ast_stream_topology_alloc();
1893         if (!topology) {
1894                 return NULL;
1895         }
1896
1897         for (i = 0; i < num_streams; ++i) {
1898                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1899                 struct ast_stream *stream;
1900
1901                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1902                 if (!caps) {
1903                         goto fail;
1904                 }
1905                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1906                         goto fail;
1907                 }
1908                 stream = ast_stream_alloc(params[i].name, params[i].type);
1909                 if (!stream) {
1910                         goto fail;
1911                 }
1912                 ast_stream_set_formats(stream, caps);
1913                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1914                         ast_stream_free(stream);
1915                         goto fail;
1916                 }
1917         }
1918
1919         return topology;
1920
1921 fail:
1922         ast_stream_topology_free(topology);
1923         return NULL;
1924 }
1925
1926 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1927         const struct stream_parameters *params)
1928 {
1929         struct ast_format_cap *stream_caps;
1930         struct ast_format_cap *params_caps;
1931
1932         if (ast_stream_get_type(stream) != params->type) {
1933                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1934                         ast_codec_media_type2str(params->type),
1935                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1936                 return -1;
1937         }
1938         if (strcmp(ast_stream_get_name(stream), params->name)) {
1939                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1940                         params->name, ast_stream_get_name(stream));
1941                 return -1;
1942         }
1943
1944         stream_caps = ast_stream_get_formats(stream);
1945         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1946         if (!params_caps) {
1947                 ast_test_status_update(test, "Allocation error on capabilities\n");
1948                 return -1;
1949         }
1950         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
1951
1952         if (ast_format_cap_identical(stream_caps, params_caps)) {
1953                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
1954                         ast_stream_get_name(stream));
1955                 ao2_cleanup(params_caps);
1956                 return -1;
1957         }
1958
1959         ao2_cleanup(params_caps);
1960         return 0;
1961 }
1962
1963 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
1964         const struct stream_parameters *params, size_t num_streams)
1965 {
1966         int i;
1967
1968         if (ast_stream_topology_get_count(topology) < num_streams) {
1969                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
1970                         ast_stream_topology_get_count(topology), num_streams);
1971                 return -1;
1972         }
1973
1974         for (i = 0; i < ARRAY_LEN(params); ++i) {
1975                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
1976                         return -1;
1977                 }
1978         }
1979
1980         return 0;
1981 }
1982
1983 AST_TEST_DEFINE(sfu_append_source_streams)
1984 {
1985         enum ast_test_result_state res = AST_TEST_FAIL;
1986         static const struct stream_parameters bob_streams[] = {
1987                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1988                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1989         };
1990         static const struct stream_parameters alice_streams[] = {
1991                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
1992                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
1993         };
1994         static const struct stream_parameters alice_dest_stream = {
1995                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
1996         };
1997         static const struct stream_parameters bob_dest_stream = {
1998                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
1999         };
2000         struct ast_stream_topology *topology_alice = NULL;
2001         struct ast_stream_topology *topology_bob = NULL;
2002
2003         switch (cmd) {
2004         case TEST_INIT:
2005                 info->name = "sfu_append_source_streams";
2006                 info->category = "/bridges/bridge_softmix/";
2007                 info->summary = "Test appending of video streams";
2008                 info->description =
2009                         "This tests does stuff.";
2010                 return AST_TEST_NOT_RUN;
2011         case TEST_EXECUTE:
2012                 break;
2013         }
2014
2015         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2016         if (!topology_alice) {
2017                 goto end;
2018         }
2019
2020         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2021         if (!topology_bob) {
2022                 goto end;
2023         }
2024
2025         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2026                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2027                 goto end;
2028         }
2029
2030         if (ast_stream_topology_get_count(topology_alice) != 3) {
2031                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2032                         ast_stream_topology_get_count(topology_alice), 3);
2033                 goto end;
2034         }
2035
2036         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2037                 goto end;
2038         }
2039
2040         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2041                 goto end;
2042         }
2043
2044         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2045                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2046                 goto end;
2047         }
2048
2049         if (ast_stream_topology_get_count(topology_bob) != 3) {
2050                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2051                         ast_stream_topology_get_count(topology_bob), 3);
2052                 goto end;
2053         }
2054
2055         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2056                 goto end;
2057         }
2058
2059         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2060                 goto end;
2061         }
2062
2063         res = AST_TEST_PASS;
2064
2065 end:
2066         ast_stream_topology_free(topology_alice);
2067         ast_stream_topology_free(topology_bob);
2068         return res;
2069 }
2070
2071 AST_TEST_DEFINE(sfu_remove_destination_streams)
2072 {
2073         enum ast_test_result_state res = AST_TEST_FAIL;
2074         static const struct stream_parameters params[] = {
2075                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2076                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2077                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2078                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2079         };
2080         static const struct {
2081                 const char *channel_name;
2082                 int num_streams;
2083                 int params_index[4];
2084         } removal_results[] = {
2085                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2086                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2087                 { "", 4, { 0, 1, 2, 3 }, },
2088         };
2089         struct ast_stream_topology *orig = NULL;
2090         int i;
2091
2092         switch (cmd) {
2093         case TEST_INIT:
2094                 info->name = "sfu_remove_destination_streams";
2095                 info->category = "/bridges/bridge_softmix/";
2096                 info->summary = "Test removal of destination video streams";
2097                 info->description =
2098                         "This tests does stuff.";
2099                 return AST_TEST_NOT_RUN;
2100         case TEST_EXECUTE:
2101                 break;
2102         }
2103
2104         orig = build_topology(params, ARRAY_LEN(params));
2105         if (!orig) {
2106                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2107                 goto end;
2108         }
2109
2110         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2111                 int j;
2112
2113                 remove_destination_streams(orig, removal_results[i].channel_name);
2114
2115                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2116                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2117                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2118                         goto end;
2119                 }
2120
2121                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2122                         struct ast_stream *actual;
2123                         struct ast_stream *expected;
2124                         int orig_index;
2125
2126                         actual = ast_stream_topology_get_stream(orig, j);
2127
2128                         orig_index = removal_results[i].params_index[j];
2129                         expected = ast_stream_topology_get_stream(orig, orig_index);
2130
2131                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2132                                 ast_stream_get_formats(expected))) {
2133                                 struct ast_str *expected_str;
2134                                 struct ast_str *actual_str;
2135
2136                                 expected_str = ast_str_alloca(64);
2137                                 actual_str = ast_str_alloca(64);
2138
2139                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2140                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2141                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2142                                 goto end;
2143                         }
2144
2145                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2146                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2147                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2148                                 goto end;
2149                         }
2150                 }
2151         }
2152
2153         res = AST_TEST_PASS;
2154
2155 end:
2156         ast_stream_topology_free(orig);
2157         return res;
2158 }
2159
2160 #endif
2161
2162 static int unload_module(void)
2163 {
2164         ast_bridge_technology_unregister(&softmix_bridge);
2165         AST_TEST_UNREGISTER(sfu_append_source_streams);
2166         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2167         return 0;
2168 }
2169
2170 static int load_module(void)
2171 {
2172         if (ast_bridge_technology_register(&softmix_bridge)) {
2173                 unload_module();
2174                 return AST_MODULE_LOAD_DECLINE;
2175         }
2176         AST_TEST_REGISTER(sfu_append_source_streams);
2177         AST_TEST_REGISTER(sfu_remove_destination_streams);
2178         return AST_MODULE_LOAD_SUCCESS;
2179 }
2180
2181 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");