Merge "bridge_softmix.c: Change remove_destination_streams() return meaning."
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                                                   and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO &&
422                 strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
423                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
424                 return 1;
425         }
426
427         return 0;
428 }
429
430 /*!
431  * \brief Determine if a stream is a video destination stream.
432  *
433  * A source channel name can be provided to narrow this to a destination stream
434  * for a particular source channel. Further, a source stream name can be provided
435  * to narrow this to a particular source stream's destination. However, empty strings
436  * can be provided to match any destination video stream, regardless of source channel
437  * or source stream.
438  *
439  * \param stream The stream to test
440  * \param source_channel_name The name of a source video channel to match
441  * \param source_stream_name The name of the source video stream to match
442  * \retval 1 The stream is a video destination stream
443  * \retval 0 The stream is not a video destination stream
444  */
445 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
446         const char *source_stream_name)
447 {
448         char *dest_video_name;
449         size_t dest_video_name_len;
450
451         if (ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
452                 return 0;
453         }
454
455         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
456
457         if (!ast_strlen_zero(source_channel_name)) {
458                 dest_video_name_len += strlen(source_channel_name) + 1;
459                 if (!ast_strlen_zero(source_stream_name)) {
460                         dest_video_name_len += strlen(source_stream_name) + 1;
461                 }
462         }
463         dest_video_name = ast_alloca(dest_video_name_len);
464
465         if (!ast_strlen_zero(source_channel_name)) {
466                 if (!ast_strlen_zero(source_stream_name)) {
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 } else {
473                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                                 source_channel_name);
476                         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
477                 }
478         } else {
479                 snprintf(dest_video_name, dest_video_name_len, "%s",
480                         SOFTBRIDGE_VIDEO_DEST_PREFIX);
481                 return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
482         }
483
484         return 0;
485 }
486
487 static int append_source_streams(struct ast_stream_topology *dest,
488         const char *channel_name,
489         const struct ast_stream_topology *source)
490 {
491         int i;
492
493         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
494                 struct ast_stream *stream;
495                 struct ast_stream *stream_clone;
496                 char *stream_clone_name = NULL;
497
498                 stream = ast_stream_topology_get_stream(source, i);
499                 if (!is_video_source(stream)) {
500                         continue;
501                 }
502
503                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
504                         channel_name, ast_stream_get_name(stream)) < 0) {
505                         return -1;
506                 }
507
508                 stream_clone = ast_stream_clone(stream, stream_clone_name);
509                 ast_free(stream_clone_name);
510                 if (!stream_clone) {
511                         return -1;
512                 }
513                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
514                         ast_stream_free(stream_clone);
515                         return -1;
516                 }
517         }
518
519         return 0;
520 }
521
522 static int append_all_streams(struct ast_stream_topology *dest,
523         const struct ast_stream_topology *source)
524 {
525         int i;
526         int dest_index = 0;
527
528         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
529                 struct ast_stream *clone;
530                 int added = 0;
531
532                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
533                 if (!clone) {
534                         return -1;
535                 }
536
537                 /* If we can reuse an existing removed stream then do so */
538                 while (dest_index < ast_stream_topology_get_count(dest)) {
539                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
540
541                         dest_index++;
542
543                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
544                                 /* This cannot fail because dest_index - 1 is less than the
545                                  * current count in dest. */
546                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
547                                 added = 1;
548                                 break;
549                         }
550                 }
551
552                 /* If no removed stream exists that we took the place of append the stream */
553                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
554                         ast_stream_free(clone);
555                         return -1;
556                 }
557         }
558
559         return 0;
560 }
561
562 /*!
563  * \brief Issue channel stream topology change requests.
564  *
565  * When in SFU mode, each participant needs to be able to
566  * send video directly to other participants in the bridge.
567  * This means that all participants need to have their topologies
568  * updated. The joiner needs to have destination streams for
569  * all current participants, and the current participants need
570  * to have destinations streams added for the joiner's sources.
571  *
572  * \param joiner The channel that is joining the softmix bridge
573  * \param participants The current participants in the softmix bridge
574  */
575 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
576 {
577         struct ast_stream_topology *joiner_video = NULL;
578         struct ast_bridge_channel *participant;
579         int res;
580         struct softmix_channel *sc;
581
582         joiner_video = ast_stream_topology_alloc();
583         if (!joiner_video) {
584                 return;
585         }
586
587         sc = joiner->tech_pvt;
588
589         ast_channel_lock(joiner->chan);
590         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
591         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
592         ast_channel_unlock(joiner->chan);
593
594         if (res || !sc->topology) {
595                 goto cleanup;
596         }
597
598         AST_LIST_TRAVERSE(participants, participant, entry) {
599                 if (participant == joiner) {
600                         continue;
601                 }
602                 ast_channel_lock(participant->chan);
603                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
604                                 ast_channel_get_stream_topology(participant->chan));
605                 ast_channel_unlock(participant->chan);
606                 if (res) {
607                         goto cleanup;
608                 }
609         }
610
611         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
612
613         AST_LIST_TRAVERSE(participants, participant, entry) {
614                 if (participant == joiner) {
615                         continue;
616                 }
617
618                 sc = participant->tech_pvt;
619                 if (append_all_streams(sc->topology, joiner_video)) {
620                         goto cleanup;
621                 }
622                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
623         }
624
625 cleanup:
626         ast_stream_topology_free(joiner_video);
627 }
628
629 /*! \brief Function called when a channel is joined into the bridge */
630 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
631 {
632         struct softmix_channel *sc;
633         struct softmix_bridge_data *softmix_data;
634         int set_binaural = 0;
635         /*
636          * If false, the channel will be convolved, but since it is a non stereo channel, output
637          * will be mono.
638          */
639         int skip_binaural_output = 1;
640         int pos_id;
641         int is_announcement = 0;
642         int samplerate_change;
643
644         softmix_data = bridge->tech_pvt;
645         if (!softmix_data) {
646                 return -1;
647         }
648
649         /* Create a new softmix_channel structure and allocate various things on it */
650         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
651                 return -1;
652         }
653
654         samplerate_change = softmix_data->internal_rate;
655         pos_id = -1;
656         if (bridge->softmix.binaural_active) {
657                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
658                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
659                         if (set_binaural) {
660                                 softmix_data->internal_rate = samplerate_change;
661                         }
662                         skip_binaural_output = 0;
663                 } else {
664                         is_announcement = 1;
665                 }
666                 if (set_binaural) {
667                         softmix_data->convolve.binaural_active = 1;
668                 }
669                 if (!skip_binaural_output)      {
670                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
671                         if (pos_id == -1) {
672                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
673                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
674                                                 ast_channel_name(bridge_channel->chan));
675                                 ast_free(sc);
676                                 return -1;
677                         }
678                 }
679         }
680
681         /* Can't forget the lock */
682         ast_mutex_init(&sc->lock);
683
684         /* Can't forget to record our pvt structure within the bridged channel structure */
685         bridge_channel->tech_pvt = sc;
686
687         set_softmix_bridge_data(softmix_data->internal_rate,
688                 softmix_data->internal_mixing_interval
689                         ? softmix_data->internal_mixing_interval
690                         : DEFAULT_SOFTMIX_INTERVAL,
691                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
692
693         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
694                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
695         }
696
697         softmix_poke_thread(softmix_data);
698         return 0;
699 }
700
701 static int remove_destination_streams(struct ast_stream_topology *topology,
702         const char *channel_name)
703 {
704         int i;
705         int stream_removed = 0;
706
707         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
708                 struct ast_stream *stream;
709
710                 stream = ast_stream_topology_get_stream(topology, i);
711
712                 if (is_video_dest(stream, channel_name, NULL)) {
713                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
714                         stream_removed = 1;
715                 }
716         }
717         return stream_removed;
718 }
719
720 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
721 {
722         struct ast_bridge_channel *participant;
723         struct softmix_channel *sc;
724
725         AST_LIST_TRAVERSE(participants, participant, entry) {
726                 sc = participant->tech_pvt;
727                 if (!remove_destination_streams(sc->topology, ast_channel_name(leaver->chan))) {
728                         continue;
729                 }
730                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
731         }
732
733         sc = leaver->tech_pvt;
734         if (remove_destination_streams(sc->topology, "")) {
735                 ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
736         }
737
738         return 0;
739 }
740
741 /*! \brief Function called when a channel leaves the bridge */
742 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
743 {
744         struct softmix_channel *sc;
745         struct softmix_bridge_data *softmix_data;
746
747         softmix_data = bridge->tech_pvt;
748         sc = bridge_channel->tech_pvt;
749         if (!sc) {
750                 return;
751         }
752
753         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
754                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
755         }
756
757         if (bridge->softmix.binaural_active) {
758                 if (sc->binaural) {
759                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
760                                         softmix_data->default_sample_size);
761                 }
762         }
763
764         bridge_channel->tech_pvt = NULL;
765
766         ast_stream_topology_free(sc->topology);
767
768         /* Drop mutex lock */
769         ast_mutex_destroy(&sc->lock);
770
771         /* Drop the factory */
772         ast_slinfactory_destroy(&sc->factory);
773
774         /* Drop any formats on the frames */
775         ao2_cleanup(sc->write_frame.subclass.format);
776
777         /* Drop the DSP */
778         ast_dsp_free(sc->dsp);
779
780         /* Eep! drop ourselves */
781         ast_free(sc);
782 }
783
784 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
785 {
786         struct ast_bridge_channel *cur;
787
788         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
789                 if (cur->suspended) {
790                         continue;
791                 }
792                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
793                         ast_bridge_channel_queue_frame(cur, frame);
794                         break;
795                 }
796         }
797 }
798
799 /*!
800  * \internal
801  * \brief Determine what to do with a video frame.
802  * \since 12.0.0
803  *
804  * \param bridge Which bridge is getting the frame
805  * \param bridge_channel Which channel is writing the frame.
806  * \param frame What is being written.
807  *
808  * \return Nothing
809  */
810 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
811 {
812         struct softmix_channel *sc;
813         int video_src_priority;
814
815         /* Determine if the video frame should be distributed or not */
816         switch (bridge->softmix.video_mode.mode) {
817         case AST_BRIDGE_VIDEO_MODE_NONE:
818                 break;
819         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
820                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
821                 if (video_src_priority == 1) {
822                         /* Pass to me and everyone else. */
823                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
824                 }
825                 break;
826         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
827                 sc = bridge_channel->tech_pvt;
828                 ast_mutex_lock(&sc->lock);
829                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
830                         sc->video_talker.energy_average,
831                         frame->subclass.frame_ending);
832                 ast_mutex_unlock(&sc->lock);
833                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
834                 if (video_src_priority == 1) {
835                         int num_src = ast_bridge_number_video_src(bridge);
836                         int echo = num_src > 1 ? 0 : 1;
837
838                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
839                 } else if (video_src_priority == 2) {
840                         softmix_pass_video_top_priority(bridge, frame);
841                 }
842                 break;
843         case AST_BRIDGE_VIDEO_MODE_SFU:
844                 /* Nothing special to do here, the bridge channel stream map will ensure the
845                  * video goes everywhere it needs to
846                  */
847                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
848                 break;
849         }
850 }
851
852 /*!
853  * \internal
854  * \brief Determine what to do with a voice frame.
855  * \since 12.0.0
856  *
857  * \param bridge Which bridge is getting the frame
858  * \param bridge_channel Which channel is writing the frame.
859  * \param frame What is being written.
860  *
861  * \return Nothing
862  */
863 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
864 {
865         struct softmix_channel *sc = bridge_channel->tech_pvt;
866         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
867         int totalsilence = 0;
868         int cur_energy = 0;
869         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
870                 bridge_channel->tech_args.silence_threshold :
871                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
872         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
873
874         /* Write the frame into the conference */
875         ast_mutex_lock(&sc->lock);
876
877         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
878                 /*
879                  * The incoming frame is not the expected format.  Update
880                  * the channel's translation path to get us slinear from
881                  * the new format for the next frame.
882                  *
883                  * There is the possibility that this frame is an old slinear
884                  * rate frame that was in flight when the softmix bridge
885                  * changed rates.  If so it will self correct on subsequent
886                  * frames.
887                  */
888                 ast_channel_lock(bridge_channel->chan);
889                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
890                         ast_channel_name(bridge_channel->chan),
891                         ast_format_get_name(frame->subclass.format),
892                         ast_format_get_name(sc->read_slin_format));
893                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
894                         sc->read_slin_format);
895                 ast_channel_unlock(bridge_channel->chan);
896         }
897
898         /* The channel will be leaving soon if there is no dsp. */
899         if (sc->dsp) {
900                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
901         }
902
903         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
904                 int cur_slot = sc->video_talker.energy_history_cur_slot;
905
906                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
907                 sc->video_talker.energy_accum += cur_energy;
908                 sc->video_talker.energy_history[cur_slot] = cur_energy;
909                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
910                 sc->video_talker.energy_history_cur_slot++;
911                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
912                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
913                 }
914         }
915
916         if (totalsilence < silence_threshold) {
917                 if (!sc->talking) {
918                         update_talking = 1;
919                 }
920                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
921         } else {
922                 if (sc->talking) {
923                         update_talking = 0;
924                 }
925                 sc->talking = 0;
926         }
927
928         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
929          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
930          * the audio by flushing the buffer before adding new audio in. */
931         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
932                 ast_slinfactory_flush(&sc->factory);
933         }
934
935         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
936          * is not determined to be talking. */
937         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
938                 ast_slinfactory_feed(&sc->factory, frame);
939         }
940
941         /* Alllll done */
942         ast_mutex_unlock(&sc->lock);
943
944         if (update_talking != -1) {
945                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
946         }
947 }
948
949 static int remove_all_original_streams(struct ast_stream_topology *dest,
950         const struct ast_stream_topology *source,
951         const struct ast_stream_topology *original)
952 {
953         int i;
954
955         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
956                 struct ast_stream *stream;
957                 int original_index;
958
959                 stream = ast_stream_topology_get_stream(source, i);
960
961                 /* Mark the existing stream as removed so we get a new one, this will get
962                  * reused on a subsequent renegotiation.
963                  */
964                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
965                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
966
967                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
968                                 struct ast_stream *removed;
969
970                                 /* Since the participant is still going to be in the bridge we
971                                  * change the name so that routing does not attempt to route video
972                                  * to this stream.
973                                  */
974                                 removed = ast_stream_clone(stream, "removed");
975                                 if (!removed) {
976                                         return -1;
977                                 }
978
979                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
980
981                                 /* The destination topology can only ever contain the same, or more,
982                                  * streams than the original so this is safe.
983                                  */
984                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
985                                         ast_stream_free(removed);
986                                         return -1;
987                                 }
988
989                                 break;
990                         }
991                 }
992         }
993
994         return 0;
995 }
996
997 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
998 {
999         struct ast_stream_topology *source_video = NULL;
1000         struct ast_bridge_channel *participant;
1001         int res;
1002
1003         source_video = ast_stream_topology_alloc();
1004         if (!source_video) {
1005                 return;
1006         }
1007
1008         ast_channel_lock(source->chan);
1009         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
1010         ast_channel_unlock(source->chan);
1011         if (res) {
1012                 goto cleanup;
1013         }
1014
1015         AST_LIST_TRAVERSE(participants, participant, entry) {
1016                 struct ast_stream_topology *original_topology;
1017                 struct softmix_channel *sc;
1018
1019                 if (participant == source) {
1020                         continue;
1021                 }
1022
1023                 sc = participant->tech_pvt;
1024
1025                 original_topology = ast_stream_topology_clone(sc->topology);
1026                 if (!original_topology) {
1027                         goto cleanup;
1028                 }
1029
1030                 /* We add all the source streams back in, if any removed streams are already present they will
1031                  * get used first followed by appending new ones.
1032                  */
1033                 if (append_all_streams(sc->topology, source_video)) {
1034                         ast_stream_topology_free(original_topology);
1035                         goto cleanup;
1036                 }
1037
1038                 /* And the original existing streams get marked as removed. This causes the remote side to see
1039                  * a new stream for the source streams.
1040                  */
1041                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1042                         ast_stream_topology_free(original_topology);
1043                         goto cleanup;
1044                 }
1045
1046                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1047                 ast_stream_topology_free(original_topology);
1048         }
1049
1050 cleanup:
1051         ast_stream_topology_free(source_video);
1052 }
1053
1054 /*!
1055  * \internal
1056  * \brief Determine what to do with a control frame.
1057  * \since 12.0.0
1058  *
1059  * \param bridge Which bridge is getting the frame
1060  * \param bridge_channel Which channel is writing the frame.
1061  * \param frame What is being written.
1062  *
1063  * \retval 0 Frame accepted into the bridge.
1064  * \retval -1 Frame needs to be deferred.
1065  */
1066 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1067 {
1068         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1069
1070         /*
1071          * XXX Softmix needs to use channel roles to determine what to
1072          * do with control frames.
1073          */
1074
1075         switch (frame->subclass.integer) {
1076         case AST_CONTROL_VIDUPDATE:
1077                 if (!bridge->softmix.video_mode.video_update_discard ||
1078                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1079                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1080                         softmix_data->last_video_update = ast_tvnow();
1081                 }
1082                 break;
1083         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1084                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1085                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1086                 }
1087                 break;
1088         default:
1089                 break;
1090         }
1091
1092         return 0;
1093 }
1094
1095 /*!
1096  * \internal
1097  * \brief Determine what to do with a frame written into the bridge.
1098  * \since 12.0.0
1099  *
1100  * \param bridge Which bridge is getting the frame
1101  * \param bridge_channel Which channel is writing the frame.
1102  * \param frame What is being written.
1103  *
1104  * \retval 0 Frame accepted into the bridge.
1105  * \retval -1 Frame needs to be deferred.
1106  *
1107  * \note On entry, bridge is already locked.
1108  */
1109 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1110 {
1111         int res = 0;
1112
1113         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1114                 /* "Accept" the frame and discard it. */
1115                 return 0;
1116         }
1117
1118         /*
1119          * XXX Softmix needs to use channel roles to determine who gets
1120          * what frame.  Possible roles: announcer, recorder, agent,
1121          * supervisor.
1122          */
1123         switch (frame->frametype) {
1124         case AST_FRAME_NULL:
1125                 /* "Accept" the frame and discard it. */
1126                 break;
1127         case AST_FRAME_DTMF_BEGIN:
1128         case AST_FRAME_DTMF_END:
1129                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1130                 break;
1131         case AST_FRAME_VOICE:
1132                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1133                 break;
1134         case AST_FRAME_VIDEO:
1135                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1136                 break;
1137         case AST_FRAME_CONTROL:
1138                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1139                 break;
1140         case AST_FRAME_BRIDGE_ACTION:
1141                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1142                 break;
1143         case AST_FRAME_BRIDGE_ACTION_SYNC:
1144                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1145                 ast_assert(0);
1146         default:
1147                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1148                 /* "Accept" the frame and discard it. */
1149                 break;
1150         }
1151
1152         return res;
1153 }
1154
1155 static void gather_softmix_stats(struct softmix_stats *stats,
1156         const struct softmix_bridge_data *softmix_data,
1157         struct ast_bridge_channel *bridge_channel)
1158 {
1159         int channel_native_rate;
1160
1161         /* Gather stats about channel sample rates. */
1162         ast_channel_lock(bridge_channel->chan);
1163         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1164                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1165         ast_channel_unlock(bridge_channel->chan);
1166
1167         if (stats->highest_supported_rate < channel_native_rate) {
1168                 stats->highest_supported_rate = channel_native_rate;
1169         }
1170         if (softmix_data->internal_rate < channel_native_rate) {
1171                 int i;
1172
1173                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1174                         if (stats->sample_rates[i] == channel_native_rate) {
1175                                 stats->num_channels[i]++;
1176                                 break;
1177                         } else if (!stats->sample_rates[i]) {
1178                                 stats->sample_rates[i] = channel_native_rate;
1179                                 stats->num_channels[i]++;
1180                                 break;
1181                         }
1182                 }
1183                 stats->num_above_internal_rate++;
1184         } else if (softmix_data->internal_rate == channel_native_rate) {
1185                 stats->num_at_internal_rate++;
1186         }
1187 }
1188
1189 /*!
1190  * \internal
1191  * \brief Analyse mixing statistics and change bridges internal rate
1192  * if necessary.
1193  *
1194  * \retval 0, no changes to internal rate
1195  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1196  */
1197 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1198                 struct softmix_bridge_data *softmix_data, int binaural_active)
1199 {
1200         int i;
1201
1202         if (binaural_active) {
1203                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1204         }
1205
1206         /*
1207          * Re-adjust the internal bridge sample rate if
1208          * 1. The bridge's internal sample rate is locked in at a sample
1209          *    rate other than the current sample rate being used.
1210          * 2. two or more channels support a higher sample rate
1211          * 3. no channels support the current sample rate or a higher rate
1212          */
1213         if (stats->locked_rate) {
1214                 /* if the rate is locked by the bridge, only update it if it differs
1215                  * from the current rate we are using. */
1216                 if (softmix_data->internal_rate != stats->locked_rate) {
1217                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1218                                 softmix_data->internal_rate, stats->locked_rate);
1219                         softmix_data->internal_rate = stats->locked_rate;
1220                         return 1;
1221                 }
1222         } else if (stats->num_above_internal_rate >= 2) {
1223                 /* the highest rate is just used as a starting point */
1224                 unsigned int best_rate = stats->highest_supported_rate;
1225                 int best_index = -1;
1226
1227                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1228                         if (stats->num_channels[i]) {
1229                                 break;
1230                         }
1231                         if (2 <= stats->num_channels[i]) {
1232                                 /* Two or more channels support this rate. */
1233                                 if (best_index == -1
1234                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1235                                         /*
1236                                          * best_rate starts out being the first sample rate
1237                                          * greater than the internal sample rate that two or
1238                                          * more channels support.
1239                                          *
1240                                          * or
1241                                          *
1242                                          * There are multiple rates above the internal rate
1243                                          * and this rate is higher than the previous rate two
1244                                          * or more channels support.
1245                                          */
1246                                         best_rate = stats->sample_rates[i];
1247                                         best_index = i;
1248                                 }
1249                         } else if (best_index == -1) {
1250                                 /*
1251                                  * It is possible that multiple channels exist with native sample
1252                                  * rates above the internal sample rate, but none of those channels
1253                                  * have the same rate in common.  In this case, the lowest sample
1254                                  * rate among those channels is picked. Over time as additional
1255                                  * statistic runs are made the internal sample rate number will
1256                                  * adjust to the most optimal sample rate, but it may take multiple
1257                                  * iterations.
1258                                  */
1259                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1260                         }
1261                 }
1262
1263                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1264                         softmix_data->internal_rate, best_rate);
1265                 softmix_data->internal_rate = best_rate;
1266                 return 1;
1267         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1268                 /* In this case, the highest supported rate is actually lower than the internal rate */
1269                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1270                         softmix_data->internal_rate, stats->highest_supported_rate);
1271                 softmix_data->internal_rate = stats->highest_supported_rate;
1272                 return 1;
1273         }
1274         return 0;
1275 }
1276
1277 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1278                 unsigned int starting_num_entries, unsigned int binaural_active)
1279 {
1280         memset(mixing_array, 0, sizeof(*mixing_array));
1281         mixing_array->max_num_entries = starting_num_entries;
1282         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1283                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1284                 return -1;
1285         }
1286         if (binaural_active) {
1287                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1288                                 sizeof(struct convolve_channel_pair *)))) {
1289                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1290                         return -1;
1291                 }
1292         }
1293         return 0;
1294 }
1295
1296 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1297                 unsigned int binaural_active)
1298 {
1299         ast_free(mixing_array->buffers);
1300         if (binaural_active) {
1301                 ast_free(mixing_array->chan_pairs);
1302         }
1303 }
1304
1305 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1306                 unsigned int num_entries, unsigned int binaural_active)
1307 {
1308         int16_t **tmp;
1309
1310         /* give it some room to grow since memory is cheap but allocations can be expensive */
1311         mixing_array->max_num_entries = num_entries;
1312         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1313                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1314                 return -1;
1315         }
1316         mixing_array->buffers = tmp;
1317
1318         if (binaural_active) {
1319                 struct convolve_channel_pair **tmp2;
1320                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1321                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1322                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1323                         return -1;
1324                 }
1325                 mixing_array->chan_pairs = tmp2;
1326         }
1327         return 0;
1328 }
1329
1330 /*!
1331  * \brief Mixing loop.
1332  *
1333  * \retval 0 on success
1334  * \retval -1 on failure
1335  */
1336 static int softmix_mixing_loop(struct ast_bridge *bridge)
1337 {
1338         struct softmix_stats stats = { { 0 }, };
1339         struct softmix_mixing_array mixing_array;
1340         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1341         struct ast_timer *timer;
1342         struct softmix_translate_helper trans_helper;
1343         int16_t buf[MAX_DATALEN];
1344 #ifdef BINAURAL_RENDERING
1345         int16_t bin_buf[MAX_DATALEN];
1346         int16_t ann_buf[MAX_DATALEN];
1347 #endif
1348         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1349         int timingfd;
1350         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1351         unsigned int idx;
1352         unsigned int x;
1353         int res = -1;
1354
1355         timer = softmix_data->timer;
1356         timingfd = ast_timer_fd(timer);
1357         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1358         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1359
1360         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1361         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1362                         bridge->softmix.binaural_active)) {
1363                 goto softmix_cleanup;
1364         }
1365
1366         /*
1367          * XXX Softmix needs to use channel roles to determine who gets
1368          * what audio mixed.
1369          */
1370         while (!softmix_data->stop && bridge->num_active) {
1371                 struct ast_bridge_channel *bridge_channel;
1372                 int timeout = -1;
1373                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1374                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1375                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1376
1377                 if (softmix_datalen > MAX_DATALEN) {
1378                         /* This should NEVER happen, but if it does we need to know about it. Almost
1379                          * all the memcpys used during this process depend on this assumption.  Rather
1380                          * than checking this over and over again through out the code, this single
1381                          * verification is done on each iteration. */
1382                         ast_log(LOG_WARNING,
1383                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1384                                 bridge->uniqueid);
1385                         goto softmix_cleanup;
1386                 }
1387
1388                 /* Grow the mixing array buffer as participants are added. */
1389                 if (mixing_array.max_num_entries < bridge->num_channels
1390                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1391                                         bridge->softmix.binaural_active)) {
1392                         goto softmix_cleanup;
1393                 }
1394
1395                 /* init the number of buffers stored in the mixing array to 0.
1396                  * As buffers are added for mixing, this number is incremented. */
1397                 mixing_array.used_entries = 0;
1398
1399                 /* These variables help determine if a rate change is required */
1400                 if (!stat_iteration_counter) {
1401                         memset(&stats, 0, sizeof(stats));
1402                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1403                 }
1404
1405                 /* If the sample rate has changed, update the translator helper */
1406                 if (update_all_rates) {
1407                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1408                 }
1409
1410 #ifdef BINAURAL_RENDERING
1411                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1412 #endif
1413
1414                 /* Go through pulling audio from each factory that has it available */
1415                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1416                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1417
1418                         if (!sc) {
1419                                 /* This channel failed to join successfully. */
1420                                 continue;
1421                         }
1422
1423                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1424                         if (update_all_rates) {
1425                                 set_softmix_bridge_data(softmix_data->internal_rate,
1426                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1427                         }
1428
1429                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1430                         if (!stat_iteration_counter) {
1431                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1432                         }
1433
1434                         /* if the channel is suspended, don't check for audio, but still gather stats */
1435                         if (bridge_channel->suspended) {
1436                                 continue;
1437                         }
1438
1439                         /* Try to get audio from the factory if available */
1440                         ast_mutex_lock(&sc->lock);
1441                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1442 #ifdef BINAURAL_RENDERING
1443                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1444                                                 ast_channel_name(bridge_channel->chan));
1445 #endif
1446                                 mixing_array.used_entries++;
1447                         }
1448                         ast_mutex_unlock(&sc->lock);
1449                 }
1450
1451                 /* mix it like crazy (non binaural channels)*/
1452                 memset(buf, 0, softmix_datalen);
1453                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1454                         for (x = 0; x < softmix_samples; ++x) {
1455                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1456                         }
1457                 }
1458
1459 #ifdef BINAURAL_RENDERING
1460                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1461 #endif
1462
1463                 /* Next step go through removing the channel's own audio and creating a good frame... */
1464                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1465                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1466
1467                         if (!sc || bridge_channel->suspended) {
1468                                 /* This channel failed to join successfully or is suspended. */
1469                                 continue;
1470                         }
1471
1472                         ast_mutex_lock(&sc->lock);
1473
1474                         /* Make SLINEAR write frame from local buffer */
1475                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1476                                 "Replace softmix channel slin format");
1477 #ifdef BINAURAL_RENDERING
1478                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1479                                         && sc->binaural) {
1480                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1481                                                 softmix_samples, buf);
1482                         } else
1483 #endif
1484                         {
1485                                 sc->write_frame.datalen = softmix_datalen;
1486                                 sc->write_frame.samples = softmix_samples;
1487                                 memcpy(sc->final_buf, buf, softmix_datalen);
1488                         }
1489                         /* process the softmix channel's new write audio */
1490                         softmix_process_write_audio(&trans_helper,
1491                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1492                                         softmix_data->default_sample_size);
1493
1494                         ast_mutex_unlock(&sc->lock);
1495
1496                         /* A frame is now ready for the channel. */
1497                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1498                 }
1499
1500                 update_all_rates = 0;
1501                 if (!stat_iteration_counter) {
1502                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1503                                         bridge->softmix.binaural_active);
1504                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1505                 }
1506                 stat_iteration_counter--;
1507
1508                 ast_bridge_unlock(bridge);
1509                 /* cleanup any translation frame data from the previous mixing iteration. */
1510                 softmix_translate_helper_cleanup(&trans_helper);
1511                 /* Wait for the timing source to tell us to wake up and get things done */
1512                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1513                 if (ast_timer_ack(timer, 1) < 0) {
1514                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1515                                 bridge->uniqueid);
1516                         ast_bridge_lock(bridge);
1517                         goto softmix_cleanup;
1518                 }
1519                 ast_bridge_lock(bridge);
1520
1521                 /* make sure to detect mixing interval changes if they occur. */
1522                 if (bridge->softmix.internal_mixing_interval
1523                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1524                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1525                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1526                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1527                 }
1528         }
1529
1530         res = 0;
1531
1532 softmix_cleanup:
1533         softmix_translate_helper_destroy(&trans_helper);
1534         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1535         return res;
1536 }
1537
1538 /*!
1539  * \internal
1540  * \brief Mixing thread.
1541  * \since 12.0.0
1542  *
1543  * \note The thread does not have its own reference to the
1544  * bridge.  The lifetime of the thread is tied to the lifetime
1545  * of the mixing technology association with the bridge.
1546  */
1547 static void *softmix_mixing_thread(void *data)
1548 {
1549         struct softmix_bridge_data *softmix_data = data;
1550         struct ast_bridge *bridge = softmix_data->bridge;
1551
1552         ast_bridge_lock(bridge);
1553         if (bridge->callid) {
1554                 ast_callid_threadassoc_add(bridge->callid);
1555         }
1556
1557         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1558
1559         while (!softmix_data->stop) {
1560                 if (!bridge->num_active) {
1561                         /* Wait for something to happen to the bridge. */
1562                         ast_bridge_unlock(bridge);
1563                         ast_mutex_lock(&softmix_data->lock);
1564                         if (!softmix_data->stop) {
1565                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1566                         }
1567                         ast_mutex_unlock(&softmix_data->lock);
1568                         ast_bridge_lock(bridge);
1569                         continue;
1570                 }
1571
1572                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1573 #ifndef BINAURAL_RENDERING
1574                         ast_bridge_lock(bridge);
1575                         bridge->softmix.binaural_active = 0;
1576                         ast_bridge_unlock(bridge);
1577                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1578                                         "compiled.\n", bridge->uniqueid);
1579 #else
1580                         /* Set and init binaural data if binaural is activated in the configuration. */
1581                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1582                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1583                                         softmix_data->internal_mixing_interval);
1584                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1585                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1586                                         == -1) {
1587                                 ast_bridge_lock(bridge);
1588                                 bridge->softmix.binaural_active = 0;
1589                                 ast_bridge_unlock(bridge);
1590                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1591                                                 "binaural processing,  Will only process mono audio.\n",
1592                                                 bridge->uniqueid);
1593                         }
1594                         softmix_data->binaural_init = 1;
1595 #endif
1596                 }
1597
1598                 if (softmix_mixing_loop(bridge)) {
1599                         /*
1600                          * A mixing error occurred.  Sleep and try again later so we
1601                          * won't flood the logs.
1602                          */
1603                         ast_bridge_unlock(bridge);
1604                         sleep(1);
1605                         ast_bridge_lock(bridge);
1606                 }
1607         }
1608
1609         ast_bridge_unlock(bridge);
1610
1611         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1612
1613         return NULL;
1614 }
1615
1616 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1617 {
1618         if (softmix_data->timer) {
1619                 ast_timer_close(softmix_data->timer);
1620                 softmix_data->timer = NULL;
1621         }
1622         ast_mutex_destroy(&softmix_data->lock);
1623         ast_cond_destroy(&softmix_data->cond);
1624         ast_free(softmix_data);
1625 }
1626
1627 /*! \brief Function called when a bridge is created */
1628 static int softmix_bridge_create(struct ast_bridge *bridge)
1629 {
1630         struct softmix_bridge_data *softmix_data;
1631
1632         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1633         if (!softmix_data) {
1634                 return -1;
1635         }
1636         softmix_data->bridge = bridge;
1637         ast_mutex_init(&softmix_data->lock);
1638         ast_cond_init(&softmix_data->cond, NULL);
1639         softmix_data->timer = ast_timer_open();
1640         if (!softmix_data->timer) {
1641                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1642                 softmix_bridge_data_destroy(softmix_data);
1643                 return -1;
1644         }
1645         /* start at minimum rate, let it grow from there */
1646         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1647         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1648
1649 #ifdef BINAURAL_RENDERING
1650         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1651                         softmix_data->internal_mixing_interval);
1652 #endif
1653
1654         bridge->tech_pvt = softmix_data;
1655
1656         /* Start the mixing thread. */
1657         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1658                 softmix_data)) {
1659                 softmix_data->thread = AST_PTHREADT_NULL;
1660                 softmix_bridge_data_destroy(softmix_data);
1661                 bridge->tech_pvt = NULL;
1662                 return -1;
1663         }
1664
1665         return 0;
1666 }
1667
1668 /*!
1669  * \internal
1670  * \brief Request the softmix mixing thread stop.
1671  * \since 12.0.0
1672  *
1673  * \param bridge Which bridge is being stopped.
1674  *
1675  * \return Nothing
1676  */
1677 static void softmix_bridge_stop(struct ast_bridge *bridge)
1678 {
1679         struct softmix_bridge_data *softmix_data;
1680
1681         softmix_data = bridge->tech_pvt;
1682         if (!softmix_data) {
1683                 return;
1684         }
1685
1686         ast_mutex_lock(&softmix_data->lock);
1687         softmix_data->stop = 1;
1688         ast_mutex_unlock(&softmix_data->lock);
1689 }
1690
1691 /*! \brief Function called when a bridge is destroyed */
1692 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1693 {
1694         struct softmix_bridge_data *softmix_data;
1695         pthread_t thread;
1696
1697         softmix_data = bridge->tech_pvt;
1698         if (!softmix_data) {
1699                 return;
1700         }
1701
1702         /* Stop the mixing thread. */
1703         ast_mutex_lock(&softmix_data->lock);
1704         softmix_data->stop = 1;
1705         ast_cond_signal(&softmix_data->cond);
1706         thread = softmix_data->thread;
1707         softmix_data->thread = AST_PTHREADT_NULL;
1708         ast_mutex_unlock(&softmix_data->lock);
1709         if (thread != AST_PTHREADT_NULL) {
1710                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1711                 pthread_join(thread, NULL);
1712         }
1713 #ifdef BINAURAL_RENDERING
1714         free_convolve_data(&softmix_data->convolve);
1715 #endif
1716         softmix_bridge_data_destroy(softmix_data);
1717         bridge->tech_pvt = NULL;
1718 }
1719
1720 /*!
1721  * \brief Map a source stream to all of its destination streams.
1722  *
1723  * \param source_stream_name Name of the source stream
1724  * \param source_channel_name Name of channel where the source stream originates
1725  * \param bridge_stream_position The slot in the bridge where source video will come from
1726  * \param participants The bridge_channels in the bridge
1727  */
1728 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1729         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1730 {
1731         struct ast_bridge_channel *participant;
1732
1733         AST_LIST_TRAVERSE(participants, participant, entry) {
1734                 int i;
1735                 struct ast_stream_topology *topology;
1736
1737                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1738                         continue;
1739                 }
1740
1741                 ast_bridge_channel_lock(participant);
1742                 ast_channel_lock(participant->chan);
1743                 topology = ast_channel_get_stream_topology(participant->chan);
1744
1745                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1746                         struct ast_stream *stream;
1747
1748                         stream = ast_stream_topology_get_stream(topology, i);
1749                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1750                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1751                                 break;
1752                         }
1753                 }
1754                 ast_channel_unlock(participant->chan);
1755                 ast_bridge_channel_unlock(participant);
1756         }
1757 }
1758
1759 /*!
1760  * \brief stream_topology_changed callback
1761  *
1762  * For most video modes, nothing beyond the ordinary is required.
1763  * For the SFU case, though, we need to completely remap the streams
1764  * in order to ensure video gets directed where it is expected to go.
1765  *
1766  * \param bridge The bridge
1767  * \param bridge_channel Channel whose topology has changed
1768  */
1769 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1770 {
1771         struct ast_bridge_channel *participant;
1772         struct ast_vector_int media_types;
1773         int nths[AST_MEDIA_TYPE_END] = {0};
1774
1775         switch (bridge->softmix.video_mode.mode) {
1776         case AST_BRIDGE_VIDEO_MODE_NONE:
1777         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1778         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1779         default:
1780                 ast_bridge_channel_stream_map(bridge_channel);
1781                 return;
1782         case AST_BRIDGE_VIDEO_MODE_SFU:
1783                 break;
1784         }
1785
1786         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1787
1788         /* First traversal: re-initialize all of the participants' stream maps */
1789         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1790                 ast_bridge_channel_lock(participant);
1791                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1792                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1793                 ast_bridge_channel_unlock(participant);
1794         }
1795
1796         /* Second traversal: Map specific video channels from their source to their destinations.
1797          *
1798          * This is similar to what is done in ast_stream_topology_map(),
1799          * except that video channels are handled differently.  Each video
1800          * source has it's own unique index on the bridge.  This way, a
1801          * particular channel's source video can be distributed to the
1802          * appropriate destination streams on the other channels.
1803          */
1804         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1805                 int i;
1806                 struct ast_stream_topology *topology;
1807
1808                 ast_bridge_channel_lock(participant);
1809                 ast_channel_lock(participant->chan);
1810
1811                 topology = ast_channel_get_stream_topology(participant->chan);
1812                 if (topology) {
1813                         /*
1814                          * Sigh.  We have to clone to avoid deadlock in
1815                          * map_source_to_destinations() because topology
1816                          * is not an ao2 object.
1817                          */
1818                         topology = ast_stream_topology_clone(topology);
1819                 }
1820                 if (!topology) {
1821                         /* Oh, my, we are in trouble. */
1822                         ast_channel_unlock(participant->chan);
1823                         ast_bridge_channel_unlock(participant);
1824                         continue;
1825                 }
1826
1827                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1828                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1829
1830                         if (is_video_source(stream)) {
1831                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1832                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1833                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1834                                 /*
1835                                  * Unlock the channel and participant to prevent
1836                                  * potential deadlock in map_source_to_destinations().
1837                                  */
1838                                 ast_channel_unlock(participant->chan);
1839                                 ast_bridge_channel_unlock(participant);
1840                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1841                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1842                                 ast_bridge_channel_lock(participant);
1843                                 ast_channel_lock(participant->chan);
1844                         } else if (is_video_dest(stream, NULL, NULL)) {
1845                                 /* We expect to never read media from video destination channels, but just
1846                                  * in case, we should set their to_bridge value to -1.
1847                                  */
1848                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1849                         } else {
1850                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1851                                  * be factored out in some way
1852                                  */
1853                                 enum ast_media_type type = ast_stream_get_type(stream);
1854                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1855                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1856
1857                                 if (index == -1) {
1858                                         AST_VECTOR_APPEND(&media_types, type);
1859                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1860                                 }
1861
1862                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1863                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1864                         }
1865                 }
1866
1867                 ast_stream_topology_free(topology);
1868
1869                 ast_channel_unlock(participant->chan);
1870                 ast_bridge_channel_unlock(participant);
1871         }
1872 }
1873
1874 static struct ast_bridge_technology softmix_bridge = {
1875         .name = "softmix",
1876         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1877         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1878         .create = softmix_bridge_create,
1879         .stop = softmix_bridge_stop,
1880         .destroy = softmix_bridge_destroy,
1881         .join = softmix_bridge_join,
1882         .leave = softmix_bridge_leave,
1883         .unsuspend = softmix_bridge_unsuspend,
1884         .write = softmix_bridge_write,
1885         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1886 };
1887
1888 #ifdef TEST_FRAMEWORK
1889 struct stream_parameters {
1890         const char *name;
1891         const char *formats;
1892         enum ast_media_type type;
1893 };
1894
1895 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1896 {
1897         struct ast_stream_topology *topology;
1898         size_t i;
1899
1900         topology = ast_stream_topology_alloc();
1901         if (!topology) {
1902                 return NULL;
1903         }
1904
1905         for (i = 0; i < num_streams; ++i) {
1906                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1907                 struct ast_stream *stream;
1908
1909                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1910                 if (!caps) {
1911                         goto fail;
1912                 }
1913                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1914                         goto fail;
1915                 }
1916                 stream = ast_stream_alloc(params[i].name, params[i].type);
1917                 if (!stream) {
1918                         goto fail;
1919                 }
1920                 ast_stream_set_formats(stream, caps);
1921                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1922                         ast_stream_free(stream);
1923                         goto fail;
1924                 }
1925         }
1926
1927         return topology;
1928
1929 fail:
1930         ast_stream_topology_free(topology);
1931         return NULL;
1932 }
1933
1934 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1935         const struct stream_parameters *params)
1936 {
1937         struct ast_format_cap *stream_caps;
1938         struct ast_format_cap *params_caps;
1939
1940         if (ast_stream_get_type(stream) != params->type) {
1941                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1942                         ast_codec_media_type2str(params->type),
1943                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1944                 return -1;
1945         }
1946         if (strcmp(ast_stream_get_name(stream), params->name)) {
1947                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1948                         params->name, ast_stream_get_name(stream));
1949                 return -1;
1950         }
1951
1952         stream_caps = ast_stream_get_formats(stream);
1953         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1954         if (!params_caps) {
1955                 ast_test_status_update(test, "Allocation error on capabilities\n");
1956                 return -1;
1957         }
1958         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
1959
1960         if (ast_format_cap_identical(stream_caps, params_caps)) {
1961                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
1962                         ast_stream_get_name(stream));
1963                 ao2_cleanup(params_caps);
1964                 return -1;
1965         }
1966
1967         ao2_cleanup(params_caps);
1968         return 0;
1969 }
1970
1971 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
1972         const struct stream_parameters *params, size_t num_streams)
1973 {
1974         int i;
1975
1976         if (ast_stream_topology_get_count(topology) < num_streams) {
1977                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
1978                         ast_stream_topology_get_count(topology), num_streams);
1979                 return -1;
1980         }
1981
1982         for (i = 0; i < ARRAY_LEN(params); ++i) {
1983                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
1984                         return -1;
1985                 }
1986         }
1987
1988         return 0;
1989 }
1990
1991 AST_TEST_DEFINE(sfu_append_source_streams)
1992 {
1993         enum ast_test_result_state res = AST_TEST_FAIL;
1994         static const struct stream_parameters bob_streams[] = {
1995                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1996                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1997         };
1998         static const struct stream_parameters alice_streams[] = {
1999                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
2000                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2001         };
2002         static const struct stream_parameters alice_dest_stream = {
2003                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
2004         };
2005         static const struct stream_parameters bob_dest_stream = {
2006                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2007         };
2008         struct ast_stream_topology *topology_alice = NULL;
2009         struct ast_stream_topology *topology_bob = NULL;
2010
2011         switch (cmd) {
2012         case TEST_INIT:
2013                 info->name = "sfu_append_source_streams";
2014                 info->category = "/bridges/bridge_softmix/";
2015                 info->summary = "Test appending of video streams";
2016                 info->description =
2017                         "This tests does stuff.";
2018                 return AST_TEST_NOT_RUN;
2019         case TEST_EXECUTE:
2020                 break;
2021         }
2022
2023         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2024         if (!topology_alice) {
2025                 goto end;
2026         }
2027
2028         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2029         if (!topology_bob) {
2030                 goto end;
2031         }
2032
2033         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2034                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2035                 goto end;
2036         }
2037
2038         if (ast_stream_topology_get_count(topology_alice) != 3) {
2039                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2040                         ast_stream_topology_get_count(topology_alice), 3);
2041                 goto end;
2042         }
2043
2044         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2045                 goto end;
2046         }
2047
2048         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2049                 goto end;
2050         }
2051
2052         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2053                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2054                 goto end;
2055         }
2056
2057         if (ast_stream_topology_get_count(topology_bob) != 3) {
2058                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2059                         ast_stream_topology_get_count(topology_bob), 3);
2060                 goto end;
2061         }
2062
2063         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2064                 goto end;
2065         }
2066
2067         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2068                 goto end;
2069         }
2070
2071         res = AST_TEST_PASS;
2072
2073 end:
2074         ast_stream_topology_free(topology_alice);
2075         ast_stream_topology_free(topology_bob);
2076         return res;
2077 }
2078
2079 AST_TEST_DEFINE(sfu_remove_destination_streams)
2080 {
2081         enum ast_test_result_state res = AST_TEST_FAIL;
2082         static const struct stream_parameters params[] = {
2083                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2084                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2085                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2086                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2087         };
2088         static const struct {
2089                 const char *channel_name;
2090                 int num_streams;
2091                 int params_index[4];
2092         } removal_results[] = {
2093                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2094                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2095                 { "", 4, { 0, 1, 2, 3 }, },
2096         };
2097         struct ast_stream_topology *orig = NULL;
2098         int i;
2099
2100         switch (cmd) {
2101         case TEST_INIT:
2102                 info->name = "sfu_remove_destination_streams";
2103                 info->category = "/bridges/bridge_softmix/";
2104                 info->summary = "Test removal of destination video streams";
2105                 info->description =
2106                         "This tests does stuff.";
2107                 return AST_TEST_NOT_RUN;
2108         case TEST_EXECUTE:
2109                 break;
2110         }
2111
2112         orig = build_topology(params, ARRAY_LEN(params));
2113         if (!orig) {
2114                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2115                 goto end;
2116         }
2117
2118         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2119                 int j;
2120
2121                 remove_destination_streams(orig, removal_results[i].channel_name);
2122
2123                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2124                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2125                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2126                         goto end;
2127                 }
2128
2129                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2130                         struct ast_stream *actual;
2131                         struct ast_stream *expected;
2132                         int orig_index;
2133
2134                         actual = ast_stream_topology_get_stream(orig, j);
2135
2136                         orig_index = removal_results[i].params_index[j];
2137                         expected = ast_stream_topology_get_stream(orig, orig_index);
2138
2139                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2140                                 ast_stream_get_formats(expected))) {
2141                                 struct ast_str *expected_str;
2142                                 struct ast_str *actual_str;
2143
2144                                 expected_str = ast_str_alloca(64);
2145                                 actual_str = ast_str_alloca(64);
2146
2147                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2148                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2149                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2150                                 goto end;
2151                         }
2152
2153                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2154                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2155                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2156                                 goto end;
2157                         }
2158                 }
2159         }
2160
2161         res = AST_TEST_PASS;
2162
2163 end:
2164         ast_stream_topology_free(orig);
2165         return res;
2166 }
2167
2168 #endif
2169
2170 static int unload_module(void)
2171 {
2172         ast_bridge_technology_unregister(&softmix_bridge);
2173         AST_TEST_UNREGISTER(sfu_append_source_streams);
2174         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2175         return 0;
2176 }
2177
2178 static int load_module(void)
2179 {
2180         if (ast_bridge_technology_register(&softmix_bridge)) {
2181                 unload_module();
2182                 return AST_MODULE_LOAD_DECLINE;
2183         }
2184         AST_TEST_REGISTER(sfu_append_source_streams);
2185         AST_TEST_REGISTER(sfu_remove_destination_streams);
2186         return AST_MODULE_LOAD_SUCCESS;
2187 }
2188
2189 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");