bridge_softmix.c: Don't match dead streams.
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                                                   and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED
422                 && ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO
423                 && strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
424                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
425                 return 1;
426         }
427
428         return 0;
429 }
430
431 /*!
432  * \brief Determine if a stream is a video destination stream.
433  *
434  * A source channel name can be provided to narrow this to a destination stream
435  * for a particular source channel. Further, a source stream name can be provided
436  * to narrow this to a particular source stream's destination. However, empty strings
437  * can be provided to match any destination video stream, regardless of source channel
438  * or source stream.
439  *
440  * \param stream The stream to test
441  * \param source_channel_name The name of a source video channel to match
442  * \param source_stream_name The name of the source video stream to match
443  * \retval 1 The stream is a video destination stream
444  * \retval 0 The stream is not a video destination stream
445  */
446 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
447         const char *source_stream_name)
448 {
449         char *dest_video_name;
450         size_t dest_video_name_len;
451
452         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED
453                 || ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
454                 return 0;
455         }
456
457         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
458         if (!ast_strlen_zero(source_channel_name)) {
459                 dest_video_name_len += strlen(source_channel_name) + 1;
460                 if (!ast_strlen_zero(source_stream_name)) {
461                         dest_video_name_len += strlen(source_stream_name) + 1;
462                 }
463
464                 dest_video_name = ast_alloca(dest_video_name_len);
465                 if (!ast_strlen_zero(source_stream_name)) {
466                         /* We are looking for an exact stream name */
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 }
473                 snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                         SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                         source_channel_name);
476         } else {
477                 dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
478         }
479
480         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
481 }
482
483 static int append_source_streams(struct ast_stream_topology *dest,
484         const char *channel_name,
485         const struct ast_stream_topology *source)
486 {
487         int i;
488
489         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
490                 struct ast_stream *stream;
491                 struct ast_stream *stream_clone;
492                 char *stream_clone_name = NULL;
493
494                 stream = ast_stream_topology_get_stream(source, i);
495                 if (!is_video_source(stream)) {
496                         continue;
497                 }
498
499                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
500                         channel_name, ast_stream_get_name(stream)) < 0) {
501                         return -1;
502                 }
503
504                 stream_clone = ast_stream_clone(stream, stream_clone_name);
505                 ast_free(stream_clone_name);
506                 if (!stream_clone) {
507                         return -1;
508                 }
509                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
510                         ast_stream_free(stream_clone);
511                         return -1;
512                 }
513         }
514
515         return 0;
516 }
517
518 static int append_all_streams(struct ast_stream_topology *dest,
519         const struct ast_stream_topology *source)
520 {
521         int i;
522         int dest_index = 0;
523
524         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
525                 struct ast_stream *clone;
526                 int added = 0;
527
528                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
529                 if (!clone) {
530                         return -1;
531                 }
532
533                 /* If we can reuse an existing removed stream then do so */
534                 while (dest_index < ast_stream_topology_get_count(dest)) {
535                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
536
537                         dest_index++;
538
539                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
540                                 /* This cannot fail because dest_index - 1 is less than the
541                                  * current count in dest. */
542                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
543                                 added = 1;
544                                 break;
545                         }
546                 }
547
548                 /* If no removed stream exists that we took the place of append the stream */
549                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
550                         ast_stream_free(clone);
551                         return -1;
552                 }
553         }
554
555         return 0;
556 }
557
558 /*!
559  * \brief Issue channel stream topology change requests.
560  *
561  * When in SFU mode, each participant needs to be able to
562  * send video directly to other participants in the bridge.
563  * This means that all participants need to have their topologies
564  * updated. The joiner needs to have destination streams for
565  * all current participants, and the current participants need
566  * to have destinations streams added for the joiner's sources.
567  *
568  * \param joiner The channel that is joining the softmix bridge
569  * \param participants The current participants in the softmix bridge
570  */
571 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
572 {
573         struct ast_stream_topology *joiner_video = NULL;
574         struct ast_bridge_channel *participant;
575         int res;
576         struct softmix_channel *sc;
577
578         joiner_video = ast_stream_topology_alloc();
579         if (!joiner_video) {
580                 return;
581         }
582
583         sc = joiner->tech_pvt;
584
585         ast_channel_lock(joiner->chan);
586         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
587         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
588         ast_channel_unlock(joiner->chan);
589
590         if (res || !sc->topology) {
591                 goto cleanup;
592         }
593
594         AST_LIST_TRAVERSE(participants, participant, entry) {
595                 if (participant == joiner) {
596                         continue;
597                 }
598                 ast_channel_lock(participant->chan);
599                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
600                                 ast_channel_get_stream_topology(participant->chan));
601                 ast_channel_unlock(participant->chan);
602                 if (res) {
603                         goto cleanup;
604                 }
605         }
606
607         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
608
609         AST_LIST_TRAVERSE(participants, participant, entry) {
610                 if (participant == joiner) {
611                         continue;
612                 }
613
614                 sc = participant->tech_pvt;
615                 if (append_all_streams(sc->topology, joiner_video)) {
616                         goto cleanup;
617                 }
618                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
619         }
620
621 cleanup:
622         ast_stream_topology_free(joiner_video);
623 }
624
625 /*! \brief Function called when a channel is joined into the bridge */
626 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
627 {
628         struct softmix_channel *sc;
629         struct softmix_bridge_data *softmix_data;
630         int set_binaural = 0;
631         /*
632          * If false, the channel will be convolved, but since it is a non stereo channel, output
633          * will be mono.
634          */
635         int skip_binaural_output = 1;
636         int pos_id;
637         int is_announcement = 0;
638         int samplerate_change;
639
640         softmix_data = bridge->tech_pvt;
641         if (!softmix_data) {
642                 return -1;
643         }
644
645         /* Create a new softmix_channel structure and allocate various things on it */
646         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
647                 return -1;
648         }
649
650         samplerate_change = softmix_data->internal_rate;
651         pos_id = -1;
652         if (bridge->softmix.binaural_active) {
653                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
654                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
655                         if (set_binaural) {
656                                 softmix_data->internal_rate = samplerate_change;
657                         }
658                         skip_binaural_output = 0;
659                 } else {
660                         is_announcement = 1;
661                 }
662                 if (set_binaural) {
663                         softmix_data->convolve.binaural_active = 1;
664                 }
665                 if (!skip_binaural_output)      {
666                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
667                         if (pos_id == -1) {
668                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
669                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
670                                                 ast_channel_name(bridge_channel->chan));
671                                 return -1;
672                         }
673                 }
674         }
675
676         /* Can't forget the lock */
677         ast_mutex_init(&sc->lock);
678
679         /* Can't forget to record our pvt structure within the bridged channel structure */
680         bridge_channel->tech_pvt = sc;
681
682         set_softmix_bridge_data(softmix_data->internal_rate,
683                 softmix_data->internal_mixing_interval
684                         ? softmix_data->internal_mixing_interval
685                         : DEFAULT_SOFTMIX_INTERVAL,
686                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
687
688         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
689                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
690         }
691
692         softmix_poke_thread(softmix_data);
693         return 0;
694 }
695
696 static void remove_destination_streams(struct ast_stream_topology *topology,
697         const char *channel_name)
698 {
699         int i;
700
701         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
702                 struct ast_stream *stream;
703
704                 stream = ast_stream_topology_get_stream(topology, i);
705
706                 if (is_video_dest(stream, channel_name, NULL)) {
707                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
708                 }
709         }
710 }
711
712 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
713 {
714         struct ast_bridge_channel *participant;
715         struct softmix_channel *sc;
716
717         AST_LIST_TRAVERSE(participants, participant, entry) {
718                 sc = participant->tech_pvt;
719                 remove_destination_streams(sc->topology, ast_channel_name(leaver->chan));
720                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
721         }
722
723         sc = leaver->tech_pvt;
724         remove_destination_streams(sc->topology, "");
725         ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
726
727         return 0;
728 }
729
730 /*! \brief Function called when a channel leaves the bridge */
731 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
732 {
733         struct softmix_channel *sc;
734         struct softmix_bridge_data *softmix_data;
735
736         softmix_data = bridge->tech_pvt;
737         sc = bridge_channel->tech_pvt;
738         if (!sc) {
739                 return;
740         }
741
742         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
743                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
744         }
745
746         if (bridge->softmix.binaural_active) {
747                 if (sc->binaural) {
748                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
749                                         softmix_data->default_sample_size);
750                 }
751         }
752
753         bridge_channel->tech_pvt = NULL;
754
755         ast_stream_topology_free(sc->topology);
756
757         /* Drop mutex lock */
758         ast_mutex_destroy(&sc->lock);
759
760         /* Drop the factory */
761         ast_slinfactory_destroy(&sc->factory);
762
763         /* Drop any formats on the frames */
764         ao2_cleanup(sc->write_frame.subclass.format);
765
766         /* Drop the DSP */
767         ast_dsp_free(sc->dsp);
768
769         /* Eep! drop ourselves */
770         ast_free(sc);
771 }
772
773 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
774 {
775         struct ast_bridge_channel *cur;
776
777         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
778                 if (cur->suspended) {
779                         continue;
780                 }
781                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
782                         ast_bridge_channel_queue_frame(cur, frame);
783                         break;
784                 }
785         }
786 }
787
788 /*!
789  * \internal
790  * \brief Determine what to do with a video frame.
791  * \since 12.0.0
792  *
793  * \param bridge Which bridge is getting the frame
794  * \param bridge_channel Which channel is writing the frame.
795  * \param frame What is being written.
796  *
797  * \return Nothing
798  */
799 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
800 {
801         struct softmix_channel *sc;
802         int video_src_priority;
803
804         /* Determine if the video frame should be distributed or not */
805         switch (bridge->softmix.video_mode.mode) {
806         case AST_BRIDGE_VIDEO_MODE_NONE:
807                 break;
808         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
809                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
810                 if (video_src_priority == 1) {
811                         /* Pass to me and everyone else. */
812                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
813                 }
814                 break;
815         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
816                 sc = bridge_channel->tech_pvt;
817                 ast_mutex_lock(&sc->lock);
818                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
819                         sc->video_talker.energy_average,
820                         frame->subclass.frame_ending);
821                 ast_mutex_unlock(&sc->lock);
822                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
823                 if (video_src_priority == 1) {
824                         int num_src = ast_bridge_number_video_src(bridge);
825                         int echo = num_src > 1 ? 0 : 1;
826
827                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
828                 } else if (video_src_priority == 2) {
829                         softmix_pass_video_top_priority(bridge, frame);
830                 }
831                 break;
832         case AST_BRIDGE_VIDEO_MODE_SFU:
833                 /* Nothing special to do here, the bridge channel stream map will ensure the
834                  * video goes everywhere it needs to
835                  */
836                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
837                 break;
838         }
839 }
840
841 /*!
842  * \internal
843  * \brief Determine what to do with a voice frame.
844  * \since 12.0.0
845  *
846  * \param bridge Which bridge is getting the frame
847  * \param bridge_channel Which channel is writing the frame.
848  * \param frame What is being written.
849  *
850  * \return Nothing
851  */
852 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
853 {
854         struct softmix_channel *sc = bridge_channel->tech_pvt;
855         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
856         int totalsilence = 0;
857         int cur_energy = 0;
858         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
859                 bridge_channel->tech_args.silence_threshold :
860                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
861         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
862
863         /* Write the frame into the conference */
864         ast_mutex_lock(&sc->lock);
865
866         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
867                 /*
868                  * The incoming frame is not the expected format.  Update
869                  * the channel's translation path to get us slinear from
870                  * the new format for the next frame.
871                  *
872                  * There is the possibility that this frame is an old slinear
873                  * rate frame that was in flight when the softmix bridge
874                  * changed rates.  If so it will self correct on subsequent
875                  * frames.
876                  */
877                 ast_channel_lock(bridge_channel->chan);
878                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
879                         ast_channel_name(bridge_channel->chan),
880                         ast_format_get_name(frame->subclass.format),
881                         ast_format_get_name(sc->read_slin_format));
882                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
883                         sc->read_slin_format);
884                 ast_channel_unlock(bridge_channel->chan);
885         }
886
887         /* The channel will be leaving soon if there is no dsp. */
888         if (sc->dsp) {
889                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
890         }
891
892         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
893                 int cur_slot = sc->video_talker.energy_history_cur_slot;
894
895                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
896                 sc->video_talker.energy_accum += cur_energy;
897                 sc->video_talker.energy_history[cur_slot] = cur_energy;
898                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
899                 sc->video_talker.energy_history_cur_slot++;
900                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
901                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
902                 }
903         }
904
905         if (totalsilence < silence_threshold) {
906                 if (!sc->talking) {
907                         update_talking = 1;
908                 }
909                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
910         } else {
911                 if (sc->talking) {
912                         update_talking = 0;
913                 }
914                 sc->talking = 0;
915         }
916
917         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
918          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
919          * the audio by flushing the buffer before adding new audio in. */
920         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
921                 ast_slinfactory_flush(&sc->factory);
922         }
923
924         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
925          * is not determined to be talking. */
926         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
927                 ast_slinfactory_feed(&sc->factory, frame);
928         }
929
930         /* Alllll done */
931         ast_mutex_unlock(&sc->lock);
932
933         if (update_talking != -1) {
934                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
935         }
936 }
937
938 static int remove_all_original_streams(struct ast_stream_topology *dest,
939         const struct ast_stream_topology *source,
940         const struct ast_stream_topology *original)
941 {
942         int i;
943
944         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
945                 struct ast_stream *stream;
946                 int original_index;
947
948                 stream = ast_stream_topology_get_stream(source, i);
949
950                 /* Mark the existing stream as removed so we get a new one, this will get
951                  * reused on a subsequent renegotiation.
952                  */
953                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
954                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
955
956                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
957                                 struct ast_stream *removed;
958
959                                 /* Since the participant is still going to be in the bridge we
960                                  * change the name so that routing does not attempt to route video
961                                  * to this stream.
962                                  */
963                                 removed = ast_stream_clone(stream, "removed");
964                                 if (!removed) {
965                                         return -1;
966                                 }
967
968                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
969
970                                 /* The destination topology can only ever contain the same, or more,
971                                  * streams than the original so this is safe.
972                                  */
973                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
974                                         ast_stream_free(removed);
975                                         return -1;
976                                 }
977
978                                 break;
979                         }
980                 }
981         }
982
983         return 0;
984 }
985
986 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
987 {
988         struct ast_stream_topology *source_video = NULL;
989         struct ast_bridge_channel *participant;
990         int res;
991
992         source_video = ast_stream_topology_alloc();
993         if (!source_video) {
994                 return;
995         }
996
997         ast_channel_lock(source->chan);
998         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
999         ast_channel_unlock(source->chan);
1000         if (res) {
1001                 goto cleanup;
1002         }
1003
1004         AST_LIST_TRAVERSE(participants, participant, entry) {
1005                 struct ast_stream_topology *original_topology;
1006                 struct softmix_channel *sc;
1007
1008                 if (participant == source) {
1009                         continue;
1010                 }
1011
1012                 sc = participant->tech_pvt;
1013
1014                 original_topology = ast_stream_topology_clone(sc->topology);
1015                 if (!original_topology) {
1016                         goto cleanup;
1017                 }
1018
1019                 /* We add all the source streams back in, if any removed streams are already present they will
1020                  * get used first followed by appending new ones.
1021                  */
1022                 if (append_all_streams(sc->topology, source_video)) {
1023                         ast_stream_topology_free(original_topology);
1024                         goto cleanup;
1025                 }
1026
1027                 /* And the original existing streams get marked as removed. This causes the remote side to see
1028                  * a new stream for the source streams.
1029                  */
1030                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1031                         ast_stream_topology_free(original_topology);
1032                         goto cleanup;
1033                 }
1034
1035                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1036                 ast_stream_topology_free(original_topology);
1037         }
1038
1039 cleanup:
1040         ast_stream_topology_free(source_video);
1041 }
1042
1043 /*!
1044  * \internal
1045  * \brief Determine what to do with a control frame.
1046  * \since 12.0.0
1047  *
1048  * \param bridge Which bridge is getting the frame
1049  * \param bridge_channel Which channel is writing the frame.
1050  * \param frame What is being written.
1051  *
1052  * \retval 0 Frame accepted into the bridge.
1053  * \retval -1 Frame needs to be deferred.
1054  */
1055 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1056 {
1057         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1058
1059         /*
1060          * XXX Softmix needs to use channel roles to determine what to
1061          * do with control frames.
1062          */
1063
1064         switch (frame->subclass.integer) {
1065         case AST_CONTROL_VIDUPDATE:
1066                 if (!bridge->softmix.video_mode.video_update_discard ||
1067                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1068                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1069                         softmix_data->last_video_update = ast_tvnow();
1070                 }
1071                 break;
1072         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1073                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1074                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1075                 }
1076                 break;
1077         default:
1078                 break;
1079         }
1080
1081         return 0;
1082 }
1083
1084 /*!
1085  * \internal
1086  * \brief Determine what to do with a frame written into the bridge.
1087  * \since 12.0.0
1088  *
1089  * \param bridge Which bridge is getting the frame
1090  * \param bridge_channel Which channel is writing the frame.
1091  * \param frame What is being written.
1092  *
1093  * \retval 0 Frame accepted into the bridge.
1094  * \retval -1 Frame needs to be deferred.
1095  *
1096  * \note On entry, bridge is already locked.
1097  */
1098 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1099 {
1100         int res = 0;
1101
1102         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1103                 /* "Accept" the frame and discard it. */
1104                 return 0;
1105         }
1106
1107         /*
1108          * XXX Softmix needs to use channel roles to determine who gets
1109          * what frame.  Possible roles: announcer, recorder, agent,
1110          * supervisor.
1111          */
1112         switch (frame->frametype) {
1113         case AST_FRAME_NULL:
1114                 /* "Accept" the frame and discard it. */
1115                 break;
1116         case AST_FRAME_DTMF_BEGIN:
1117         case AST_FRAME_DTMF_END:
1118                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1119                 break;
1120         case AST_FRAME_VOICE:
1121                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1122                 break;
1123         case AST_FRAME_VIDEO:
1124                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1125                 break;
1126         case AST_FRAME_CONTROL:
1127                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1128                 break;
1129         case AST_FRAME_BRIDGE_ACTION:
1130                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1131                 break;
1132         case AST_FRAME_BRIDGE_ACTION_SYNC:
1133                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1134                 ast_assert(0);
1135         default:
1136                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1137                 /* "Accept" the frame and discard it. */
1138                 break;
1139         }
1140
1141         return res;
1142 }
1143
1144 static void gather_softmix_stats(struct softmix_stats *stats,
1145         const struct softmix_bridge_data *softmix_data,
1146         struct ast_bridge_channel *bridge_channel)
1147 {
1148         int channel_native_rate;
1149
1150         /* Gather stats about channel sample rates. */
1151         ast_channel_lock(bridge_channel->chan);
1152         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1153                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1154         ast_channel_unlock(bridge_channel->chan);
1155
1156         if (stats->highest_supported_rate < channel_native_rate) {
1157                 stats->highest_supported_rate = channel_native_rate;
1158         }
1159         if (softmix_data->internal_rate < channel_native_rate) {
1160                 int i;
1161
1162                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1163                         if (stats->sample_rates[i] == channel_native_rate) {
1164                                 stats->num_channels[i]++;
1165                                 break;
1166                         } else if (!stats->sample_rates[i]) {
1167                                 stats->sample_rates[i] = channel_native_rate;
1168                                 stats->num_channels[i]++;
1169                                 break;
1170                         }
1171                 }
1172                 stats->num_above_internal_rate++;
1173         } else if (softmix_data->internal_rate == channel_native_rate) {
1174                 stats->num_at_internal_rate++;
1175         }
1176 }
1177
1178 /*!
1179  * \internal
1180  * \brief Analyse mixing statistics and change bridges internal rate
1181  * if necessary.
1182  *
1183  * \retval 0, no changes to internal rate
1184  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1185  */
1186 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1187                 struct softmix_bridge_data *softmix_data, int binaural_active)
1188 {
1189         int i;
1190
1191         if (binaural_active) {
1192                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1193         }
1194
1195         /*
1196          * Re-adjust the internal bridge sample rate if
1197          * 1. The bridge's internal sample rate is locked in at a sample
1198          *    rate other than the current sample rate being used.
1199          * 2. two or more channels support a higher sample rate
1200          * 3. no channels support the current sample rate or a higher rate
1201          */
1202         if (stats->locked_rate) {
1203                 /* if the rate is locked by the bridge, only update it if it differs
1204                  * from the current rate we are using. */
1205                 if (softmix_data->internal_rate != stats->locked_rate) {
1206                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1207                                 softmix_data->internal_rate, stats->locked_rate);
1208                         softmix_data->internal_rate = stats->locked_rate;
1209                         return 1;
1210                 }
1211         } else if (stats->num_above_internal_rate >= 2) {
1212                 /* the highest rate is just used as a starting point */
1213                 unsigned int best_rate = stats->highest_supported_rate;
1214                 int best_index = -1;
1215
1216                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1217                         if (stats->num_channels[i]) {
1218                                 break;
1219                         }
1220                         if (2 <= stats->num_channels[i]) {
1221                                 /* Two or more channels support this rate. */
1222                                 if (best_index == -1
1223                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1224                                         /*
1225                                          * best_rate starts out being the first sample rate
1226                                          * greater than the internal sample rate that two or
1227                                          * more channels support.
1228                                          *
1229                                          * or
1230                                          *
1231                                          * There are multiple rates above the internal rate
1232                                          * and this rate is higher than the previous rate two
1233                                          * or more channels support.
1234                                          */
1235                                         best_rate = stats->sample_rates[i];
1236                                         best_index = i;
1237                                 }
1238                         } else if (best_index == -1) {
1239                                 /*
1240                                  * It is possible that multiple channels exist with native sample
1241                                  * rates above the internal sample rate, but none of those channels
1242                                  * have the same rate in common.  In this case, the lowest sample
1243                                  * rate among those channels is picked. Over time as additional
1244                                  * statistic runs are made the internal sample rate number will
1245                                  * adjust to the most optimal sample rate, but it may take multiple
1246                                  * iterations.
1247                                  */
1248                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1249                         }
1250                 }
1251
1252                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1253                         softmix_data->internal_rate, best_rate);
1254                 softmix_data->internal_rate = best_rate;
1255                 return 1;
1256         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1257                 /* In this case, the highest supported rate is actually lower than the internal rate */
1258                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1259                         softmix_data->internal_rate, stats->highest_supported_rate);
1260                 softmix_data->internal_rate = stats->highest_supported_rate;
1261                 return 1;
1262         }
1263         return 0;
1264 }
1265
1266 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1267                 unsigned int starting_num_entries, unsigned int binaural_active)
1268 {
1269         memset(mixing_array, 0, sizeof(*mixing_array));
1270         mixing_array->max_num_entries = starting_num_entries;
1271         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1272                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1273                 return -1;
1274         }
1275         if (binaural_active) {
1276                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1277                                 sizeof(struct convolve_channel_pair *)))) {
1278                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1279                         return -1;
1280                 }
1281         }
1282         return 0;
1283 }
1284
1285 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1286                 unsigned int binaural_active)
1287 {
1288         ast_free(mixing_array->buffers);
1289         if (binaural_active) {
1290                 ast_free(mixing_array->chan_pairs);
1291         }
1292 }
1293
1294 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1295                 unsigned int num_entries, unsigned int binaural_active)
1296 {
1297         int16_t **tmp;
1298
1299         /* give it some room to grow since memory is cheap but allocations can be expensive */
1300         mixing_array->max_num_entries = num_entries;
1301         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1302                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1303                 return -1;
1304         }
1305         if (binaural_active) {
1306                 struct convolve_channel_pair **tmp2;
1307                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1308                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1309                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1310                         return -1;
1311                 }
1312                 mixing_array->chan_pairs = tmp2;
1313         }
1314         mixing_array->buffers = tmp;
1315         return 0;
1316 }
1317
1318 /*!
1319  * \brief Mixing loop.
1320  *
1321  * \retval 0 on success
1322  * \retval -1 on failure
1323  */
1324 static int softmix_mixing_loop(struct ast_bridge *bridge)
1325 {
1326         struct softmix_stats stats = { { 0 }, };
1327         struct softmix_mixing_array mixing_array;
1328         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1329         struct ast_timer *timer;
1330         struct softmix_translate_helper trans_helper;
1331         int16_t buf[MAX_DATALEN];
1332 #ifdef BINAURAL_RENDERING
1333         int16_t bin_buf[MAX_DATALEN];
1334         int16_t ann_buf[MAX_DATALEN];
1335 #endif
1336         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1337         int timingfd;
1338         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1339         unsigned int idx;
1340         unsigned int x;
1341         int res = -1;
1342
1343         timer = softmix_data->timer;
1344         timingfd = ast_timer_fd(timer);
1345         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1346         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1347
1348         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1349         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1350                         bridge->softmix.binaural_active)) {
1351                 goto softmix_cleanup;
1352         }
1353
1354         /*
1355          * XXX Softmix needs to use channel roles to determine who gets
1356          * what audio mixed.
1357          */
1358         while (!softmix_data->stop && bridge->num_active) {
1359                 struct ast_bridge_channel *bridge_channel;
1360                 int timeout = -1;
1361                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1362                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1363                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1364
1365                 if (softmix_datalen > MAX_DATALEN) {
1366                         /* This should NEVER happen, but if it does we need to know about it. Almost
1367                          * all the memcpys used during this process depend on this assumption.  Rather
1368                          * than checking this over and over again through out the code, this single
1369                          * verification is done on each iteration. */
1370                         ast_log(LOG_WARNING,
1371                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1372                                 bridge->uniqueid);
1373                         goto softmix_cleanup;
1374                 }
1375
1376                 /* Grow the mixing array buffer as participants are added. */
1377                 if (mixing_array.max_num_entries < bridge->num_channels
1378                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1379                                         bridge->softmix.binaural_active)) {
1380                         goto softmix_cleanup;
1381                 }
1382
1383                 /* init the number of buffers stored in the mixing array to 0.
1384                  * As buffers are added for mixing, this number is incremented. */
1385                 mixing_array.used_entries = 0;
1386
1387                 /* These variables help determine if a rate change is required */
1388                 if (!stat_iteration_counter) {
1389                         memset(&stats, 0, sizeof(stats));
1390                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1391                 }
1392
1393                 /* If the sample rate has changed, update the translator helper */
1394                 if (update_all_rates) {
1395                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1396                 }
1397
1398 #ifdef BINAURAL_RENDERING
1399                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1400 #endif
1401
1402                 /* Go through pulling audio from each factory that has it available */
1403                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1404                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1405
1406                         if (!sc) {
1407                                 /* This channel failed to join successfully. */
1408                                 continue;
1409                         }
1410
1411                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1412                         if (update_all_rates) {
1413                                 set_softmix_bridge_data(softmix_data->internal_rate,
1414                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1415                         }
1416
1417                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1418                         if (!stat_iteration_counter) {
1419                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1420                         }
1421
1422                         /* if the channel is suspended, don't check for audio, but still gather stats */
1423                         if (bridge_channel->suspended) {
1424                                 continue;
1425                         }
1426
1427                         /* Try to get audio from the factory if available */
1428                         ast_mutex_lock(&sc->lock);
1429                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1430 #ifdef BINAURAL_RENDERING
1431                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1432                                                 ast_channel_name(bridge_channel->chan));
1433 #endif
1434                                 mixing_array.used_entries++;
1435                         }
1436                         ast_mutex_unlock(&sc->lock);
1437                 }
1438
1439                 /* mix it like crazy (non binaural channels)*/
1440                 memset(buf, 0, softmix_datalen);
1441                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1442                         for (x = 0; x < softmix_samples; ++x) {
1443                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1444                         }
1445                 }
1446
1447 #ifdef BINAURAL_RENDERING
1448                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1449 #endif
1450
1451                 /* Next step go through removing the channel's own audio and creating a good frame... */
1452                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1453                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1454
1455                         if (!sc || bridge_channel->suspended) {
1456                                 /* This channel failed to join successfully or is suspended. */
1457                                 continue;
1458                         }
1459
1460                         ast_mutex_lock(&sc->lock);
1461
1462                         /* Make SLINEAR write frame from local buffer */
1463                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1464                                 "Replace softmix channel slin format");
1465 #ifdef BINAURAL_RENDERING
1466                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1467                                         && sc->binaural) {
1468                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1469                                                 softmix_samples, buf);
1470                         } else
1471 #endif
1472                         {
1473                                 sc->write_frame.datalen = softmix_datalen;
1474                                 sc->write_frame.samples = softmix_samples;
1475                                 memcpy(sc->final_buf, buf, softmix_datalen);
1476                         }
1477                         /* process the softmix channel's new write audio */
1478                         softmix_process_write_audio(&trans_helper,
1479                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1480                                         softmix_data->default_sample_size);
1481
1482                         ast_mutex_unlock(&sc->lock);
1483
1484                         /* A frame is now ready for the channel. */
1485                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1486                 }
1487
1488                 update_all_rates = 0;
1489                 if (!stat_iteration_counter) {
1490                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1491                                         bridge->softmix.binaural_active);
1492                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1493                 }
1494                 stat_iteration_counter--;
1495
1496                 ast_bridge_unlock(bridge);
1497                 /* cleanup any translation frame data from the previous mixing iteration. */
1498                 softmix_translate_helper_cleanup(&trans_helper);
1499                 /* Wait for the timing source to tell us to wake up and get things done */
1500                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1501                 if (ast_timer_ack(timer, 1) < 0) {
1502                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1503                                 bridge->uniqueid);
1504                         ast_bridge_lock(bridge);
1505                         goto softmix_cleanup;
1506                 }
1507                 ast_bridge_lock(bridge);
1508
1509                 /* make sure to detect mixing interval changes if they occur. */
1510                 if (bridge->softmix.internal_mixing_interval
1511                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1512                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1513                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1514                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1515                 }
1516         }
1517
1518         res = 0;
1519
1520 softmix_cleanup:
1521         softmix_translate_helper_destroy(&trans_helper);
1522         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1523         return res;
1524 }
1525
1526 /*!
1527  * \internal
1528  * \brief Mixing thread.
1529  * \since 12.0.0
1530  *
1531  * \note The thread does not have its own reference to the
1532  * bridge.  The lifetime of the thread is tied to the lifetime
1533  * of the mixing technology association with the bridge.
1534  */
1535 static void *softmix_mixing_thread(void *data)
1536 {
1537         struct softmix_bridge_data *softmix_data = data;
1538         struct ast_bridge *bridge = softmix_data->bridge;
1539
1540         ast_bridge_lock(bridge);
1541         if (bridge->callid) {
1542                 ast_callid_threadassoc_add(bridge->callid);
1543         }
1544
1545         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1546
1547         while (!softmix_data->stop) {
1548                 if (!bridge->num_active) {
1549                         /* Wait for something to happen to the bridge. */
1550                         ast_bridge_unlock(bridge);
1551                         ast_mutex_lock(&softmix_data->lock);
1552                         if (!softmix_data->stop) {
1553                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1554                         }
1555                         ast_mutex_unlock(&softmix_data->lock);
1556                         ast_bridge_lock(bridge);
1557                         continue;
1558                 }
1559
1560                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1561 #ifndef BINAURAL_RENDERING
1562                         ast_bridge_lock(bridge);
1563                         bridge->softmix.binaural_active = 0;
1564                         ast_bridge_unlock(bridge);
1565                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1566                                         "compiled.\n", bridge->uniqueid);
1567 #else
1568                         /* Set and init binaural data if binaural is activated in the configuration. */
1569                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1570                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1571                                         softmix_data->internal_mixing_interval);
1572                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1573                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1574                                         == -1) {
1575                                 ast_bridge_lock(bridge);
1576                                 bridge->softmix.binaural_active = 0;
1577                                 ast_bridge_unlock(bridge);
1578                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1579                                                 "binaural processing,  Will only process mono audio.\n",
1580                                                 bridge->uniqueid);
1581                         }
1582                         softmix_data->binaural_init = 1;
1583 #endif
1584                 }
1585
1586                 if (softmix_mixing_loop(bridge)) {
1587                         /*
1588                          * A mixing error occurred.  Sleep and try again later so we
1589                          * won't flood the logs.
1590                          */
1591                         ast_bridge_unlock(bridge);
1592                         sleep(1);
1593                         ast_bridge_lock(bridge);
1594                 }
1595         }
1596
1597         ast_bridge_unlock(bridge);
1598
1599         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1600
1601         return NULL;
1602 }
1603
1604 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1605 {
1606         if (softmix_data->timer) {
1607                 ast_timer_close(softmix_data->timer);
1608                 softmix_data->timer = NULL;
1609         }
1610         ast_mutex_destroy(&softmix_data->lock);
1611         ast_cond_destroy(&softmix_data->cond);
1612         ast_free(softmix_data);
1613 }
1614
1615 /*! \brief Function called when a bridge is created */
1616 static int softmix_bridge_create(struct ast_bridge *bridge)
1617 {
1618         struct softmix_bridge_data *softmix_data;
1619
1620         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1621         if (!softmix_data) {
1622                 return -1;
1623         }
1624         softmix_data->bridge = bridge;
1625         ast_mutex_init(&softmix_data->lock);
1626         ast_cond_init(&softmix_data->cond, NULL);
1627         softmix_data->timer = ast_timer_open();
1628         if (!softmix_data->timer) {
1629                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1630                 softmix_bridge_data_destroy(softmix_data);
1631                 return -1;
1632         }
1633         /* start at minimum rate, let it grow from there */
1634         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1635         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1636
1637 #ifdef BINAURAL_RENDERING
1638         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1639                         softmix_data->internal_mixing_interval);
1640 #endif
1641
1642         bridge->tech_pvt = softmix_data;
1643
1644         /* Start the mixing thread. */
1645         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1646                 softmix_data)) {
1647                 softmix_data->thread = AST_PTHREADT_NULL;
1648                 softmix_bridge_data_destroy(softmix_data);
1649                 bridge->tech_pvt = NULL;
1650                 return -1;
1651         }
1652
1653         return 0;
1654 }
1655
1656 /*!
1657  * \internal
1658  * \brief Request the softmix mixing thread stop.
1659  * \since 12.0.0
1660  *
1661  * \param bridge Which bridge is being stopped.
1662  *
1663  * \return Nothing
1664  */
1665 static void softmix_bridge_stop(struct ast_bridge *bridge)
1666 {
1667         struct softmix_bridge_data *softmix_data;
1668
1669         softmix_data = bridge->tech_pvt;
1670         if (!softmix_data) {
1671                 return;
1672         }
1673
1674         ast_mutex_lock(&softmix_data->lock);
1675         softmix_data->stop = 1;
1676         ast_mutex_unlock(&softmix_data->lock);
1677 }
1678
1679 /*! \brief Function called when a bridge is destroyed */
1680 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1681 {
1682         struct softmix_bridge_data *softmix_data;
1683         pthread_t thread;
1684
1685         softmix_data = bridge->tech_pvt;
1686         if (!softmix_data) {
1687                 return;
1688         }
1689
1690         /* Stop the mixing thread. */
1691         ast_mutex_lock(&softmix_data->lock);
1692         softmix_data->stop = 1;
1693         ast_cond_signal(&softmix_data->cond);
1694         thread = softmix_data->thread;
1695         softmix_data->thread = AST_PTHREADT_NULL;
1696         ast_mutex_unlock(&softmix_data->lock);
1697         if (thread != AST_PTHREADT_NULL) {
1698                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1699                 pthread_join(thread, NULL);
1700         }
1701 #ifdef BINAURAL_RENDERING
1702         free_convolve_data(&softmix_data->convolve);
1703 #endif
1704         softmix_bridge_data_destroy(softmix_data);
1705         bridge->tech_pvt = NULL;
1706 }
1707
1708 /*!
1709  * \brief Map a source stream to all of its destination streams.
1710  *
1711  * \param source_stream_name Name of the source stream
1712  * \param source_channel_name Name of channel where the source stream originates
1713  * \param bridge_stream_position The slot in the bridge where source video will come from
1714  * \param participants The bridge_channels in the bridge
1715  */
1716 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1717         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1718 {
1719         struct ast_bridge_channel *participant;
1720
1721         AST_LIST_TRAVERSE(participants, participant, entry) {
1722                 int i;
1723                 struct ast_stream_topology *topology;
1724
1725                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1726                         continue;
1727                 }
1728
1729                 ast_bridge_channel_lock(participant);
1730                 ast_channel_lock(participant->chan);
1731                 topology = ast_channel_get_stream_topology(participant->chan);
1732
1733                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1734                         struct ast_stream *stream;
1735
1736                         stream = ast_stream_topology_get_stream(topology, i);
1737                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1738                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1739                                 break;
1740                         }
1741                 }
1742                 ast_channel_unlock(participant->chan);
1743                 ast_bridge_channel_unlock(participant);
1744         }
1745 }
1746
1747 /*!
1748  * \brief stream_topology_changed callback
1749  *
1750  * For most video modes, nothing beyond the ordinary is required.
1751  * For the SFU case, though, we need to completely remap the streams
1752  * in order to ensure video gets directed where it is expected to go.
1753  *
1754  * \param bridge The bridge
1755  * \param bridge_channel Channel whose topology has changed
1756  */
1757 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1758 {
1759         struct ast_bridge_channel *participant;
1760         struct ast_vector_int media_types;
1761         int nths[AST_MEDIA_TYPE_END] = {0};
1762
1763         switch (bridge->softmix.video_mode.mode) {
1764         case AST_BRIDGE_VIDEO_MODE_NONE:
1765         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1766         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1767         default:
1768                 ast_bridge_channel_stream_map(bridge_channel);
1769                 return;
1770         case AST_BRIDGE_VIDEO_MODE_SFU:
1771                 break;
1772         }
1773
1774         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1775
1776         /* First traversal: re-initialize all of the participants' stream maps */
1777         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1778                 ast_bridge_channel_lock(participant);
1779                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1780                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1781                 ast_bridge_channel_unlock(participant);
1782         }
1783
1784         /* Second traversal: Map specific video channels from their source to their destinations.
1785          *
1786          * This is similar to what is done in ast_stream_topology_map(),
1787          * except that video channels are handled differently.  Each video
1788          * source has it's own unique index on the bridge.  This way, a
1789          * particular channel's source video can be distributed to the
1790          * appropriate destination streams on the other channels.
1791          */
1792         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1793                 int i;
1794                 struct ast_stream_topology *topology;
1795
1796                 ast_bridge_channel_lock(participant);
1797                 ast_channel_lock(participant->chan);
1798
1799                 topology = ast_channel_get_stream_topology(participant->chan);
1800                 if (topology) {
1801                         /*
1802                          * Sigh.  We have to clone to avoid deadlock in
1803                          * map_source_to_destinations() because topology
1804                          * is not an ao2 object.
1805                          */
1806                         topology = ast_stream_topology_clone(topology);
1807                 }
1808                 if (!topology) {
1809                         /* Oh, my, we are in trouble. */
1810                         ast_channel_unlock(participant->chan);
1811                         ast_bridge_channel_unlock(participant);
1812                         continue;
1813                 }
1814
1815                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1816                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1817
1818                         if (is_video_source(stream)) {
1819                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1820                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1821                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1822                                 /*
1823                                  * Unlock the channel and participant to prevent
1824                                  * potential deadlock in map_source_to_destinations().
1825                                  */
1826                                 ast_channel_unlock(participant->chan);
1827                                 ast_bridge_channel_unlock(participant);
1828                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1829                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1830                                 ast_bridge_channel_lock(participant);
1831                                 ast_channel_lock(participant->chan);
1832                         } else if (is_video_dest(stream, NULL, NULL)) {
1833                                 /* We expect to never read media from video destination channels, but just
1834                                  * in case, we should set their to_bridge value to -1.
1835                                  */
1836                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1837                         } else {
1838                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1839                                  * be factored out in some way
1840                                  */
1841                                 enum ast_media_type type = ast_stream_get_type(stream);
1842                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1843                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1844
1845                                 if (index == -1) {
1846                                         AST_VECTOR_APPEND(&media_types, type);
1847                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1848                                 }
1849
1850                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1851                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1852                         }
1853                 }
1854
1855                 ast_stream_topology_free(topology);
1856
1857                 ast_channel_unlock(participant->chan);
1858                 ast_bridge_channel_unlock(participant);
1859         }
1860 }
1861
1862 static struct ast_bridge_technology softmix_bridge = {
1863         .name = "softmix",
1864         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1865         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1866         .create = softmix_bridge_create,
1867         .stop = softmix_bridge_stop,
1868         .destroy = softmix_bridge_destroy,
1869         .join = softmix_bridge_join,
1870         .leave = softmix_bridge_leave,
1871         .unsuspend = softmix_bridge_unsuspend,
1872         .write = softmix_bridge_write,
1873         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1874 };
1875
1876 #ifdef TEST_FRAMEWORK
1877 struct stream_parameters {
1878         const char *name;
1879         const char *formats;
1880         enum ast_media_type type;
1881 };
1882
1883 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1884 {
1885         struct ast_stream_topology *topology;
1886         size_t i;
1887
1888         topology = ast_stream_topology_alloc();
1889         if (!topology) {
1890                 return NULL;
1891         }
1892
1893         for (i = 0; i < num_streams; ++i) {
1894                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1895                 struct ast_stream *stream;
1896
1897                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1898                 if (!caps) {
1899                         goto fail;
1900                 }
1901                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1902                         goto fail;
1903                 }
1904                 stream = ast_stream_alloc(params[i].name, params[i].type);
1905                 if (!stream) {
1906                         goto fail;
1907                 }
1908                 ast_stream_set_formats(stream, caps);
1909                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1910                         ast_stream_free(stream);
1911                         goto fail;
1912                 }
1913         }
1914
1915         return topology;
1916
1917 fail:
1918         ast_stream_topology_free(topology);
1919         return NULL;
1920 }
1921
1922 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1923         const struct stream_parameters *params)
1924 {
1925         struct ast_format_cap *stream_caps;
1926         struct ast_format_cap *params_caps;
1927
1928         if (ast_stream_get_type(stream) != params->type) {
1929                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1930                         ast_codec_media_type2str(params->type),
1931                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1932                 return -1;
1933         }
1934         if (strcmp(ast_stream_get_name(stream), params->name)) {
1935                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1936                         params->name, ast_stream_get_name(stream));
1937                 return -1;
1938         }
1939
1940         stream_caps = ast_stream_get_formats(stream);
1941         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1942         if (!params_caps) {
1943                 ast_test_status_update(test, "Allocation error on capabilities\n");
1944                 return -1;
1945         }
1946         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
1947
1948         if (ast_format_cap_identical(stream_caps, params_caps)) {
1949                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
1950                         ast_stream_get_name(stream));
1951                 ao2_cleanup(params_caps);
1952                 return -1;
1953         }
1954
1955         ao2_cleanup(params_caps);
1956         return 0;
1957 }
1958
1959 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
1960         const struct stream_parameters *params, size_t num_streams)
1961 {
1962         int i;
1963
1964         if (ast_stream_topology_get_count(topology) < num_streams) {
1965                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
1966                         ast_stream_topology_get_count(topology), num_streams);
1967                 return -1;
1968         }
1969
1970         for (i = 0; i < ARRAY_LEN(params); ++i) {
1971                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
1972                         return -1;
1973                 }
1974         }
1975
1976         return 0;
1977 }
1978
1979 AST_TEST_DEFINE(sfu_append_source_streams)
1980 {
1981         enum ast_test_result_state res = AST_TEST_FAIL;
1982         static const struct stream_parameters bob_streams[] = {
1983                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1984                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1985         };
1986         static const struct stream_parameters alice_streams[] = {
1987                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
1988                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
1989         };
1990         static const struct stream_parameters alice_dest_stream = {
1991                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
1992         };
1993         static const struct stream_parameters bob_dest_stream = {
1994                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
1995         };
1996         struct ast_stream_topology *topology_alice = NULL;
1997         struct ast_stream_topology *topology_bob = NULL;
1998
1999         switch (cmd) {
2000         case TEST_INIT:
2001                 info->name = "sfu_append_source_streams";
2002                 info->category = "/bridges/bridge_softmix/";
2003                 info->summary = "Test appending of video streams";
2004                 info->description =
2005                         "This tests does stuff.";
2006                 return AST_TEST_NOT_RUN;
2007         case TEST_EXECUTE:
2008                 break;
2009         }
2010
2011         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2012         if (!topology_alice) {
2013                 goto end;
2014         }
2015
2016         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2017         if (!topology_bob) {
2018                 goto end;
2019         }
2020
2021         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2022                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2023                 goto end;
2024         }
2025
2026         if (ast_stream_topology_get_count(topology_alice) != 3) {
2027                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2028                         ast_stream_topology_get_count(topology_alice), 3);
2029                 goto end;
2030         }
2031
2032         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2033                 goto end;
2034         }
2035
2036         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2037                 goto end;
2038         }
2039
2040         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2041                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2042                 goto end;
2043         }
2044
2045         if (ast_stream_topology_get_count(topology_bob) != 3) {
2046                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2047                         ast_stream_topology_get_count(topology_bob), 3);
2048                 goto end;
2049         }
2050
2051         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2052                 goto end;
2053         }
2054
2055         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2056                 goto end;
2057         }
2058
2059         res = AST_TEST_PASS;
2060
2061 end:
2062         ast_stream_topology_free(topology_alice);
2063         ast_stream_topology_free(topology_bob);
2064         return res;
2065 }
2066
2067 AST_TEST_DEFINE(sfu_remove_destination_streams)
2068 {
2069         enum ast_test_result_state res = AST_TEST_FAIL;
2070         static const struct stream_parameters params[] = {
2071                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2072                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2073                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2074                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2075         };
2076         static const struct {
2077                 const char *channel_name;
2078                 int num_streams;
2079                 int params_index[4];
2080         } removal_results[] = {
2081                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2082                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2083                 { "", 4, { 0, 1, 2, 3 }, },
2084         };
2085         struct ast_stream_topology *orig = NULL;
2086         int i;
2087
2088         switch (cmd) {
2089         case TEST_INIT:
2090                 info->name = "sfu_remove_destination_streams";
2091                 info->category = "/bridges/bridge_softmix/";
2092                 info->summary = "Test removal of destination video streams";
2093                 info->description =
2094                         "This tests does stuff.";
2095                 return AST_TEST_NOT_RUN;
2096         case TEST_EXECUTE:
2097                 break;
2098         }
2099
2100         orig = build_topology(params, ARRAY_LEN(params));
2101         if (!orig) {
2102                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2103                 goto end;
2104         }
2105
2106         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2107                 int j;
2108
2109                 remove_destination_streams(orig, removal_results[i].channel_name);
2110
2111                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2112                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2113                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2114                         goto end;
2115                 }
2116
2117                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2118                         struct ast_stream *actual;
2119                         struct ast_stream *expected;
2120                         int orig_index;
2121
2122                         actual = ast_stream_topology_get_stream(orig, j);
2123
2124                         orig_index = removal_results[i].params_index[j];
2125                         expected = ast_stream_topology_get_stream(orig, orig_index);
2126
2127                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2128                                 ast_stream_get_formats(expected))) {
2129                                 struct ast_str *expected_str;
2130                                 struct ast_str *actual_str;
2131
2132                                 expected_str = ast_str_alloca(64);
2133                                 actual_str = ast_str_alloca(64);
2134
2135                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2136                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2137                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2138                                 goto end;
2139                         }
2140
2141                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2142                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2143                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2144                                 goto end;
2145                         }
2146                 }
2147         }
2148
2149         res = AST_TEST_PASS;
2150
2151 end:
2152         ast_stream_topology_free(orig);
2153         return res;
2154 }
2155
2156 #endif
2157
2158 static int unload_module(void)
2159 {
2160         ast_bridge_technology_unregister(&softmix_bridge);
2161         AST_TEST_UNREGISTER(sfu_append_source_streams);
2162         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2163         return 0;
2164 }
2165
2166 static int load_module(void)
2167 {
2168         if (ast_bridge_technology_register(&softmix_bridge)) {
2169                 unload_module();
2170                 return AST_MODULE_LOAD_DECLINE;
2171         }
2172         AST_TEST_REGISTER(sfu_append_source_streams);
2173         AST_TEST_REGISTER(sfu_remove_destination_streams);
2174         return AST_MODULE_LOAD_SUCCESS;
2175 }
2176
2177 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");