f11de6d3c07e63010502999bbdf01b09fea8b09f
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                       and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO &&
422                 strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
423                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
424                 return 1;
425         }
426
427         return 0;
428 }
429
430 /*!
431  * \brief Determine if a stream is a video destination stream.
432  *
433  * A source channel name can be provided to narrow this to a destination stream
434  * for a particular source channel. Further, a source stream name can be provided
435  * to narrow this to a particular source stream's destination. However, empty strings
436  * can be provided to match any destination video stream, regardless of source channel
437  * or source stream.
438  *
439  * \param stream The stream to test
440  * \param source_channel_name The name of a source video channel to match
441  * \param source_stream_name The name of the source video stream to match
442  * \retval 1 The stream is a video destination stream
443  * \retval 0 The stream is not a video destination stream
444  */
445 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
446         const char *source_stream_name)
447 {
448         char *dest_video_name;
449         size_t dest_video_name_len;
450
451         if (ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
452                 return 0;
453         }
454
455         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
456
457         if (!ast_strlen_zero(source_channel_name)) {
458                 dest_video_name_len += strlen(source_channel_name) + 1;
459                 if (!ast_strlen_zero(source_stream_name)) {
460                         dest_video_name_len += strlen(source_stream_name) + 1;
461                 }
462         }
463         dest_video_name = ast_alloca(dest_video_name_len);
464
465         if (!ast_strlen_zero(source_channel_name)) {
466                 if (!ast_strlen_zero(source_stream_name)) {
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 } else {
473                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                                 source_channel_name);
476                         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
477                 }
478         } else {
479                 snprintf(dest_video_name, dest_video_name_len, "%s",
480                         SOFTBRIDGE_VIDEO_DEST_PREFIX);
481                 return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
482         }
483
484         return 0;
485 }
486
487 static int append_source_streams(struct ast_stream_topology *dest,
488         const char *channel_name,
489         const struct ast_stream_topology *source)
490 {
491         int i;
492
493         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
494                 struct ast_stream *stream;
495                 struct ast_stream *stream_clone;
496                 char *stream_clone_name;
497                 size_t stream_clone_name_len;
498
499                 stream = ast_stream_topology_get_stream(source, i);
500                 if (!is_video_source(stream)) {
501                         continue;
502                 }
503
504                 /* The +3 is for the two underscore separators and null terminator */
505                 stream_clone_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + strlen(channel_name) + strlen(ast_stream_get_name(stream)) + 3;
506                 stream_clone_name = ast_alloca(stream_clone_name_len);
507                 snprintf(stream_clone_name, stream_clone_name_len, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
508                         channel_name, ast_stream_get_name(stream));
509
510                 stream_clone = ast_stream_clone(stream, stream_clone_name);
511                 if (!stream_clone) {
512                         return -1;
513                 }
514                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
515                         ast_stream_free(stream_clone);
516                         return -1;
517                 }
518         }
519
520         return 0;
521 }
522
523 static int append_all_streams(struct ast_stream_topology *dest,
524         const struct ast_stream_topology *source)
525 {
526         int i;
527         int dest_index = 0;
528
529         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
530                 struct ast_stream *clone;
531                 int added = 0;
532
533                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
534                 if (!clone) {
535                         return -1;
536                 }
537
538                 /* If we can reuse an existing removed stream then do so */
539                 while (dest_index < ast_stream_topology_get_count(dest)) {
540                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
541
542                         dest_index++;
543
544                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
545                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
546                                 added = 1;
547                                 break;
548                         }
549                 }
550
551                 /* If no removed stream exists that we took the place of append the stream */
552                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
553                         ast_stream_free(clone);
554                         return -1;
555                 }
556         }
557
558         return 0;
559 }
560
561 /*!
562  * \brief Issue channel stream topology change requests.
563  *
564  * When in SFU mode, each participant needs to be able to
565  * send video directly to other participants in the bridge.
566  * This means that all participants need to have their topologies
567  * updated. The joiner needs to have destination streams for
568  * all current participants, and the current participants need
569  * to have destinations streams added for the joiner's sources.
570  *
571  * \param joiner The channel that is joining the softmix bridge
572  * \param participants The current participants in the softmix bridge
573  */
574 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
575 {
576         struct ast_stream_topology *joiner_topology = NULL;
577         struct ast_stream_topology *joiner_video = NULL;
578         struct ast_stream_topology *existing_video = NULL;
579         struct ast_bridge_channel *participant;
580         int res;
581
582         joiner_video = ast_stream_topology_alloc();
583         if (!joiner_video) {
584                 return;
585         }
586
587         ast_channel_lock(joiner->chan);
588         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
589         ast_channel_unlock(joiner->chan);
590
591         if (res) {
592                 goto cleanup;
593         }
594
595         existing_video = ast_stream_topology_alloc();
596         if (!existing_video) {
597                 goto cleanup;
598         }
599
600         AST_LIST_TRAVERSE(participants, participant, entry) {
601                 if (participant == joiner) {
602                         continue;
603                 }
604                 ast_channel_lock(participant->chan);
605                 res = append_source_streams(existing_video, ast_channel_name(participant->chan),
606                                 ast_channel_get_stream_topology(participant->chan));
607                 ast_channel_unlock(participant->chan);
608                 if (res) {
609                         goto cleanup;
610                 }
611         }
612
613         ast_channel_lock(joiner->chan);
614         joiner_topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
615         ast_channel_unlock(joiner->chan);
616         if (!joiner_topology) {
617                 goto cleanup;
618         }
619         if (append_all_streams(joiner_topology, existing_video)) {
620                 goto cleanup;
621         }
622         ast_channel_request_stream_topology_change(joiner->chan, joiner_topology, NULL);
623
624         AST_LIST_TRAVERSE(participants, participant, entry) {
625                 struct ast_stream_topology *participant_topology;
626
627                 if (participant == joiner) {
628                         continue;
629                 }
630                 ast_channel_lock(participant->chan);
631                 participant_topology = ast_stream_topology_clone(ast_channel_get_stream_topology(participant->chan));
632                 ast_channel_unlock(participant->chan);
633                 if (!participant_topology) {
634                         goto cleanup;
635                 }
636                 if (append_all_streams(participant_topology, joiner_video)) {
637                         ast_stream_topology_free(participant_topology);
638                         goto cleanup;
639                 }
640                 ast_channel_request_stream_topology_change(participant->chan, participant_topology, NULL);
641                 ast_stream_topology_free(participant_topology);
642         }
643
644 cleanup:
645         ast_stream_topology_free(joiner_video);
646         ast_stream_topology_free(existing_video);
647         ast_stream_topology_free(joiner_topology);
648 }
649
650 /*! \brief Function called when a channel is joined into the bridge */
651 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
652 {
653         struct softmix_channel *sc;
654         struct softmix_bridge_data *softmix_data;
655         int set_binaural = 0;
656         /*
657          * If false, the channel will be convolved, but since it is a non stereo channel, output
658          * will be mono.
659          */
660         int skip_binaural_output = 1;
661         int pos_id;
662         int is_announcement = 0;
663         int samplerate_change;
664
665         softmix_data = bridge->tech_pvt;
666         if (!softmix_data) {
667                 return -1;
668         }
669
670         /* Create a new softmix_channel structure and allocate various things on it */
671         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
672                 return -1;
673         }
674
675         samplerate_change = softmix_data->internal_rate;
676         pos_id = -1;
677         if (bridge->softmix.binaural_active) {
678                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
679                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
680                         if (set_binaural) {
681                                 softmix_data->internal_rate = samplerate_change;
682                         }
683                         skip_binaural_output = 0;
684                 } else {
685                         is_announcement = 1;
686                 }
687                 if (set_binaural) {
688                         softmix_data->convolve.binaural_active = 1;
689                 }
690                 if (!skip_binaural_output)      {
691                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
692                         if (pos_id == -1) {
693                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
694                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
695                                                 ast_channel_name(bridge_channel->chan));
696                                 return -1;
697                         }
698                 }
699         }
700
701         /* Can't forget the lock */
702         ast_mutex_init(&sc->lock);
703
704         /* Can't forget to record our pvt structure within the bridged channel structure */
705         bridge_channel->tech_pvt = sc;
706
707         set_softmix_bridge_data(softmix_data->internal_rate,
708                 softmix_data->internal_mixing_interval
709                         ? softmix_data->internal_mixing_interval
710                         : DEFAULT_SOFTMIX_INTERVAL,
711                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
712
713         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
714                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
715         }
716
717         softmix_poke_thread(softmix_data);
718         return 0;
719 }
720
721 static int remove_destination_streams(struct ast_stream_topology *dest,
722         const char *channel_name,
723         const struct ast_stream_topology *source)
724 {
725         int i;
726
727         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
728                 struct ast_stream *stream;
729                 struct ast_stream *stream_clone;
730
731                 stream = ast_stream_topology_get_stream(source, i);
732
733                 stream_clone = ast_stream_clone(stream, NULL);
734                 if (!stream_clone) {
735                         continue;
736                 }
737
738                 if (is_video_dest(stream, channel_name, NULL)) {
739                         ast_stream_set_state(stream_clone, AST_STREAM_STATE_REMOVED);
740                 }
741
742                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
743                         ast_stream_free(stream_clone);
744                 }
745         }
746
747         return 0;
748 }
749
750 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
751 {
752         struct ast_stream_topology *leaver_topology;
753         struct ast_bridge_channel *participant;
754
755         leaver_topology = ast_stream_topology_alloc();
756         if (!leaver_topology) {
757                 return -1;
758         }
759
760         AST_LIST_TRAVERSE(participants, participant, entry) {
761                 struct ast_stream_topology *participant_topology;
762
763                 participant_topology = ast_stream_topology_alloc();
764                 if (!participant_topology) {
765                         continue;
766                 }
767
768                 ast_channel_lock(participant->chan);
769                 remove_destination_streams(participant_topology, ast_channel_name(leaver->chan), ast_channel_get_stream_topology(participant->chan));
770                 ast_channel_unlock(participant->chan);
771                 ast_channel_request_stream_topology_change(participant->chan, participant_topology, NULL);
772                 ast_stream_topology_free(participant_topology);
773         }
774
775         ast_channel_lock(leaver->chan);
776         remove_destination_streams(leaver_topology, "", ast_channel_get_stream_topology(leaver->chan));
777         ast_channel_unlock(leaver->chan);
778         ast_channel_request_stream_topology_change(leaver->chan, leaver_topology, NULL);
779         ast_stream_topology_free(leaver_topology);
780
781         return 0;
782 }
783
784 /*! \brief Function called when a channel leaves the bridge */
785 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
786 {
787         struct softmix_channel *sc;
788         struct softmix_bridge_data *softmix_data;
789
790         softmix_data = bridge->tech_pvt;
791         sc = bridge_channel->tech_pvt;
792         if (!sc) {
793                 return;
794         }
795
796         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
797                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
798         }
799
800         if (bridge->softmix.binaural_active) {
801                 if (sc->binaural) {
802                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
803                                         softmix_data->default_sample_size);
804                 }
805         }
806
807         bridge_channel->tech_pvt = NULL;
808
809         /* Drop mutex lock */
810         ast_mutex_destroy(&sc->lock);
811
812         /* Drop the factory */
813         ast_slinfactory_destroy(&sc->factory);
814
815         /* Drop any formats on the frames */
816         ao2_cleanup(sc->write_frame.subclass.format);
817
818         /* Drop the DSP */
819         ast_dsp_free(sc->dsp);
820
821         /* Eep! drop ourselves */
822         ast_free(sc);
823 }
824
825 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
826 {
827         struct ast_bridge_channel *cur;
828
829         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
830                 if (cur->suspended) {
831                         continue;
832                 }
833                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
834                         ast_bridge_channel_queue_frame(cur, frame);
835                         break;
836                 }
837         }
838 }
839
840 /*!
841  * \internal
842  * \brief Determine what to do with a video frame.
843  * \since 12.0.0
844  *
845  * \param bridge Which bridge is getting the frame
846  * \param bridge_channel Which channel is writing the frame.
847  * \param frame What is being written.
848  *
849  * \return Nothing
850  */
851 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
852 {
853         struct softmix_channel *sc;
854         int video_src_priority;
855
856         /* Determine if the video frame should be distributed or not */
857         switch (bridge->softmix.video_mode.mode) {
858         case AST_BRIDGE_VIDEO_MODE_NONE:
859                 break;
860         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
861                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
862                 if (video_src_priority == 1) {
863                         /* Pass to me and everyone else. */
864                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
865                 }
866                 break;
867         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
868                 sc = bridge_channel->tech_pvt;
869                 ast_mutex_lock(&sc->lock);
870                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
871                         sc->video_talker.energy_average,
872                         frame->subclass.frame_ending);
873                 ast_mutex_unlock(&sc->lock);
874                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
875                 if (video_src_priority == 1) {
876                         int num_src = ast_bridge_number_video_src(bridge);
877                         int echo = num_src > 1 ? 0 : 1;
878
879                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
880                 } else if (video_src_priority == 2) {
881                         softmix_pass_video_top_priority(bridge, frame);
882                 }
883                 break;
884         case AST_BRIDGE_VIDEO_MODE_SFU:
885                 /* Nothing special to do here, the bridge channel stream map will ensure the
886                  * video goes everywhere it needs to
887                  */
888                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
889                 break;
890         }
891 }
892
893 /*!
894  * \internal
895  * \brief Determine what to do with a voice frame.
896  * \since 12.0.0
897  *
898  * \param bridge Which bridge is getting the frame
899  * \param bridge_channel Which channel is writing the frame.
900  * \param frame What is being written.
901  *
902  * \return Nothing
903  */
904 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
905 {
906         struct softmix_channel *sc = bridge_channel->tech_pvt;
907         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
908         int totalsilence = 0;
909         int cur_energy = 0;
910         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
911                 bridge_channel->tech_args.silence_threshold :
912                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
913         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
914
915         /* Write the frame into the conference */
916         ast_mutex_lock(&sc->lock);
917
918         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
919                 /*
920                  * The incoming frame is not the expected format.  Update
921                  * the channel's translation path to get us slinear from
922                  * the new format for the next frame.
923                  *
924                  * There is the possibility that this frame is an old slinear
925                  * rate frame that was in flight when the softmix bridge
926                  * changed rates.  If so it will self correct on subsequent
927                  * frames.
928                  */
929                 ast_channel_lock(bridge_channel->chan);
930                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
931                         ast_channel_name(bridge_channel->chan),
932                         ast_format_get_name(frame->subclass.format),
933                         ast_format_get_name(sc->read_slin_format));
934                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
935                         sc->read_slin_format);
936                 ast_channel_unlock(bridge_channel->chan);
937         }
938
939         /* The channel will be leaving soon if there is no dsp. */
940         if (sc->dsp) {
941                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
942         }
943
944         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
945                 int cur_slot = sc->video_talker.energy_history_cur_slot;
946
947                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
948                 sc->video_talker.energy_accum += cur_energy;
949                 sc->video_talker.energy_history[cur_slot] = cur_energy;
950                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
951                 sc->video_talker.energy_history_cur_slot++;
952                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
953                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
954                 }
955         }
956
957         if (totalsilence < silence_threshold) {
958                 if (!sc->talking) {
959                         update_talking = 1;
960                 }
961                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
962         } else {
963                 if (sc->talking) {
964                         update_talking = 0;
965                 }
966                 sc->talking = 0;
967         }
968
969         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
970          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
971          * the audio by flushing the buffer before adding new audio in. */
972         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
973                 ast_slinfactory_flush(&sc->factory);
974         }
975
976         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
977          * is not determined to be talking. */
978         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
979                 ast_slinfactory_feed(&sc->factory, frame);
980         }
981
982         /* Alllll done */
983         ast_mutex_unlock(&sc->lock);
984
985         if (update_talking != -1) {
986                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
987         }
988 }
989
990 /*!
991  * \internal
992  * \brief Determine what to do with a control frame.
993  * \since 12.0.0
994  *
995  * \param bridge Which bridge is getting the frame
996  * \param bridge_channel Which channel is writing the frame.
997  * \param frame What is being written.
998  *
999  * \retval 0 Frame accepted into the bridge.
1000  * \retval -1 Frame needs to be deferred.
1001  */
1002 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1003 {
1004         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1005
1006         /*
1007          * XXX Softmix needs to use channel roles to determine what to
1008          * do with control frames.
1009          */
1010
1011         switch (frame->subclass.integer) {
1012         case AST_CONTROL_VIDUPDATE:
1013                 if (!bridge->softmix.video_mode.video_update_discard ||
1014                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1015                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1016                         softmix_data->last_video_update = ast_tvnow();
1017                 }
1018                 break;
1019         default:
1020                 break;
1021         }
1022
1023         return 0;
1024 }
1025
1026 /*!
1027  * \internal
1028  * \brief Determine what to do with a frame written into the bridge.
1029  * \since 12.0.0
1030  *
1031  * \param bridge Which bridge is getting the frame
1032  * \param bridge_channel Which channel is writing the frame.
1033  * \param frame What is being written.
1034  *
1035  * \retval 0 Frame accepted into the bridge.
1036  * \retval -1 Frame needs to be deferred.
1037  *
1038  * \note On entry, bridge is already locked.
1039  */
1040 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1041 {
1042         int res = 0;
1043
1044         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1045                 /* "Accept" the frame and discard it. */
1046                 return 0;
1047         }
1048
1049         /*
1050          * XXX Softmix needs to use channel roles to determine who gets
1051          * what frame.  Possible roles: announcer, recorder, agent,
1052          * supervisor.
1053          */
1054         switch (frame->frametype) {
1055         case AST_FRAME_NULL:
1056                 /* "Accept" the frame and discard it. */
1057                 break;
1058         case AST_FRAME_DTMF_BEGIN:
1059         case AST_FRAME_DTMF_END:
1060                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1061                 break;
1062         case AST_FRAME_VOICE:
1063                 if (bridge_channel) {
1064                         softmix_bridge_write_voice(bridge, bridge_channel, frame);
1065                 }
1066                 break;
1067         case AST_FRAME_VIDEO:
1068                 if (bridge_channel) {
1069                         softmix_bridge_write_video(bridge, bridge_channel, frame);
1070                 }
1071                 break;
1072         case AST_FRAME_CONTROL:
1073                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1074                 break;
1075         case AST_FRAME_BRIDGE_ACTION:
1076                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1077                 break;
1078         case AST_FRAME_BRIDGE_ACTION_SYNC:
1079                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1080                 ast_assert(0);
1081         default:
1082                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1083                 /* "Accept" the frame and discard it. */
1084                 break;
1085         }
1086
1087         return res;
1088 }
1089
1090 static void gather_softmix_stats(struct softmix_stats *stats,
1091         const struct softmix_bridge_data *softmix_data,
1092         struct ast_bridge_channel *bridge_channel)
1093 {
1094         int channel_native_rate;
1095
1096         /* Gather stats about channel sample rates. */
1097         ast_channel_lock(bridge_channel->chan);
1098         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1099                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1100         ast_channel_unlock(bridge_channel->chan);
1101
1102         if (stats->highest_supported_rate < channel_native_rate) {
1103                 stats->highest_supported_rate = channel_native_rate;
1104         }
1105         if (softmix_data->internal_rate < channel_native_rate) {
1106                 int i;
1107
1108                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1109                         if (stats->sample_rates[i] == channel_native_rate) {
1110                                 stats->num_channels[i]++;
1111                                 break;
1112                         } else if (!stats->sample_rates[i]) {
1113                                 stats->sample_rates[i] = channel_native_rate;
1114                                 stats->num_channels[i]++;
1115                                 break;
1116                         }
1117                 }
1118                 stats->num_above_internal_rate++;
1119         } else if (softmix_data->internal_rate == channel_native_rate) {
1120                 stats->num_at_internal_rate++;
1121         }
1122 }
1123
1124 /*!
1125  * \internal
1126  * \brief Analyse mixing statistics and change bridges internal rate
1127  * if necessary.
1128  *
1129  * \retval 0, no changes to internal rate
1130  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1131  */
1132 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1133                 struct softmix_bridge_data *softmix_data, int binaural_active)
1134 {
1135         int i;
1136
1137         if (binaural_active) {
1138                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1139         }
1140
1141         /*
1142          * Re-adjust the internal bridge sample rate if
1143          * 1. The bridge's internal sample rate is locked in at a sample
1144          *    rate other than the current sample rate being used.
1145          * 2. two or more channels support a higher sample rate
1146          * 3. no channels support the current sample rate or a higher rate
1147          */
1148         if (stats->locked_rate) {
1149                 /* if the rate is locked by the bridge, only update it if it differs
1150                  * from the current rate we are using. */
1151                 if (softmix_data->internal_rate != stats->locked_rate) {
1152                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1153                                 softmix_data->internal_rate, stats->locked_rate);
1154                         softmix_data->internal_rate = stats->locked_rate;
1155                         return 1;
1156                 }
1157         } else if (stats->num_above_internal_rate >= 2) {
1158                 /* the highest rate is just used as a starting point */
1159                 unsigned int best_rate = stats->highest_supported_rate;
1160                 int best_index = -1;
1161
1162                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1163                         if (stats->num_channels[i]) {
1164                                 break;
1165                         }
1166                         if (2 <= stats->num_channels[i]) {
1167                                 /* Two or more channels support this rate. */
1168                                 if (best_index == -1
1169                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1170                                         /*
1171                                          * best_rate starts out being the first sample rate
1172                                          * greater than the internal sample rate that two or
1173                                          * more channels support.
1174                                          *
1175                                          * or
1176                                          *
1177                                          * There are multiple rates above the internal rate
1178                                          * and this rate is higher than the previous rate two
1179                                          * or more channels support.
1180                                          */
1181                                         best_rate = stats->sample_rates[i];
1182                                         best_index = i;
1183                                 }
1184                         } else if (best_index == -1) {
1185                                 /*
1186                                  * It is possible that multiple channels exist with native sample
1187                                  * rates above the internal sample rate, but none of those channels
1188                                  * have the same rate in common.  In this case, the lowest sample
1189                                  * rate among those channels is picked. Over time as additional
1190                                  * statistic runs are made the internal sample rate number will
1191                                  * adjust to the most optimal sample rate, but it may take multiple
1192                                  * iterations.
1193                                  */
1194                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1195                         }
1196                 }
1197
1198                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1199                         softmix_data->internal_rate, best_rate);
1200                 softmix_data->internal_rate = best_rate;
1201                 return 1;
1202         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1203                 /* In this case, the highest supported rate is actually lower than the internal rate */
1204                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1205                         softmix_data->internal_rate, stats->highest_supported_rate);
1206                 softmix_data->internal_rate = stats->highest_supported_rate;
1207                 return 1;
1208         }
1209         return 0;
1210 }
1211
1212 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1213                 unsigned int starting_num_entries, unsigned int binaural_active)
1214 {
1215         memset(mixing_array, 0, sizeof(*mixing_array));
1216         mixing_array->max_num_entries = starting_num_entries;
1217         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1218                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1219                 return -1;
1220         }
1221         if (binaural_active) {
1222                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1223                                 sizeof(struct convolve_channel_pair *)))) {
1224                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1225                         return -1;
1226                 }
1227         }
1228         return 0;
1229 }
1230
1231 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1232                 unsigned int binaural_active)
1233 {
1234         ast_free(mixing_array->buffers);
1235         if (binaural_active) {
1236                 ast_free(mixing_array->chan_pairs);
1237         }
1238 }
1239
1240 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1241                 unsigned int num_entries, unsigned int binaural_active)
1242 {
1243         int16_t **tmp;
1244
1245         /* give it some room to grow since memory is cheap but allocations can be expensive */
1246         mixing_array->max_num_entries = num_entries;
1247         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1248                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1249                 return -1;
1250         }
1251         if (binaural_active) {
1252                 struct convolve_channel_pair **tmp2;
1253                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1254                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1255                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1256                         return -1;
1257                 }
1258                 mixing_array->chan_pairs = tmp2;
1259         }
1260         mixing_array->buffers = tmp;
1261         return 0;
1262 }
1263
1264 /*!
1265  * \brief Mixing loop.
1266  *
1267  * \retval 0 on success
1268  * \retval -1 on failure
1269  */
1270 static int softmix_mixing_loop(struct ast_bridge *bridge)
1271 {
1272         struct softmix_stats stats = { { 0 }, };
1273         struct softmix_mixing_array mixing_array;
1274         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1275         struct ast_timer *timer;
1276         struct softmix_translate_helper trans_helper;
1277         int16_t buf[MAX_DATALEN];
1278 #ifdef BINAURAL_RENDERING
1279         int16_t bin_buf[MAX_DATALEN];
1280         int16_t ann_buf[MAX_DATALEN];
1281 #endif
1282         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1283         int timingfd;
1284         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1285         unsigned int idx;
1286         unsigned int x;
1287         int res = -1;
1288
1289         timer = softmix_data->timer;
1290         timingfd = ast_timer_fd(timer);
1291         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1292         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1293
1294         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1295         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1296                         bridge->softmix.binaural_active)) {
1297                 goto softmix_cleanup;
1298         }
1299
1300         /*
1301          * XXX Softmix needs to use channel roles to determine who gets
1302          * what audio mixed.
1303          */
1304         while (!softmix_data->stop && bridge->num_active) {
1305                 struct ast_bridge_channel *bridge_channel;
1306                 int timeout = -1;
1307                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1308                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1309                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1310
1311                 if (softmix_datalen > MAX_DATALEN) {
1312                         /* This should NEVER happen, but if it does we need to know about it. Almost
1313                          * all the memcpys used during this process depend on this assumption.  Rather
1314                          * than checking this over and over again through out the code, this single
1315                          * verification is done on each iteration. */
1316                         ast_log(LOG_WARNING,
1317                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1318                                 bridge->uniqueid);
1319                         goto softmix_cleanup;
1320                 }
1321
1322                 /* Grow the mixing array buffer as participants are added. */
1323                 if (mixing_array.max_num_entries < bridge->num_channels
1324                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1325                                         bridge->softmix.binaural_active)) {
1326                         goto softmix_cleanup;
1327                 }
1328
1329                 /* init the number of buffers stored in the mixing array to 0.
1330                  * As buffers are added for mixing, this number is incremented. */
1331                 mixing_array.used_entries = 0;
1332
1333                 /* These variables help determine if a rate change is required */
1334                 if (!stat_iteration_counter) {
1335                         memset(&stats, 0, sizeof(stats));
1336                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1337                 }
1338
1339                 /* If the sample rate has changed, update the translator helper */
1340                 if (update_all_rates) {
1341                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1342                 }
1343
1344 #ifdef BINAURAL_RENDERING
1345                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1346 #endif
1347
1348                 /* Go through pulling audio from each factory that has it available */
1349                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1350                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1351
1352                         if (!sc) {
1353                                 /* This channel failed to join successfully. */
1354                                 continue;
1355                         }
1356
1357                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1358                         if (update_all_rates) {
1359                                 set_softmix_bridge_data(softmix_data->internal_rate,
1360                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1361                         }
1362
1363                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1364                         if (!stat_iteration_counter) {
1365                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1366                         }
1367
1368                         /* if the channel is suspended, don't check for audio, but still gather stats */
1369                         if (bridge_channel->suspended) {
1370                                 continue;
1371                         }
1372
1373                         /* Try to get audio from the factory if available */
1374                         ast_mutex_lock(&sc->lock);
1375                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1376 #ifdef BINAURAL_RENDERING
1377                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1378                                                 ast_channel_name(bridge_channel->chan));
1379 #endif
1380                                 mixing_array.used_entries++;
1381                         }
1382                         ast_mutex_unlock(&sc->lock);
1383                 }
1384
1385                 /* mix it like crazy (non binaural channels)*/
1386                 memset(buf, 0, softmix_datalen);
1387                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1388                         for (x = 0; x < softmix_samples; ++x) {
1389                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1390                         }
1391                 }
1392
1393 #ifdef BINAURAL_RENDERING
1394                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1395 #endif
1396
1397                 /* Next step go through removing the channel's own audio and creating a good frame... */
1398                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1399                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1400
1401                         if (!sc || bridge_channel->suspended) {
1402                                 /* This channel failed to join successfully or is suspended. */
1403                                 continue;
1404                         }
1405
1406                         ast_mutex_lock(&sc->lock);
1407
1408                         /* Make SLINEAR write frame from local buffer */
1409                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1410                                 "Replace softmix channel slin format");
1411 #ifdef BINAURAL_RENDERING
1412                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1413                                         && sc->binaural) {
1414                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1415                                                 softmix_samples, buf);
1416                         } else
1417 #endif
1418                         {
1419                                 sc->write_frame.datalen = softmix_datalen;
1420                                 sc->write_frame.samples = softmix_samples;
1421                                 memcpy(sc->final_buf, buf, softmix_datalen);
1422                         }
1423                         /* process the softmix channel's new write audio */
1424                         softmix_process_write_audio(&trans_helper,
1425                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1426                                         softmix_data->default_sample_size);
1427
1428                         ast_mutex_unlock(&sc->lock);
1429
1430                         /* A frame is now ready for the channel. */
1431                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1432                 }
1433
1434                 update_all_rates = 0;
1435                 if (!stat_iteration_counter) {
1436                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1437                                         bridge->softmix.binaural_active);
1438                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1439                 }
1440                 stat_iteration_counter--;
1441
1442                 ast_bridge_unlock(bridge);
1443                 /* cleanup any translation frame data from the previous mixing iteration. */
1444                 softmix_translate_helper_cleanup(&trans_helper);
1445                 /* Wait for the timing source to tell us to wake up and get things done */
1446                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1447                 if (ast_timer_ack(timer, 1) < 0) {
1448                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1449                                 bridge->uniqueid);
1450                         ast_bridge_lock(bridge);
1451                         goto softmix_cleanup;
1452                 }
1453                 ast_bridge_lock(bridge);
1454
1455                 /* make sure to detect mixing interval changes if they occur. */
1456                 if (bridge->softmix.internal_mixing_interval
1457                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1458                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1459                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1460                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1461                 }
1462         }
1463
1464         res = 0;
1465
1466 softmix_cleanup:
1467         softmix_translate_helper_destroy(&trans_helper);
1468         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1469         return res;
1470 }
1471
1472 /*!
1473  * \internal
1474  * \brief Mixing thread.
1475  * \since 12.0.0
1476  *
1477  * \note The thread does not have its own reference to the
1478  * bridge.  The lifetime of the thread is tied to the lifetime
1479  * of the mixing technology association with the bridge.
1480  */
1481 static void *softmix_mixing_thread(void *data)
1482 {
1483         struct softmix_bridge_data *softmix_data = data;
1484         struct ast_bridge *bridge = softmix_data->bridge;
1485
1486         ast_bridge_lock(bridge);
1487         if (bridge->callid) {
1488                 ast_callid_threadassoc_add(bridge->callid);
1489         }
1490
1491         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1492
1493         while (!softmix_data->stop) {
1494                 if (!bridge->num_active) {
1495                         /* Wait for something to happen to the bridge. */
1496                         ast_bridge_unlock(bridge);
1497                         ast_mutex_lock(&softmix_data->lock);
1498                         if (!softmix_data->stop) {
1499                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1500                         }
1501                         ast_mutex_unlock(&softmix_data->lock);
1502                         ast_bridge_lock(bridge);
1503                         continue;
1504                 }
1505
1506                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1507 #ifndef BINAURAL_RENDERING
1508                         ast_bridge_lock(bridge);
1509                         bridge->softmix.binaural_active = 0;
1510                         ast_bridge_unlock(bridge);
1511                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1512                                         "compiled.\n", bridge->uniqueid);
1513 #else
1514                         /* Set and init binaural data if binaural is activated in the configuration. */
1515                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1516                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1517                                         softmix_data->internal_mixing_interval);
1518                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1519                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1520                                         == -1) {
1521                                 ast_bridge_lock(bridge);
1522                                 bridge->softmix.binaural_active = 0;
1523                                 ast_bridge_unlock(bridge);
1524                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1525                                                 "binaural processing,  Will only process mono audio.\n",
1526                                                 bridge->uniqueid);
1527                         }
1528                         softmix_data->binaural_init = 1;
1529 #endif
1530                 }
1531
1532                 if (softmix_mixing_loop(bridge)) {
1533                         /*
1534                          * A mixing error occurred.  Sleep and try again later so we
1535                          * won't flood the logs.
1536                          */
1537                         ast_bridge_unlock(bridge);
1538                         sleep(1);
1539                         ast_bridge_lock(bridge);
1540                 }
1541         }
1542
1543         ast_bridge_unlock(bridge);
1544
1545         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1546
1547         return NULL;
1548 }
1549
1550 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1551 {
1552         if (softmix_data->timer) {
1553                 ast_timer_close(softmix_data->timer);
1554                 softmix_data->timer = NULL;
1555         }
1556         ast_mutex_destroy(&softmix_data->lock);
1557         ast_cond_destroy(&softmix_data->cond);
1558         ast_free(softmix_data);
1559 }
1560
1561 /*! \brief Function called when a bridge is created */
1562 static int softmix_bridge_create(struct ast_bridge *bridge)
1563 {
1564         struct softmix_bridge_data *softmix_data;
1565
1566         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1567         if (!softmix_data) {
1568                 return -1;
1569         }
1570         softmix_data->bridge = bridge;
1571         ast_mutex_init(&softmix_data->lock);
1572         ast_cond_init(&softmix_data->cond, NULL);
1573         softmix_data->timer = ast_timer_open();
1574         if (!softmix_data->timer) {
1575                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1576                 softmix_bridge_data_destroy(softmix_data);
1577                 return -1;
1578         }
1579         /* start at minimum rate, let it grow from there */
1580         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1581         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1582
1583 #ifdef BINAURAL_RENDERING
1584         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1585                         softmix_data->internal_mixing_interval);
1586 #endif
1587
1588         bridge->tech_pvt = softmix_data;
1589
1590         /* Start the mixing thread. */
1591         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1592                 softmix_data)) {
1593                 softmix_data->thread = AST_PTHREADT_NULL;
1594                 softmix_bridge_data_destroy(softmix_data);
1595                 bridge->tech_pvt = NULL;
1596                 return -1;
1597         }
1598
1599         return 0;
1600 }
1601
1602 /*!
1603  * \internal
1604  * \brief Request the softmix mixing thread stop.
1605  * \since 12.0.0
1606  *
1607  * \param bridge Which bridge is being stopped.
1608  *
1609  * \return Nothing
1610  */
1611 static void softmix_bridge_stop(struct ast_bridge *bridge)
1612 {
1613         struct softmix_bridge_data *softmix_data;
1614
1615         softmix_data = bridge->tech_pvt;
1616         if (!softmix_data) {
1617                 return;
1618         }
1619
1620         ast_mutex_lock(&softmix_data->lock);
1621         softmix_data->stop = 1;
1622         ast_mutex_unlock(&softmix_data->lock);
1623 }
1624
1625 /*! \brief Function called when a bridge is destroyed */
1626 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1627 {
1628         struct softmix_bridge_data *softmix_data;
1629         pthread_t thread;
1630
1631         softmix_data = bridge->tech_pvt;
1632         if (!softmix_data) {
1633                 return;
1634         }
1635
1636         /* Stop the mixing thread. */
1637         ast_mutex_lock(&softmix_data->lock);
1638         softmix_data->stop = 1;
1639         ast_cond_signal(&softmix_data->cond);
1640         thread = softmix_data->thread;
1641         softmix_data->thread = AST_PTHREADT_NULL;
1642         ast_mutex_unlock(&softmix_data->lock);
1643         if (thread != AST_PTHREADT_NULL) {
1644                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1645                 pthread_join(thread, NULL);
1646         }
1647 #ifdef BINAURAL_RENDERING
1648         free_convolve_data(&softmix_data->convolve);
1649 #endif
1650         softmix_bridge_data_destroy(softmix_data);
1651         bridge->tech_pvt = NULL;
1652 }
1653
1654 /*!
1655  * \brief Map a source stream to all of its destination streams.
1656  *
1657  * \param source_stream_name Name of the source stream
1658  * \param source_channel_name Name of channel where the source stream originates
1659  * \param bridge_stream_position The slot in the bridge where source video will come from
1660  * \param participants The bridge_channels in the bridge
1661  */
1662 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1663         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1664 {
1665         struct ast_bridge_channel *participant;
1666
1667         AST_LIST_TRAVERSE(participants, participant, entry) {
1668                 int i;
1669                 struct ast_stream_topology *topology;
1670
1671                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1672                         continue;
1673                 }
1674
1675                 ast_bridge_channel_lock(participant);
1676                 ast_channel_lock(participant->chan);
1677                 topology = ast_channel_get_stream_topology(participant->chan);
1678
1679                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1680                         struct ast_stream *stream;
1681
1682                         stream = ast_stream_topology_get_stream(topology, i);
1683                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1684                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1685                                 break;
1686                         }
1687                 }
1688                 ast_channel_unlock(participant->chan);
1689                 ast_bridge_channel_unlock(participant);
1690         }
1691 }
1692
1693 /*\brief stream_topology_changed callback
1694  *
1695  * For most video modes, nothing beyond the ordinary is required.
1696  * For the SFU case, though, we need to completely remap the streams
1697  * in order to ensure video gets directed where it is expected to go.
1698  *
1699  * \param bridge The bridge
1700  * \param bridge_channel Channel whose topology has changed
1701  */
1702 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1703 {
1704         struct ast_bridge_channel *participant;
1705         struct ast_vector_int media_types;
1706         int nths[AST_MEDIA_TYPE_END] = {0};
1707
1708         switch (bridge->softmix.video_mode.mode) {
1709         case AST_BRIDGE_VIDEO_MODE_NONE:
1710         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1711         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1712         default:
1713                 ast_bridge_channel_stream_map(bridge_channel);
1714                 return;
1715         case AST_BRIDGE_VIDEO_MODE_SFU:
1716                 break;
1717         }
1718
1719         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1720
1721         /* First traversal: re-initialize all of the participants' stream maps */
1722         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1723                 ast_bridge_channel_lock(participant);
1724                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1725                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1726                 ast_bridge_channel_unlock(participant);
1727         }
1728
1729         /* Second traversal: Map specific video channels from their source to their destinations.
1730          *
1731          * This is similar to what is done in ast_stream_topology_map(), except that
1732          * video channels are handled differently. Each video source has it's own
1733          * unique index on the bridge. this way, a particular channel's source video
1734          * can be distributed to the appropriate destination streams on the other
1735          * channels
1736          */
1737         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1738                 int i;
1739                 struct ast_stream_topology *topology;
1740
1741                 ast_channel_lock(participant->chan);
1742
1743                 topology = ast_channel_get_stream_topology(participant->chan);
1744
1745                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1746                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1747                         ast_bridge_channel_lock(participant);
1748                         if (is_video_source(stream)) {
1749                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1750                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1751                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1752                                 /* Unlock the participant to prevent potential deadlock
1753                                  * in map_source_to_destinations
1754                                  */
1755                                 ast_bridge_channel_unlock(participant);
1756                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1757                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1758                                 ast_bridge_channel_lock(participant);
1759                         } else if (is_video_dest(stream, NULL, NULL)) {
1760                                 /* We expect to never read media from video destination channels, but just
1761                                  * in case, we should set their to_bridge value to -1.
1762                                  */
1763                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1764                         } else {
1765                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1766                                  * be factored out in some way
1767                                  */
1768                                 enum ast_media_type type = ast_stream_get_type(stream);
1769                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1770                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1771
1772                                 if (index == -1) {
1773                                         AST_VECTOR_APPEND(&media_types, type);
1774                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1775                                 }
1776
1777                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1778                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1779                         }
1780                         ast_bridge_channel_unlock(participant);
1781                 }
1782
1783                 ast_channel_unlock(participant->chan);
1784         }
1785 }
1786
1787 static struct ast_bridge_technology softmix_bridge = {
1788         .name = "softmix",
1789         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1790         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1791         .create = softmix_bridge_create,
1792         .stop = softmix_bridge_stop,
1793         .destroy = softmix_bridge_destroy,
1794         .join = softmix_bridge_join,
1795         .leave = softmix_bridge_leave,
1796         .unsuspend = softmix_bridge_unsuspend,
1797         .write = softmix_bridge_write,
1798         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1799 };
1800
1801 #ifdef TEST_FRAMEWORK
1802 struct stream_parameters {
1803         const char *name;
1804         const char *formats;
1805         enum ast_media_type type;
1806 };
1807
1808 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1809 {
1810         struct ast_stream_topology *topology;
1811         size_t i;
1812
1813         topology = ast_stream_topology_alloc();
1814         if (!topology) {
1815                 return NULL;
1816         }
1817
1818         for (i = 0; i < num_streams; ++i) {
1819                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1820                 struct ast_stream *stream;
1821
1822                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1823                 if (!caps) {
1824                         goto fail;
1825                 }
1826                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1827                         goto fail;
1828                 }
1829                 stream = ast_stream_alloc(params[i].name, params[i].type);
1830                 if (!stream) {
1831                         goto fail;
1832                 }
1833                 ast_stream_set_formats(stream, caps);
1834                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1835                         ast_stream_free(stream);
1836                         goto fail;
1837                 }
1838         }
1839
1840         return topology;
1841
1842 fail:
1843         ast_stream_topology_free(topology);
1844         return NULL;
1845 }
1846
1847 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1848         const struct stream_parameters *params)
1849 {
1850         struct ast_format_cap *stream_caps;
1851         struct ast_format_cap *params_caps;
1852
1853         if (ast_stream_get_type(stream) != params->type) {
1854                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1855                         ast_codec_media_type2str(params->type),
1856                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1857                 return -1;
1858         }
1859         if (strcmp(ast_stream_get_name(stream), params->name)) {
1860                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1861                         params->name, ast_stream_get_name(stream));
1862                 return -1;
1863         }
1864
1865         stream_caps = ast_stream_get_formats(stream);
1866         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1867         if (!params_caps) {
1868                 ast_test_status_update(test, "Allocation error on capabilities\n");
1869                 return -1;
1870         }
1871         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
1872
1873         if (ast_format_cap_identical(stream_caps, params_caps)) {
1874                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
1875                         ast_stream_get_name(stream));
1876                 ao2_cleanup(params_caps);
1877                 return -1;
1878         }
1879
1880         ao2_cleanup(params_caps);
1881         return 0;
1882 }
1883
1884 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
1885         const struct stream_parameters *params, size_t num_streams)
1886 {
1887         int i;
1888
1889         if (ast_stream_topology_get_count(topology) < num_streams) {
1890                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
1891                         ast_stream_topology_get_count(topology), num_streams);
1892                 return -1;
1893         }
1894
1895         for (i = 0; i < ARRAY_LEN(params); ++i) {
1896                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
1897                         return -1;
1898                 }
1899         }
1900
1901         return 0;
1902 }
1903
1904 AST_TEST_DEFINE(sfu_append_source_streams)
1905 {
1906         enum ast_test_result_state res = AST_TEST_FAIL;
1907         static const struct stream_parameters bob_streams[] = {
1908                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1909                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1910         };
1911         static const struct stream_parameters alice_streams[] = {
1912                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
1913                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
1914         };
1915         static const struct stream_parameters alice_dest_stream = {
1916                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
1917         };
1918         static const struct stream_parameters bob_dest_stream = {
1919                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
1920         };
1921         struct ast_stream_topology *topology_alice = NULL;
1922         struct ast_stream_topology *topology_bob = NULL;
1923
1924         switch (cmd) {
1925         case TEST_INIT:
1926                 info->name = "sfu_append_source_streams";
1927                 info->category = "/bridges/bridge_softmix/";
1928                 info->summary = "Test appending of video streams";
1929                 info->description =
1930                         "This tests does stuff.";
1931                 return AST_TEST_NOT_RUN;
1932         case TEST_EXECUTE:
1933                 break;
1934         }
1935
1936         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
1937         if (!topology_alice) {
1938                 goto end;
1939         }
1940
1941         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
1942         if (!topology_bob) {
1943                 goto end;
1944         }
1945
1946         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
1947                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
1948                 goto end;
1949         }
1950
1951         if (ast_stream_topology_get_count(topology_alice) != 3) {
1952                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
1953                         ast_stream_topology_get_count(topology_alice), 3);
1954                 goto end;
1955         }
1956
1957         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
1958                 goto end;
1959         }
1960
1961         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
1962                 goto end;
1963         }
1964
1965         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
1966                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
1967                 goto end;
1968         }
1969
1970         if (ast_stream_topology_get_count(topology_bob) != 3) {
1971                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
1972                         ast_stream_topology_get_count(topology_bob), 3);
1973                 goto end;
1974         }
1975
1976         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
1977                 goto end;
1978         }
1979
1980         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
1981                 goto end;
1982         }
1983
1984         res = AST_TEST_PASS;
1985
1986 end:
1987         ast_stream_topology_free(topology_alice);
1988         ast_stream_topology_free(topology_bob);
1989         return res;
1990 }
1991
1992 AST_TEST_DEFINE(sfu_remove_destination_streams)
1993 {
1994         enum ast_test_result_state res = AST_TEST_FAIL;
1995         static const struct stream_parameters params[] = {
1996                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1997                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1998                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
1999                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2000         };
2001         static const struct {
2002                 const char *channel_name;
2003                 int num_streams;
2004                 int params_index[4];
2005         } removal_results[] = {
2006                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2007                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2008                 { "", 4, { 0, 1, 2, 3 }, },
2009         };
2010         struct ast_stream_topology *orig = NULL;
2011         struct ast_stream_topology *result = NULL;
2012         int i;
2013
2014         switch (cmd) {
2015         case TEST_INIT:
2016                 info->name = "sfu_remove_destination_streams";
2017                 info->category = "/bridges/bridge_softmix/";
2018                 info->summary = "Test removal of destination video streams";
2019                 info->description =
2020                         "This tests does stuff.";
2021                 return AST_TEST_NOT_RUN;
2022         case TEST_EXECUTE:
2023                 break;
2024         }
2025
2026         orig = build_topology(params, ARRAY_LEN(params));
2027         if (!orig) {
2028                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2029                 goto end;
2030         }
2031
2032         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2033                 int j;
2034
2035                 result = ast_stream_topology_alloc();
2036                 if (!result) {
2037                         ast_test_status_update(test, "Unable to allocate result stream topology\n");
2038                         goto end;
2039                 }
2040
2041                 if (remove_destination_streams(result, removal_results[i].channel_name, orig)) {
2042                         ast_test_status_update(test, "Failure while attempting to remove video streams\n");
2043                         goto end;
2044                 }
2045
2046                 if (ast_stream_topology_get_count(result) != removal_results[i].num_streams) {
2047                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2048                                 ast_stream_topology_get_count(result), removal_results[i].num_streams);
2049                         goto end;
2050                 }
2051
2052                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2053                         struct ast_stream *actual;
2054                         struct ast_stream *expected;
2055                         int orig_index;
2056
2057                         actual = ast_stream_topology_get_stream(result, j);
2058
2059                         orig_index = removal_results[i].params_index[j];
2060                         expected = ast_stream_topology_get_stream(orig, orig_index);
2061
2062                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2063                                 ast_stream_get_formats(expected))) {
2064                                 struct ast_str *expected_str;
2065                                 struct ast_str *actual_str;
2066
2067                                 expected_str = ast_str_alloca(64);
2068                                 actual_str = ast_str_alloca(64);
2069
2070                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2071                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2072                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2073                                 goto end;
2074                         }
2075
2076                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2077                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2078                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2079                                 goto end;
2080                         }
2081                 }
2082         }
2083
2084         res = AST_TEST_PASS;
2085
2086 end:
2087         ast_stream_topology_free(orig);
2088         ast_stream_topology_free(result);
2089         return res;
2090 }
2091
2092 #endif
2093
2094 static int unload_module(void)
2095 {
2096         ast_bridge_technology_unregister(&softmix_bridge);
2097         AST_TEST_UNREGISTER(sfu_append_source_streams);
2098         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2099         return 0;
2100 }
2101
2102 static int load_module(void)
2103 {
2104         if (ast_bridge_technology_register(&softmix_bridge)) {
2105                 unload_module();
2106                 return AST_MODULE_LOAD_DECLINE;
2107         }
2108         AST_TEST_REGISTER(sfu_append_source_streams);
2109         AST_TEST_REGISTER(sfu_remove_destination_streams);
2110         return AST_MODULE_LOAD_SUCCESS;
2111 }
2112
2113 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");