Merge "bridge_softmix.c: Report not talking immediately when muted."
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /*!
57  * \brief Default time in ms of silence necessary to declare talking stopped by the bridge.
58  *
59  * \details
60  * This is the time at which a channel's own audio will stop getting
61  * mixed out of its own write audio stream because it is no longer talking.
62  */
63 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
64
65 /*! Default minimum average magnitude threshold to determine talking by the DSP. */
66 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
67
68 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
69 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
70 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
71
72 struct softmix_stats {
73         /*! Each index represents a sample rate used above the internal rate. */
74         unsigned int sample_rates[16];
75         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
76         unsigned int num_channels[16];
77         /*! The number of channels above the internal sample rate */
78         unsigned int num_above_internal_rate;
79         /*! The number of channels at the internal sample rate */
80         unsigned int num_at_internal_rate;
81         /*! The absolute highest sample rate preferred by any channel in the bridge */
82         unsigned int highest_supported_rate;
83         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
84         unsigned int locked_rate;
85 };
86
87 struct softmix_translate_helper_entry {
88         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
89                                                                   and re-init if it was usable. */
90         struct ast_format *dst_format; /*!< The destination format for this helper */
91         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
92         struct ast_frame *out_frame; /*!< The output frame from the last translation */
93         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
94 };
95
96 struct softmix_translate_helper {
97         struct ast_format *slin_src; /*!< the source format expected for all the translators */
98         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
99 };
100
101 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
102 {
103         struct softmix_translate_helper_entry *entry;
104         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
105                 return NULL;
106         }
107         entry->dst_format = ao2_bump(dst);
108         /* initialize this to one so that the first time through the cleanup code after
109            allocation it won't be removed from the entry list */
110         entry->num_times_requested = 1;
111         return entry;
112 }
113
114 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
115 {
116         ao2_cleanup(entry->dst_format);
117
118         if (entry->trans_pvt) {
119                 ast_translator_free_path(entry->trans_pvt);
120         }
121         if (entry->out_frame) {
122                 ast_frfree(entry->out_frame);
123         }
124         ast_free(entry);
125         return NULL;
126 }
127
128 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
129 {
130         memset(trans_helper, 0, sizeof(*trans_helper));
131         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
132 }
133
134 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
135 {
136         struct softmix_translate_helper_entry *entry;
137
138         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
139                 softmix_translate_helper_free_entry(entry);
140         }
141 }
142
143 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
144 {
145         struct softmix_translate_helper_entry *entry;
146
147         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
148         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
149                 if (entry->trans_pvt) {
150                         ast_translator_free_path(entry->trans_pvt);
151                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
152                                 AST_LIST_REMOVE_CURRENT(entry);
153                                 entry = softmix_translate_helper_free_entry(entry);
154                         }
155                 }
156         }
157         AST_LIST_TRAVERSE_SAFE_END;
158 }
159
160 /*!
161  * \internal
162  * \brief Get the next available audio on the softmix channel's read stream
163  * and determine if it should be mixed out or not on the write stream.
164  *
165  * \retval pointer to buffer containing the exact number of samples requested on success.
166  * \retval NULL if no samples are present
167  */
168 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
169 {
170         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
171                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
172                 sc->have_audio = 1;
173                 return sc->our_buf;
174         }
175         sc->have_audio = 0;
176         return NULL;
177 }
178
179 /*!
180  * \internal
181  * \brief Process a softmix channel's write audio
182  *
183  * \details This function will remove the channel's talking from its own audio if present and
184  * possibly even do the channel's write translation for it depending on how many other
185  * channels use the same write format.
186  */
187 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
188         struct ast_format *raw_write_fmt,
189         struct softmix_channel *sc, unsigned int default_sample_size)
190 {
191         struct softmix_translate_helper_entry *entry = NULL;
192         int i;
193
194         /* If we provided audio that was not determined to be silence,
195          * then take it out while in slinear format. */
196         if (sc->have_audio && sc->talking && !sc->binaural) {
197                 for (i = 0; i < sc->write_frame.samples; i++) {
198                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
199                 }
200                 /* check to see if any entries exist for the format. if not we'll want
201                    to remove it during cleanup */
202                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
203                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
204                                 ++entry->num_times_requested;
205                                 break;
206                         }
207                 }
208                 /* do not do any special write translate optimization if we had to make
209                  * a special mix for them to remove their own audio. */
210                 return;
211         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
212                 /*
213                  * Binaural audio requires special saturated substract since we have two
214                  * audio signals per channel now.
215                  */
216                 softmix_process_write_binaural_audio(sc, default_sample_size);
217                 return;
218         }
219
220         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
221            of translation paths and track the number of references for each type. Each one of the same
222            type should be able to use the same out_frame. Since the optimization is only necessary for
223            multiple channels (>=2) using the same codec make sure resources are allocated only when
224            needed and released when not (see also softmix_translate_helper_cleanup */
225         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
226                 if (sc->binaural != 0) {
227                         continue;
228                 }
229                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
230                         entry->num_times_requested++;
231                 } else {
232                         continue;
233                 }
234                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
235                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
236                 }
237                 if (entry->trans_pvt && !entry->out_frame) {
238                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
239                 }
240                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
241                                 && entry->out_frame->datalen < MAX_DATALEN) {
242                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
243                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
244                         sc->write_frame.datalen = entry->out_frame->datalen;
245                         sc->write_frame.samples = entry->out_frame->samples;
246                 }
247                 break;
248         }
249
250         /* add new entry into list if this format destination was not matched. */
251         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
252                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
253         }
254 }
255
256 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
257 {
258         struct softmix_translate_helper_entry *entry;
259
260         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
261                 /* if it hasn't been requested then remove it */
262                 if (!entry->num_times_requested) {
263                         AST_LIST_REMOVE_CURRENT(entry);
264                         softmix_translate_helper_free_entry(entry);
265                         continue;
266                 }
267
268                 if (entry->out_frame) {
269                         ast_frfree(entry->out_frame);
270                         entry->out_frame = NULL;
271                 }
272
273                 /* nothing is optimized for a single path reference, so there is
274                    no reason to continue to hold onto the codec */
275                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
276                         ast_translator_free_path(entry->trans_pvt);
277                         entry->trans_pvt = NULL;
278                 }
279
280                 /* for each iteration (a mixing run) in the bridge softmix thread the number
281                    of references to a given entry is recalculated, so reset the number of
282                    times requested */
283                 entry->num_times_requested = 0;
284         }
285         AST_LIST_TRAVERSE_SAFE_END;
286 }
287
288 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
289 {
290         struct softmix_channel *sc = bridge_channel->tech_pvt;
291         struct ast_format *slin_format;
292         int setup_fail;
293
294 #ifdef BINAURAL_RENDERING
295         if (interval != BINAURAL_MIXING_INTERVAL) {
296                 interval = BINAURAL_MIXING_INTERVAL;
297         }
298 #endif
299
300         /* The callers have already ensured that sc is never NULL. */
301         ast_assert(sc != NULL);
302
303         slin_format = ast_format_cache_get_slin_by_rate(rate);
304
305         ast_mutex_lock(&sc->lock);
306         if (reset) {
307                 ast_slinfactory_destroy(&sc->factory);
308                 ast_dsp_free(sc->dsp);
309         }
310
311         /* Setup write frame parameters */
312         sc->write_frame.frametype = AST_FRAME_VOICE;
313         /*
314          * NOTE: The write_frame format holds a reference because translation
315          * could be needed and the format changed to the translated format
316          * for the channel.  The translated format may not be a
317          * static cached format.
318          */
319         ao2_replace(sc->write_frame.subclass.format, slin_format);
320         sc->write_frame.data.ptr = sc->final_buf;
321         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
322         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
323
324         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
325         sc->rate = rate;
326
327         /* If the channel will contain binaural data we will set a identifier in the channel
328          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
329         if (set_binaural == 1) {
330                 sc->binaural = 1;
331         } else if (set_binaural == 0) {
332                 sc->binaural = 0;
333         }
334
335         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
336          * and doesn't have to be done if we just updating sample rates. */
337         if (binaural_pos_id != -1) {
338                 sc->binaural_pos = binaural_pos_id;
339         }
340         if (is_announcement != -1) {
341                 sc->is_announcement = is_announcement;
342         }
343
344         /*
345          * NOTE: The read_slin_format does not hold a reference because it
346          * will always be a signed linear format.
347          */
348         sc->read_slin_format = slin_format;
349
350         /* Setup smoother */
351         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
352
353         /* set new read and write formats on channel. */
354         ast_channel_lock(bridge_channel->chan);
355         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
356                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
357         ast_channel_unlock(bridge_channel->chan);
358
359         /* If channel contains binaural data we will set it here for the trans_pvt. */
360         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
361                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
362         } else if (set_binaural == 0) {
363                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
364         }
365
366         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
367         sc->dsp = ast_dsp_new_with_rate(rate);
368         if (setup_fail || !sc->dsp) {
369                 /* Bad news.  Could not setup the channel for softmix. */
370                 ast_mutex_unlock(&sc->lock);
371                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
372                 return;
373         }
374
375         /* we want to aggressively detect silence to avoid feedback */
376         if (bridge_channel->tech_args.talking_threshold) {
377                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
378         } else {
379                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
380         }
381
382         ast_mutex_unlock(&sc->lock);
383 }
384
385 /*!
386  * \internal
387  * \brief Poke the mixing thread in case it is waiting for an active channel.
388  * \since 12.0.0
389  *
390  * \param softmix_data Bridge mixing data.
391  *
392  * \return Nothing
393  */
394 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
395 {
396         ast_mutex_lock(&softmix_data->lock);
397         ast_cond_signal(&softmix_data->cond);
398         ast_mutex_unlock(&softmix_data->lock);
399 }
400
401 /*! \brief Function called when a channel is unsuspended from the bridge */
402 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
403 {
404 #ifdef BINAURAL_RENDERING
405         struct softmix_channel *sc = bridge_channel->tech_pvt;
406         if (sc->binaural) {
407                 /* Restore some usefull data if it was a binaural channel */
408                 struct ast_format *slin_format;
409
410                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
411                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
412         }
413 #endif
414         if (bridge->tech_pvt) {
415                 softmix_poke_thread(bridge->tech_pvt);
416         }
417 }
418
419 /*!
420  * \brief Determine if a stream is a video source stream.
421  *
422  * \param stream The stream to test
423  * \retval 1 The stream is a video source
424  * \retval 0 The stream is not a video source
425  */
426 static int is_video_source(const struct ast_stream *stream)
427 {
428         if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED
429                 && ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO
430                 && strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
431                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
432                 return 1;
433         }
434
435         return 0;
436 }
437
438 /*!
439  * \brief Determine if a stream is a video destination stream.
440  *
441  * A source channel name can be provided to narrow this to a destination stream
442  * for a particular source channel. Further, a source stream name can be provided
443  * to narrow this to a particular source stream's destination. However, empty strings
444  * can be provided to match any destination video stream, regardless of source channel
445  * or source stream.
446  *
447  * \param stream The stream to test
448  * \param source_channel_name The name of a source video channel to match
449  * \param source_stream_name The name of the source video stream to match
450  * \retval 1 The stream is a video destination stream
451  * \retval 0 The stream is not a video destination stream
452  */
453 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
454         const char *source_stream_name)
455 {
456         char *dest_video_name;
457         size_t dest_video_name_len;
458
459         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED
460                 || ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
461                 return 0;
462         }
463
464         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
465         if (!ast_strlen_zero(source_channel_name)) {
466                 dest_video_name_len += strlen(source_channel_name) + 1;
467                 if (!ast_strlen_zero(source_stream_name)) {
468                         dest_video_name_len += strlen(source_stream_name) + 1;
469                 }
470
471                 dest_video_name = ast_alloca(dest_video_name_len);
472                 if (!ast_strlen_zero(source_stream_name)) {
473                         /* We are looking for an exact stream name */
474                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
475                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
476                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
477                                 source_stream_name);
478                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
479                 }
480                 snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
481                         SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
482                         source_channel_name);
483         } else {
484                 dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
485         }
486
487         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
488 }
489
490 static int append_source_streams(struct ast_stream_topology *dest,
491         const char *channel_name,
492         const struct ast_stream_topology *source)
493 {
494         int i;
495
496         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
497                 struct ast_stream *stream;
498                 struct ast_stream *stream_clone;
499                 char *stream_clone_name = NULL;
500
501                 stream = ast_stream_topology_get_stream(source, i);
502                 if (!is_video_source(stream)) {
503                         continue;
504                 }
505
506                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
507                         channel_name, ast_stream_get_name(stream)) < 0) {
508                         return -1;
509                 }
510
511                 stream_clone = ast_stream_clone(stream, stream_clone_name);
512                 ast_free(stream_clone_name);
513                 if (!stream_clone) {
514                         return -1;
515                 }
516                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
517                         ast_stream_free(stream_clone);
518                         return -1;
519                 }
520         }
521
522         return 0;
523 }
524
525 static int append_all_streams(struct ast_stream_topology *dest,
526         const struct ast_stream_topology *source)
527 {
528         int i;
529         int dest_index = 0;
530
531         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
532                 struct ast_stream *clone;
533                 int added = 0;
534
535                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
536                 if (!clone) {
537                         return -1;
538                 }
539
540                 /* If we can reuse an existing removed stream then do so */
541                 while (dest_index < ast_stream_topology_get_count(dest)) {
542                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
543
544                         dest_index++;
545
546                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
547                                 /* This cannot fail because dest_index - 1 is less than the
548                                  * current count in dest. */
549                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
550                                 added = 1;
551                                 break;
552                         }
553                 }
554
555                 /* If no removed stream exists that we took the place of append the stream */
556                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
557                         ast_stream_free(clone);
558                         return -1;
559                 }
560         }
561
562         return 0;
563 }
564
565 /*!
566  * \brief Issue channel stream topology change requests.
567  *
568  * When in SFU mode, each participant needs to be able to
569  * send video directly to other participants in the bridge.
570  * This means that all participants need to have their topologies
571  * updated. The joiner needs to have destination streams for
572  * all current participants, and the current participants need
573  * to have destinations streams added for the joiner's sources.
574  *
575  * \param joiner The channel that is joining the softmix bridge
576  * \param participants The current participants in the softmix bridge
577  */
578 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
579 {
580         struct ast_stream_topology *joiner_video = NULL;
581         struct ast_bridge_channel *participant;
582         int res;
583         struct softmix_channel *sc;
584
585         joiner_video = ast_stream_topology_alloc();
586         if (!joiner_video) {
587                 return;
588         }
589
590         sc = joiner->tech_pvt;
591
592         ast_channel_lock(joiner->chan);
593         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
594         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
595         ast_channel_unlock(joiner->chan);
596
597         if (res || !sc->topology) {
598                 goto cleanup;
599         }
600
601         AST_LIST_TRAVERSE(participants, participant, entry) {
602                 if (participant == joiner) {
603                         continue;
604                 }
605                 ast_channel_lock(participant->chan);
606                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
607                                 ast_channel_get_stream_topology(participant->chan));
608                 ast_channel_unlock(participant->chan);
609                 if (res) {
610                         goto cleanup;
611                 }
612         }
613
614         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
615
616         AST_LIST_TRAVERSE(participants, participant, entry) {
617                 if (participant == joiner) {
618                         continue;
619                 }
620
621                 sc = participant->tech_pvt;
622                 if (append_all_streams(sc->topology, joiner_video)) {
623                         goto cleanup;
624                 }
625                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
626         }
627
628 cleanup:
629         ast_stream_topology_free(joiner_video);
630 }
631
632 /*! \brief Function called when a channel is joined into the bridge */
633 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
634 {
635         struct softmix_channel *sc;
636         struct softmix_bridge_data *softmix_data;
637         int set_binaural = 0;
638         /*
639          * If false, the channel will be convolved, but since it is a non stereo channel, output
640          * will be mono.
641          */
642         int skip_binaural_output = 1;
643         int pos_id;
644         int is_announcement = 0;
645         int samplerate_change;
646
647         softmix_data = bridge->tech_pvt;
648         if (!softmix_data) {
649                 return -1;
650         }
651
652         /* Create a new softmix_channel structure and allocate various things on it */
653         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
654                 return -1;
655         }
656
657         samplerate_change = softmix_data->internal_rate;
658         pos_id = -1;
659         if (bridge->softmix.binaural_active) {
660                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
661                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
662                         if (set_binaural) {
663                                 softmix_data->internal_rate = samplerate_change;
664                         }
665                         skip_binaural_output = 0;
666                 } else {
667                         is_announcement = 1;
668                 }
669                 if (set_binaural) {
670                         softmix_data->convolve.binaural_active = 1;
671                 }
672                 if (!skip_binaural_output)      {
673                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
674                         if (pos_id == -1) {
675                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
676                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
677                                                 ast_channel_name(bridge_channel->chan));
678                                 ast_free(sc);
679                                 return -1;
680                         }
681                 }
682         }
683
684         /* Can't forget the lock */
685         ast_mutex_init(&sc->lock);
686
687         /* Can't forget to record our pvt structure within the bridged channel structure */
688         bridge_channel->tech_pvt = sc;
689
690         set_softmix_bridge_data(softmix_data->internal_rate,
691                 softmix_data->internal_mixing_interval
692                         ? softmix_data->internal_mixing_interval
693                         : DEFAULT_SOFTMIX_INTERVAL,
694                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
695
696         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
697                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
698         }
699
700         softmix_poke_thread(softmix_data);
701         return 0;
702 }
703
704 static int remove_destination_streams(struct ast_stream_topology *topology,
705         const char *channel_name)
706 {
707         int i;
708         int stream_removed = 0;
709
710         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
711                 struct ast_stream *stream;
712
713                 stream = ast_stream_topology_get_stream(topology, i);
714
715                 if (is_video_dest(stream, channel_name, NULL)) {
716                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
717                         stream_removed = 1;
718                 }
719         }
720         return stream_removed;
721 }
722
723 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
724 {
725         struct ast_bridge_channel *participant;
726         struct softmix_channel *sc;
727
728         AST_LIST_TRAVERSE(participants, participant, entry) {
729                 sc = participant->tech_pvt;
730                 if (!remove_destination_streams(sc->topology, ast_channel_name(leaver->chan))) {
731                         continue;
732                 }
733                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
734         }
735
736         sc = leaver->tech_pvt;
737         if (remove_destination_streams(sc->topology, "")) {
738                 ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
739         }
740
741         return 0;
742 }
743
744 /*! \brief Function called when a channel leaves the bridge */
745 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
746 {
747         struct softmix_channel *sc;
748         struct softmix_bridge_data *softmix_data;
749
750         softmix_data = bridge->tech_pvt;
751         sc = bridge_channel->tech_pvt;
752         if (!sc) {
753                 return;
754         }
755
756         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
757                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
758         }
759
760         if (bridge->softmix.binaural_active) {
761                 if (sc->binaural) {
762                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
763                                         softmix_data->default_sample_size);
764                 }
765         }
766
767         bridge_channel->tech_pvt = NULL;
768
769         ast_stream_topology_free(sc->topology);
770
771         /* Drop mutex lock */
772         ast_mutex_destroy(&sc->lock);
773
774         /* Drop the factory */
775         ast_slinfactory_destroy(&sc->factory);
776
777         /* Drop any formats on the frames */
778         ao2_cleanup(sc->write_frame.subclass.format);
779
780         /* Drop the DSP */
781         ast_dsp_free(sc->dsp);
782
783         /* Eep! drop ourselves */
784         ast_free(sc);
785 }
786
787 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
788 {
789         struct ast_bridge_channel *cur;
790
791         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
792                 if (cur->suspended) {
793                         continue;
794                 }
795                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
796                         ast_bridge_channel_queue_frame(cur, frame);
797                         break;
798                 }
799         }
800 }
801
802 /*!
803  * \internal
804  * \brief Determine what to do with a video frame.
805  * \since 12.0.0
806  *
807  * \param bridge Which bridge is getting the frame
808  * \param bridge_channel Which channel is writing the frame.
809  * \param frame What is being written.
810  *
811  * \return Nothing
812  */
813 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
814 {
815         struct softmix_channel *sc;
816         int video_src_priority;
817
818         /* Determine if the video frame should be distributed or not */
819         switch (bridge->softmix.video_mode.mode) {
820         case AST_BRIDGE_VIDEO_MODE_NONE:
821                 break;
822         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
823                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
824                 if (video_src_priority == 1) {
825                         /* Pass to me and everyone else. */
826                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
827                 }
828                 break;
829         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
830                 sc = bridge_channel->tech_pvt;
831                 ast_mutex_lock(&sc->lock);
832                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
833                         sc->video_talker.energy_average,
834                         frame->subclass.frame_ending);
835                 ast_mutex_unlock(&sc->lock);
836                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
837                 if (video_src_priority == 1) {
838                         int num_src = ast_bridge_number_video_src(bridge);
839                         int echo = num_src > 1 ? 0 : 1;
840
841                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
842                 } else if (video_src_priority == 2) {
843                         softmix_pass_video_top_priority(bridge, frame);
844                 }
845                 break;
846         case AST_BRIDGE_VIDEO_MODE_SFU:
847                 /* Nothing special to do here, the bridge channel stream map will ensure the
848                  * video goes everywhere it needs to
849                  */
850                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
851                 break;
852         }
853 }
854
855 /*!
856  * \internal
857  * \brief Determine what to do with a voice frame.
858  * \since 12.0.0
859  *
860  * \param bridge Which bridge is getting the frame
861  * \param bridge_channel Which channel is writing the frame.
862  * \param frame What is being written.
863  *
864  * \return Nothing
865  */
866 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
867 {
868         struct softmix_channel *sc = bridge_channel->tech_pvt;
869         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
870         int silent = 0;
871         int totalsilence = 0;
872         int cur_energy = 0;
873         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
874                 bridge_channel->tech_args.silence_threshold :
875                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
876         /*
877          * If update_talking is set to 0 or 1, tell the bridge that the channel
878          * has started or stopped talking.
879          */
880         char update_talking = -1;
881
882         /* Write the frame into the conference */
883         ast_mutex_lock(&sc->lock);
884
885         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
886                 /*
887                  * The incoming frame is not the expected format.  Update
888                  * the channel's translation path to get us slinear from
889                  * the new format for the next frame.
890                  *
891                  * There is the possibility that this frame is an old slinear
892                  * rate frame that was in flight when the softmix bridge
893                  * changed rates.  If so it will self correct on subsequent
894                  * frames.
895                  */
896                 ast_channel_lock(bridge_channel->chan);
897                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
898                         ast_channel_name(bridge_channel->chan),
899                         ast_format_get_name(frame->subclass.format),
900                         ast_format_get_name(sc->read_slin_format));
901                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
902                         sc->read_slin_format);
903                 ast_channel_unlock(bridge_channel->chan);
904         }
905
906         /* The channel will be leaving soon if there is no dsp. */
907         if (sc->dsp) {
908                 silent = ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
909         }
910
911         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
912                 int cur_slot = sc->video_talker.energy_history_cur_slot;
913
914                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
915                 sc->video_talker.energy_accum += cur_energy;
916                 sc->video_talker.energy_history[cur_slot] = cur_energy;
917                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
918                 sc->video_talker.energy_history_cur_slot++;
919                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
920                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
921                 }
922         }
923
924         if (totalsilence < silence_threshold) {
925                 if (!sc->talking && !silent) {
926                         /* Tell the write process we have audio to be mixed out */
927                         sc->talking = 1;
928                         update_talking = 1;
929                 }
930         } else {
931                 if (sc->talking) {
932                         sc->talking = 0;
933                         update_talking = 0;
934                 }
935         }
936
937         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
938          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
939          * the audio by flushing the buffer before adding new audio in. */
940         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
941                 ast_slinfactory_flush(&sc->factory);
942         }
943
944         if (sc->talking || !bridge_channel->tech_args.drop_silence) {
945                 /* Add frame to the smoother for mixing with other channels. */
946                 ast_slinfactory_feed(&sc->factory, frame);
947         }
948
949         /* Alllll done */
950         ast_mutex_unlock(&sc->lock);
951
952         if (update_talking != -1) {
953                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
954         }
955 }
956
957 /*!
958  * \internal
959  * \brief Check for voice status updates.
960  * \since 13.20.0
961  *
962  * \param bridge Which bridge we are in
963  * \param bridge_channel Which channel we are checking
964  *
965  * \return Nothing
966  */
967 static void softmix_bridge_check_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
968 {
969         struct softmix_channel *sc = bridge_channel->tech_pvt;
970
971         if (sc->talking
972                 && bridge_channel->features->mute) {
973                 /*
974                  * We were muted while we were talking.
975                  *
976                  * Immediately stop contributing to mixing
977                  * and report no longer talking.
978                  */
979                 ast_mutex_lock(&sc->lock);
980                 ast_slinfactory_flush(&sc->factory);
981                 sc->talking = 0;
982                 ast_mutex_unlock(&sc->lock);
983
984                 /* Notify that we are no longer talking. */
985                 ast_bridge_channel_notify_talking(bridge_channel, 0);
986         }
987 }
988
989 static int remove_all_original_streams(struct ast_stream_topology *dest,
990         const struct ast_stream_topology *source,
991         const struct ast_stream_topology *original)
992 {
993         int i;
994
995         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
996                 struct ast_stream *stream;
997                 int original_index;
998
999                 stream = ast_stream_topology_get_stream(source, i);
1000
1001                 /* Mark the existing stream as removed so we get a new one, this will get
1002                  * reused on a subsequent renegotiation.
1003                  */
1004                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
1005                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
1006
1007                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
1008                                 struct ast_stream *removed;
1009
1010                                 /* Since the participant is still going to be in the bridge we
1011                                  * change the name so that routing does not attempt to route video
1012                                  * to this stream.
1013                                  */
1014                                 removed = ast_stream_clone(stream, "removed");
1015                                 if (!removed) {
1016                                         return -1;
1017                                 }
1018
1019                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
1020
1021                                 /* The destination topology can only ever contain the same, or more,
1022                                  * streams than the original so this is safe.
1023                                  */
1024                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
1025                                         ast_stream_free(removed);
1026                                         return -1;
1027                                 }
1028
1029                                 break;
1030                         }
1031                 }
1032         }
1033
1034         return 0;
1035 }
1036
1037 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
1038 {
1039         struct ast_stream_topology *source_video = NULL;
1040         struct ast_bridge_channel *participant;
1041         int res;
1042
1043         source_video = ast_stream_topology_alloc();
1044         if (!source_video) {
1045                 return;
1046         }
1047
1048         ast_channel_lock(source->chan);
1049         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
1050         ast_channel_unlock(source->chan);
1051         if (res) {
1052                 goto cleanup;
1053         }
1054
1055         AST_LIST_TRAVERSE(participants, participant, entry) {
1056                 struct ast_stream_topology *original_topology;
1057                 struct softmix_channel *sc;
1058
1059                 if (participant == source) {
1060                         continue;
1061                 }
1062
1063                 sc = participant->tech_pvt;
1064
1065                 original_topology = ast_stream_topology_clone(sc->topology);
1066                 if (!original_topology) {
1067                         goto cleanup;
1068                 }
1069
1070                 /* We add all the source streams back in, if any removed streams are already present they will
1071                  * get used first followed by appending new ones.
1072                  */
1073                 if (append_all_streams(sc->topology, source_video)) {
1074                         ast_stream_topology_free(original_topology);
1075                         goto cleanup;
1076                 }
1077
1078                 /* And the original existing streams get marked as removed. This causes the remote side to see
1079                  * a new stream for the source streams.
1080                  */
1081                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1082                         ast_stream_topology_free(original_topology);
1083                         goto cleanup;
1084                 }
1085
1086                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1087                 ast_stream_topology_free(original_topology);
1088         }
1089
1090 cleanup:
1091         ast_stream_topology_free(source_video);
1092 }
1093
1094 /*!
1095  * \internal
1096  * \brief Determine what to do with a control frame.
1097  * \since 12.0.0
1098  *
1099  * \param bridge Which bridge is getting the frame
1100  * \param bridge_channel Which channel is writing the frame.
1101  * \param frame What is being written.
1102  *
1103  * \retval 0 Frame accepted into the bridge.
1104  * \retval -1 Frame needs to be deferred.
1105  */
1106 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1107 {
1108         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1109
1110         /*
1111          * XXX Softmix needs to use channel roles to determine what to
1112          * do with control frames.
1113          */
1114
1115         switch (frame->subclass.integer) {
1116         case AST_CONTROL_VIDUPDATE:
1117                 if (!bridge->softmix.video_mode.video_update_discard ||
1118                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1119                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1120                         softmix_data->last_video_update = ast_tvnow();
1121                 }
1122                 break;
1123         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1124                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1125                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1126                 }
1127                 break;
1128         default:
1129                 break;
1130         }
1131
1132         return 0;
1133 }
1134
1135 /*!
1136  * \internal
1137  * \brief Determine what to do with a frame written into the bridge.
1138  * \since 12.0.0
1139  *
1140  * \param bridge Which bridge is getting the frame
1141  * \param bridge_channel Which channel is writing the frame.
1142  * \param frame What is being written.
1143  *
1144  * \retval 0 Frame accepted into the bridge.
1145  * \retval -1 Frame needs to be deferred.
1146  *
1147  * \note On entry, bridge is already locked.
1148  */
1149 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1150 {
1151         int res = 0;
1152
1153         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1154                 /* "Accept" the frame and discard it. */
1155                 return 0;
1156         }
1157
1158         /*
1159          * XXX Softmix needs to use channel roles to determine who gets
1160          * what frame.  Possible roles: announcer, recorder, agent,
1161          * supervisor.
1162          */
1163         switch (frame->frametype) {
1164         case AST_FRAME_NULL:
1165                 /* "Accept" the frame and discard it. */
1166                 softmix_bridge_check_voice(bridge, bridge_channel);
1167                 break;
1168         case AST_FRAME_DTMF_BEGIN:
1169         case AST_FRAME_DTMF_END:
1170                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1171                 break;
1172         case AST_FRAME_VOICE:
1173                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1174                 break;
1175         case AST_FRAME_VIDEO:
1176                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1177                 break;
1178         case AST_FRAME_CONTROL:
1179                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1180                 break;
1181         case AST_FRAME_BRIDGE_ACTION:
1182                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1183                 break;
1184         case AST_FRAME_BRIDGE_ACTION_SYNC:
1185                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1186                 ast_assert(0);
1187         default:
1188                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1189                 /* "Accept" the frame and discard it. */
1190                 break;
1191         }
1192
1193         return res;
1194 }
1195
1196 static void gather_softmix_stats(struct softmix_stats *stats,
1197         const struct softmix_bridge_data *softmix_data,
1198         struct ast_bridge_channel *bridge_channel)
1199 {
1200         int channel_native_rate;
1201
1202         /* Gather stats about channel sample rates. */
1203         ast_channel_lock(bridge_channel->chan);
1204         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1205                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1206         ast_channel_unlock(bridge_channel->chan);
1207
1208         if (stats->highest_supported_rate < channel_native_rate) {
1209                 stats->highest_supported_rate = channel_native_rate;
1210         }
1211         if (softmix_data->internal_rate < channel_native_rate) {
1212                 int i;
1213
1214                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1215                         if (stats->sample_rates[i] == channel_native_rate) {
1216                                 stats->num_channels[i]++;
1217                                 break;
1218                         } else if (!stats->sample_rates[i]) {
1219                                 stats->sample_rates[i] = channel_native_rate;
1220                                 stats->num_channels[i]++;
1221                                 break;
1222                         }
1223                 }
1224                 stats->num_above_internal_rate++;
1225         } else if (softmix_data->internal_rate == channel_native_rate) {
1226                 stats->num_at_internal_rate++;
1227         }
1228 }
1229
1230 /*!
1231  * \internal
1232  * \brief Analyse mixing statistics and change bridges internal rate
1233  * if necessary.
1234  *
1235  * \retval 0, no changes to internal rate
1236  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1237  */
1238 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1239                 struct softmix_bridge_data *softmix_data, int binaural_active)
1240 {
1241         int i;
1242
1243         if (binaural_active) {
1244                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1245         }
1246
1247         /*
1248          * Re-adjust the internal bridge sample rate if
1249          * 1. The bridge's internal sample rate is locked in at a sample
1250          *    rate other than the current sample rate being used.
1251          * 2. two or more channels support a higher sample rate
1252          * 3. no channels support the current sample rate or a higher rate
1253          */
1254         if (stats->locked_rate) {
1255                 /* if the rate is locked by the bridge, only update it if it differs
1256                  * from the current rate we are using. */
1257                 if (softmix_data->internal_rate != stats->locked_rate) {
1258                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1259                                 softmix_data->internal_rate, stats->locked_rate);
1260                         softmix_data->internal_rate = stats->locked_rate;
1261                         return 1;
1262                 }
1263         } else if (stats->num_above_internal_rate >= 2) {
1264                 /* the highest rate is just used as a starting point */
1265                 unsigned int best_rate = stats->highest_supported_rate;
1266                 int best_index = -1;
1267
1268                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1269                         if (stats->num_channels[i]) {
1270                                 break;
1271                         }
1272                         if (2 <= stats->num_channels[i]) {
1273                                 /* Two or more channels support this rate. */
1274                                 if (best_index == -1
1275                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1276                                         /*
1277                                          * best_rate starts out being the first sample rate
1278                                          * greater than the internal sample rate that two or
1279                                          * more channels support.
1280                                          *
1281                                          * or
1282                                          *
1283                                          * There are multiple rates above the internal rate
1284                                          * and this rate is higher than the previous rate two
1285                                          * or more channels support.
1286                                          */
1287                                         best_rate = stats->sample_rates[i];
1288                                         best_index = i;
1289                                 }
1290                         } else if (best_index == -1) {
1291                                 /*
1292                                  * It is possible that multiple channels exist with native sample
1293                                  * rates above the internal sample rate, but none of those channels
1294                                  * have the same rate in common.  In this case, the lowest sample
1295                                  * rate among those channels is picked. Over time as additional
1296                                  * statistic runs are made the internal sample rate number will
1297                                  * adjust to the most optimal sample rate, but it may take multiple
1298                                  * iterations.
1299                                  */
1300                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1301                         }
1302                 }
1303
1304                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1305                         softmix_data->internal_rate, best_rate);
1306                 softmix_data->internal_rate = best_rate;
1307                 return 1;
1308         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1309                 /* In this case, the highest supported rate is actually lower than the internal rate */
1310                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1311                         softmix_data->internal_rate, stats->highest_supported_rate);
1312                 softmix_data->internal_rate = stats->highest_supported_rate;
1313                 return 1;
1314         }
1315         return 0;
1316 }
1317
1318 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1319                 unsigned int starting_num_entries, unsigned int binaural_active)
1320 {
1321         memset(mixing_array, 0, sizeof(*mixing_array));
1322         mixing_array->max_num_entries = starting_num_entries;
1323         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1324                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1325                 return -1;
1326         }
1327         if (binaural_active) {
1328                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1329                                 sizeof(struct convolve_channel_pair *)))) {
1330                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1331                         return -1;
1332                 }
1333         }
1334         return 0;
1335 }
1336
1337 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1338                 unsigned int binaural_active)
1339 {
1340         ast_free(mixing_array->buffers);
1341         if (binaural_active) {
1342                 ast_free(mixing_array->chan_pairs);
1343         }
1344 }
1345
1346 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1347                 unsigned int num_entries, unsigned int binaural_active)
1348 {
1349         int16_t **tmp;
1350
1351         /* give it some room to grow since memory is cheap but allocations can be expensive */
1352         mixing_array->max_num_entries = num_entries;
1353         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1354                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1355                 return -1;
1356         }
1357         mixing_array->buffers = tmp;
1358
1359         if (binaural_active) {
1360                 struct convolve_channel_pair **tmp2;
1361                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1362                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1363                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1364                         return -1;
1365                 }
1366                 mixing_array->chan_pairs = tmp2;
1367         }
1368         return 0;
1369 }
1370
1371 /*!
1372  * \brief Mixing loop.
1373  *
1374  * \retval 0 on success
1375  * \retval -1 on failure
1376  */
1377 static int softmix_mixing_loop(struct ast_bridge *bridge)
1378 {
1379         struct softmix_stats stats = { { 0 }, };
1380         struct softmix_mixing_array mixing_array;
1381         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1382         struct ast_timer *timer;
1383         struct softmix_translate_helper trans_helper;
1384         int16_t buf[MAX_DATALEN];
1385 #ifdef BINAURAL_RENDERING
1386         int16_t bin_buf[MAX_DATALEN];
1387         int16_t ann_buf[MAX_DATALEN];
1388 #endif
1389         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1390         int timingfd;
1391         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1392         unsigned int idx;
1393         unsigned int x;
1394         int res = -1;
1395
1396         timer = softmix_data->timer;
1397         timingfd = ast_timer_fd(timer);
1398         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1399         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1400
1401         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1402         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1403                         bridge->softmix.binaural_active)) {
1404                 goto softmix_cleanup;
1405         }
1406
1407         /*
1408          * XXX Softmix needs to use channel roles to determine who gets
1409          * what audio mixed.
1410          */
1411         while (!softmix_data->stop && bridge->num_active) {
1412                 struct ast_bridge_channel *bridge_channel;
1413                 int timeout = -1;
1414                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1415                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1416                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1417
1418                 if (softmix_datalen > MAX_DATALEN) {
1419                         /* This should NEVER happen, but if it does we need to know about it. Almost
1420                          * all the memcpys used during this process depend on this assumption.  Rather
1421                          * than checking this over and over again through out the code, this single
1422                          * verification is done on each iteration. */
1423                         ast_log(LOG_WARNING,
1424                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1425                                 bridge->uniqueid);
1426                         goto softmix_cleanup;
1427                 }
1428
1429                 /* Grow the mixing array buffer as participants are added. */
1430                 if (mixing_array.max_num_entries < bridge->num_channels
1431                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1432                                         bridge->softmix.binaural_active)) {
1433                         goto softmix_cleanup;
1434                 }
1435
1436                 /* init the number of buffers stored in the mixing array to 0.
1437                  * As buffers are added for mixing, this number is incremented. */
1438                 mixing_array.used_entries = 0;
1439
1440                 /* These variables help determine if a rate change is required */
1441                 if (!stat_iteration_counter) {
1442                         memset(&stats, 0, sizeof(stats));
1443                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1444                 }
1445
1446                 /* If the sample rate has changed, update the translator helper */
1447                 if (update_all_rates) {
1448                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1449                 }
1450
1451 #ifdef BINAURAL_RENDERING
1452                 check_binaural_position_change(bridge, softmix_data);
1453 #endif
1454
1455                 /* Go through pulling audio from each factory that has it available */
1456                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1457                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1458
1459                         if (!sc) {
1460                                 /* This channel failed to join successfully. */
1461                                 continue;
1462                         }
1463
1464                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1465                         if (update_all_rates) {
1466                                 set_softmix_bridge_data(softmix_data->internal_rate,
1467                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1468                         }
1469
1470                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1471                         if (!stat_iteration_counter) {
1472                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1473                         }
1474
1475                         /* if the channel is suspended, don't check for audio, but still gather stats */
1476                         if (bridge_channel->suspended) {
1477                                 continue;
1478                         }
1479
1480                         /* Try to get audio from the factory if available */
1481                         ast_mutex_lock(&sc->lock);
1482                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1483 #ifdef BINAURAL_RENDERING
1484                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1485                                                 ast_channel_name(bridge_channel->chan));
1486 #endif
1487                                 mixing_array.used_entries++;
1488                         }
1489                         ast_mutex_unlock(&sc->lock);
1490                 }
1491
1492                 /* mix it like crazy (non binaural channels)*/
1493                 memset(buf, 0, softmix_datalen);
1494                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1495                         for (x = 0; x < softmix_samples; ++x) {
1496                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1497                         }
1498                 }
1499
1500 #ifdef BINAURAL_RENDERING
1501                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1502 #endif
1503
1504                 /* Next step go through removing the channel's own audio and creating a good frame... */
1505                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1506                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1507
1508                         if (!sc || bridge_channel->suspended) {
1509                                 /* This channel failed to join successfully or is suspended. */
1510                                 continue;
1511                         }
1512
1513                         ast_mutex_lock(&sc->lock);
1514
1515                         /* Make SLINEAR write frame from local buffer */
1516                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1517                                 "Replace softmix channel slin format");
1518 #ifdef BINAURAL_RENDERING
1519                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1520                                         && sc->binaural) {
1521                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1522                                                 softmix_samples, buf);
1523                         } else
1524 #endif
1525                         {
1526                                 sc->write_frame.datalen = softmix_datalen;
1527                                 sc->write_frame.samples = softmix_samples;
1528                                 memcpy(sc->final_buf, buf, softmix_datalen);
1529                         }
1530                         /* process the softmix channel's new write audio */
1531                         softmix_process_write_audio(&trans_helper,
1532                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1533                                         softmix_data->default_sample_size);
1534
1535                         ast_mutex_unlock(&sc->lock);
1536
1537                         /* A frame is now ready for the channel. */
1538                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1539                 }
1540
1541                 update_all_rates = 0;
1542                 if (!stat_iteration_counter) {
1543                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1544                                         bridge->softmix.binaural_active);
1545                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1546                 }
1547                 stat_iteration_counter--;
1548
1549                 ast_bridge_unlock(bridge);
1550                 /* cleanup any translation frame data from the previous mixing iteration. */
1551                 softmix_translate_helper_cleanup(&trans_helper);
1552                 /* Wait for the timing source to tell us to wake up and get things done */
1553                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1554                 if (ast_timer_ack(timer, 1) < 0) {
1555                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1556                                 bridge->uniqueid);
1557                         ast_bridge_lock(bridge);
1558                         goto softmix_cleanup;
1559                 }
1560                 ast_bridge_lock(bridge);
1561
1562                 /* make sure to detect mixing interval changes if they occur. */
1563                 if (bridge->softmix.internal_mixing_interval
1564                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1565                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1566                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1567                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1568                 }
1569         }
1570
1571         res = 0;
1572
1573 softmix_cleanup:
1574         softmix_translate_helper_destroy(&trans_helper);
1575         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1576         return res;
1577 }
1578
1579 /*!
1580  * \internal
1581  * \brief Mixing thread.
1582  * \since 12.0.0
1583  *
1584  * \note The thread does not have its own reference to the
1585  * bridge.  The lifetime of the thread is tied to the lifetime
1586  * of the mixing technology association with the bridge.
1587  */
1588 static void *softmix_mixing_thread(void *data)
1589 {
1590         struct softmix_bridge_data *softmix_data = data;
1591         struct ast_bridge *bridge = softmix_data->bridge;
1592
1593         ast_bridge_lock(bridge);
1594         if (bridge->callid) {
1595                 ast_callid_threadassoc_add(bridge->callid);
1596         }
1597
1598         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1599
1600         while (!softmix_data->stop) {
1601                 if (!bridge->num_active) {
1602                         /* Wait for something to happen to the bridge. */
1603                         ast_bridge_unlock(bridge);
1604                         ast_mutex_lock(&softmix_data->lock);
1605                         if (!softmix_data->stop) {
1606                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1607                         }
1608                         ast_mutex_unlock(&softmix_data->lock);
1609                         ast_bridge_lock(bridge);
1610                         continue;
1611                 }
1612
1613                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1614 #ifndef BINAURAL_RENDERING
1615                         ast_bridge_lock(bridge);
1616                         bridge->softmix.binaural_active = 0;
1617                         ast_bridge_unlock(bridge);
1618                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1619                                         "compiled.\n", bridge->uniqueid);
1620 #else
1621                         /* Set and init binaural data if binaural is activated in the configuration. */
1622                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1623                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1624                                         softmix_data->internal_mixing_interval);
1625                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1626                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1627                                         == -1) {
1628                                 ast_bridge_lock(bridge);
1629                                 bridge->softmix.binaural_active = 0;
1630                                 ast_bridge_unlock(bridge);
1631                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1632                                                 "binaural processing,  Will only process mono audio.\n",
1633                                                 bridge->uniqueid);
1634                         }
1635                         softmix_data->binaural_init = 1;
1636 #endif
1637                 }
1638
1639                 if (softmix_mixing_loop(bridge)) {
1640                         /*
1641                          * A mixing error occurred.  Sleep and try again later so we
1642                          * won't flood the logs.
1643                          */
1644                         ast_bridge_unlock(bridge);
1645                         sleep(1);
1646                         ast_bridge_lock(bridge);
1647                 }
1648         }
1649
1650         ast_bridge_unlock(bridge);
1651
1652         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1653
1654         return NULL;
1655 }
1656
1657 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1658 {
1659         if (softmix_data->timer) {
1660                 ast_timer_close(softmix_data->timer);
1661                 softmix_data->timer = NULL;
1662         }
1663         ast_mutex_destroy(&softmix_data->lock);
1664         ast_cond_destroy(&softmix_data->cond);
1665         ast_free(softmix_data);
1666 }
1667
1668 /*! \brief Function called when a bridge is created */
1669 static int softmix_bridge_create(struct ast_bridge *bridge)
1670 {
1671         struct softmix_bridge_data *softmix_data;
1672
1673         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1674         if (!softmix_data) {
1675                 return -1;
1676         }
1677         softmix_data->bridge = bridge;
1678         ast_mutex_init(&softmix_data->lock);
1679         ast_cond_init(&softmix_data->cond, NULL);
1680         softmix_data->timer = ast_timer_open();
1681         if (!softmix_data->timer) {
1682                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1683                 softmix_bridge_data_destroy(softmix_data);
1684                 return -1;
1685         }
1686         /* start at minimum rate, let it grow from there */
1687         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1688         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1689
1690 #ifdef BINAURAL_RENDERING
1691         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1692                         softmix_data->internal_mixing_interval);
1693 #endif
1694
1695         bridge->tech_pvt = softmix_data;
1696
1697         /* Start the mixing thread. */
1698         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1699                 softmix_data)) {
1700                 softmix_data->thread = AST_PTHREADT_NULL;
1701                 softmix_bridge_data_destroy(softmix_data);
1702                 bridge->tech_pvt = NULL;
1703                 return -1;
1704         }
1705
1706         return 0;
1707 }
1708
1709 /*!
1710  * \internal
1711  * \brief Request the softmix mixing thread stop.
1712  * \since 12.0.0
1713  *
1714  * \param bridge Which bridge is being stopped.
1715  *
1716  * \return Nothing
1717  */
1718 static void softmix_bridge_stop(struct ast_bridge *bridge)
1719 {
1720         struct softmix_bridge_data *softmix_data;
1721
1722         softmix_data = bridge->tech_pvt;
1723         if (!softmix_data) {
1724                 return;
1725         }
1726
1727         ast_mutex_lock(&softmix_data->lock);
1728         softmix_data->stop = 1;
1729         ast_mutex_unlock(&softmix_data->lock);
1730 }
1731
1732 /*! \brief Function called when a bridge is destroyed */
1733 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1734 {
1735         struct softmix_bridge_data *softmix_data;
1736         pthread_t thread;
1737
1738         softmix_data = bridge->tech_pvt;
1739         if (!softmix_data) {
1740                 return;
1741         }
1742
1743         /* Stop the mixing thread. */
1744         ast_mutex_lock(&softmix_data->lock);
1745         softmix_data->stop = 1;
1746         ast_cond_signal(&softmix_data->cond);
1747         thread = softmix_data->thread;
1748         softmix_data->thread = AST_PTHREADT_NULL;
1749         ast_mutex_unlock(&softmix_data->lock);
1750         if (thread != AST_PTHREADT_NULL) {
1751                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1752                 pthread_join(thread, NULL);
1753         }
1754 #ifdef BINAURAL_RENDERING
1755         free_convolve_data(&softmix_data->convolve);
1756 #endif
1757         softmix_bridge_data_destroy(softmix_data);
1758         bridge->tech_pvt = NULL;
1759 }
1760
1761 /*!
1762  * \brief Map a source stream to all of its destination streams.
1763  *
1764  * \param source_stream_name Name of the source stream
1765  * \param source_channel_name Name of channel where the source stream originates
1766  * \param bridge_stream_position The slot in the bridge where source video will come from
1767  * \param participants The bridge_channels in the bridge
1768  */
1769 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1770         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1771 {
1772         struct ast_bridge_channel *participant;
1773
1774         AST_LIST_TRAVERSE(participants, participant, entry) {
1775                 int i;
1776                 struct ast_stream_topology *topology;
1777
1778                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1779                         continue;
1780                 }
1781
1782                 ast_bridge_channel_lock(participant);
1783                 ast_channel_lock(participant->chan);
1784                 topology = ast_channel_get_stream_topology(participant->chan);
1785
1786                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1787                         struct ast_stream *stream;
1788
1789                         stream = ast_stream_topology_get_stream(topology, i);
1790                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1791                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1792                                 break;
1793                         }
1794                 }
1795                 ast_channel_unlock(participant->chan);
1796                 ast_bridge_channel_unlock(participant);
1797         }
1798 }
1799
1800 /*!
1801  * \brief stream_topology_changed callback
1802  *
1803  * For most video modes, nothing beyond the ordinary is required.
1804  * For the SFU case, though, we need to completely remap the streams
1805  * in order to ensure video gets directed where it is expected to go.
1806  *
1807  * \param bridge The bridge
1808  * \param bridge_channel Channel whose topology has changed
1809  */
1810 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1811 {
1812         struct ast_bridge_channel *participant;
1813         struct ast_vector_int media_types;
1814         int nths[AST_MEDIA_TYPE_END] = {0};
1815
1816         switch (bridge->softmix.video_mode.mode) {
1817         case AST_BRIDGE_VIDEO_MODE_NONE:
1818         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1819         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1820         default:
1821                 ast_bridge_channel_stream_map(bridge_channel);
1822                 return;
1823         case AST_BRIDGE_VIDEO_MODE_SFU:
1824                 break;
1825         }
1826
1827         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1828
1829         /* First traversal: re-initialize all of the participants' stream maps */
1830         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1831                 ast_bridge_channel_lock(participant);
1832                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1833                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1834                 ast_bridge_channel_unlock(participant);
1835         }
1836
1837         /* Second traversal: Map specific video channels from their source to their destinations.
1838          *
1839          * This is similar to what is done in ast_stream_topology_map(),
1840          * except that video channels are handled differently.  Each video
1841          * source has it's own unique index on the bridge.  This way, a
1842          * particular channel's source video can be distributed to the
1843          * appropriate destination streams on the other channels.
1844          */
1845         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1846                 int i;
1847                 struct ast_stream_topology *topology;
1848
1849                 ast_bridge_channel_lock(participant);
1850                 ast_channel_lock(participant->chan);
1851
1852                 topology = ast_channel_get_stream_topology(participant->chan);
1853                 if (topology) {
1854                         /*
1855                          * Sigh.  We have to clone to avoid deadlock in
1856                          * map_source_to_destinations() because topology
1857                          * is not an ao2 object.
1858                          */
1859                         topology = ast_stream_topology_clone(topology);
1860                 }
1861                 if (!topology) {
1862                         /* Oh, my, we are in trouble. */
1863                         ast_channel_unlock(participant->chan);
1864                         ast_bridge_channel_unlock(participant);
1865                         continue;
1866                 }
1867
1868                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1869                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1870
1871                         if (is_video_source(stream)) {
1872                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1873                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1874                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1875                                 /*
1876                                  * Unlock the channel and participant to prevent
1877                                  * potential deadlock in map_source_to_destinations().
1878                                  */
1879                                 ast_channel_unlock(participant->chan);
1880                                 ast_bridge_channel_unlock(participant);
1881                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1882                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1883                                 ast_bridge_channel_lock(participant);
1884                                 ast_channel_lock(participant->chan);
1885                         } else if (is_video_dest(stream, NULL, NULL)) {
1886                                 /* We expect to never read media from video destination channels, but just
1887                                  * in case, we should set their to_bridge value to -1.
1888                                  */
1889                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1890                         } else {
1891                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1892                                  * be factored out in some way
1893                                  */
1894                                 enum ast_media_type type = ast_stream_get_type(stream);
1895                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1896                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1897
1898                                 if (index == -1) {
1899                                         AST_VECTOR_APPEND(&media_types, type);
1900                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1901                                 }
1902
1903                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1904                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1905                         }
1906                 }
1907
1908                 ast_stream_topology_free(topology);
1909
1910                 ast_channel_unlock(participant->chan);
1911                 ast_bridge_channel_unlock(participant);
1912         }
1913 }
1914
1915 static struct ast_bridge_technology softmix_bridge = {
1916         .name = "softmix",
1917         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1918         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1919         .create = softmix_bridge_create,
1920         .stop = softmix_bridge_stop,
1921         .destroy = softmix_bridge_destroy,
1922         .join = softmix_bridge_join,
1923         .leave = softmix_bridge_leave,
1924         .unsuspend = softmix_bridge_unsuspend,
1925         .write = softmix_bridge_write,
1926         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1927 };
1928
1929 #ifdef TEST_FRAMEWORK
1930 struct stream_parameters {
1931         const char *name;
1932         const char *formats;
1933         enum ast_media_type type;
1934 };
1935
1936 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1937 {
1938         struct ast_stream_topology *topology;
1939         size_t i;
1940
1941         topology = ast_stream_topology_alloc();
1942         if (!topology) {
1943                 return NULL;
1944         }
1945
1946         for (i = 0; i < num_streams; ++i) {
1947                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1948                 struct ast_stream *stream;
1949
1950                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1951                 if (!caps) {
1952                         goto fail;
1953                 }
1954                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1955                         goto fail;
1956                 }
1957                 stream = ast_stream_alloc(params[i].name, params[i].type);
1958                 if (!stream) {
1959                         goto fail;
1960                 }
1961                 ast_stream_set_formats(stream, caps);
1962                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1963                         ast_stream_free(stream);
1964                         goto fail;
1965                 }
1966         }
1967
1968         return topology;
1969
1970 fail:
1971         ast_stream_topology_free(topology);
1972         return NULL;
1973 }
1974
1975 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1976         const struct stream_parameters *params)
1977 {
1978         struct ast_format_cap *stream_caps;
1979         struct ast_format_cap *params_caps;
1980
1981         if (ast_stream_get_type(stream) != params->type) {
1982                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1983                         ast_codec_media_type2str(params->type),
1984                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1985                 return -1;
1986         }
1987         if (strcmp(ast_stream_get_name(stream), params->name)) {
1988                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1989                         params->name, ast_stream_get_name(stream));
1990                 return -1;
1991         }
1992
1993         stream_caps = ast_stream_get_formats(stream);
1994         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1995         if (!params_caps) {
1996                 ast_test_status_update(test, "Allocation error on capabilities\n");
1997                 return -1;
1998         }
1999         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
2000
2001         if (!ast_format_cap_identical(stream_caps, params_caps)) {
2002                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
2003                         ast_stream_get_name(stream));
2004                 ao2_cleanup(params_caps);
2005                 return -1;
2006         }
2007
2008         ao2_cleanup(params_caps);
2009         return 0;
2010 }
2011
2012 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
2013         const struct stream_parameters *params, size_t num_streams)
2014 {
2015         int i;
2016
2017         if (ast_stream_topology_get_count(topology) < num_streams) {
2018                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
2019                         ast_stream_topology_get_count(topology), num_streams);
2020                 return -1;
2021         }
2022
2023         for (i = 0; i < num_streams; ++i) {
2024                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
2025                         return -1;
2026                 }
2027         }
2028
2029         return 0;
2030 }
2031
2032 AST_TEST_DEFINE(sfu_append_source_streams)
2033 {
2034         enum ast_test_result_state res = AST_TEST_FAIL;
2035         static const struct stream_parameters bob_streams[] = {
2036                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2037                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2038         };
2039         static const struct stream_parameters alice_streams[] = {
2040                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
2041                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2042         };
2043         static const struct stream_parameters alice_dest_stream = {
2044                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2045         };
2046         static const struct stream_parameters bob_dest_stream = {
2047                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "vp8", AST_MEDIA_TYPE_VIDEO,
2048         };
2049         struct ast_stream_topology *topology_alice = NULL;
2050         struct ast_stream_topology *topology_bob = NULL;
2051
2052         switch (cmd) {
2053         case TEST_INIT:
2054                 info->name = "sfu_append_source_streams";
2055                 info->category = "/bridges/bridge_softmix/";
2056                 info->summary = "Test appending of video streams";
2057                 info->description =
2058                         "This tests does stuff.";
2059                 return AST_TEST_NOT_RUN;
2060         case TEST_EXECUTE:
2061                 break;
2062         }
2063
2064         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2065         if (!topology_alice) {
2066                 goto end;
2067         }
2068
2069         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2070         if (!topology_bob) {
2071                 goto end;
2072         }
2073
2074         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2075                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2076                 goto end;
2077         }
2078
2079         if (ast_stream_topology_get_count(topology_alice) != 3) {
2080                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2081                         ast_stream_topology_get_count(topology_alice), 3);
2082                 goto end;
2083         }
2084
2085         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2086                 goto end;
2087         }
2088
2089         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2090                 goto end;
2091         }
2092
2093         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2094                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2095                 goto end;
2096         }
2097
2098         if (ast_stream_topology_get_count(topology_bob) != 3) {
2099                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2100                         ast_stream_topology_get_count(topology_bob), 3);
2101                 goto end;
2102         }
2103
2104         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2105                 goto end;
2106         }
2107
2108         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2109                 goto end;
2110         }
2111
2112         res = AST_TEST_PASS;
2113
2114 end:
2115         ast_stream_topology_free(topology_alice);
2116         ast_stream_topology_free(topology_bob);
2117         return res;
2118 }
2119
2120 AST_TEST_DEFINE(sfu_remove_destination_streams)
2121 {
2122         enum ast_test_result_state res = AST_TEST_FAIL;
2123         static const struct stream_parameters params[] = {
2124                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2125                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2126                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2127                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2128         };
2129         static const struct {
2130                 const char *channel_name;
2131                 int num_streams;
2132                 int params_index[4];
2133         } removal_results[] = {
2134                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2135                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2136                 { "", 4, { 0, 1, 2, 3 }, },
2137         };
2138         struct ast_stream_topology *orig = NULL;
2139         int i;
2140
2141         switch (cmd) {
2142         case TEST_INIT:
2143                 info->name = "sfu_remove_destination_streams";
2144                 info->category = "/bridges/bridge_softmix/";
2145                 info->summary = "Test removal of destination video streams";
2146                 info->description =
2147                         "This tests does stuff.";
2148                 return AST_TEST_NOT_RUN;
2149         case TEST_EXECUTE:
2150                 break;
2151         }
2152
2153         orig = build_topology(params, ARRAY_LEN(params));
2154         if (!orig) {
2155                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2156                 goto end;
2157         }
2158
2159         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2160                 int j;
2161
2162                 remove_destination_streams(orig, removal_results[i].channel_name);
2163
2164                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2165                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2166                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2167                         goto end;
2168                 }
2169
2170                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2171                         struct ast_stream *actual;
2172                         struct ast_stream *expected;
2173                         int orig_index;
2174
2175                         actual = ast_stream_topology_get_stream(orig, j);
2176
2177                         orig_index = removal_results[i].params_index[j];
2178                         expected = ast_stream_topology_get_stream(orig, orig_index);
2179
2180                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2181                                 ast_stream_get_formats(expected))) {
2182                                 struct ast_str *expected_str;
2183                                 struct ast_str *actual_str;
2184
2185                                 expected_str = ast_str_alloca(64);
2186                                 actual_str = ast_str_alloca(64);
2187
2188                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2189                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2190                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2191                                 goto end;
2192                         }
2193
2194                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2195                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2196                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2197                                 goto end;
2198                         }
2199                 }
2200         }
2201
2202         res = AST_TEST_PASS;
2203
2204 end:
2205         ast_stream_topology_free(orig);
2206         return res;
2207 }
2208
2209 #endif
2210
2211 static int unload_module(void)
2212 {
2213         ast_bridge_technology_unregister(&softmix_bridge);
2214         AST_TEST_UNREGISTER(sfu_append_source_streams);
2215         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2216         return 0;
2217 }
2218
2219 static int load_module(void)
2220 {
2221         if (ast_bridge_technology_register(&softmix_bridge)) {
2222                 unload_module();
2223                 return AST_MODULE_LOAD_DECLINE;
2224         }
2225         AST_TEST_REGISTER(sfu_append_source_streams);
2226         AST_TEST_REGISTER(sfu_remove_destination_streams);
2227         return AST_MODULE_LOAD_SUCCESS;
2228 }
2229
2230 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");