logger.conf.sample: add missing comment mark
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include <math.h>
37
38 #include "asterisk/stream.h"
39 #include "asterisk/test.h"
40 #include "asterisk/vector.h"
41 #include "asterisk/message.h"
42 #include "bridge_softmix/include/bridge_softmix_internal.h"
43
44 /*! The minimum sample rate of the bridge. */
45 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
46
47 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
48 #define DEFAULT_SOFTMIX_INTERVAL 20
49
50 /*! \brief Size of the buffer used for sample manipulation */
51 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
52
53 /*! \brief Number of samples we are dealing with */
54 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
55
56 /*! \brief Number of mixing iterations to perform between gathering statistics. */
57 #define SOFTMIX_STAT_INTERVAL 100
58
59 /*!
60  * \brief Default time in ms of silence necessary to declare talking stopped by the bridge.
61  *
62  * \details
63  * This is the time at which a channel's own audio will stop getting
64  * mixed out of its own write audio stream because it is no longer talking.
65  */
66 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
67
68 /*! Default minimum average magnitude threshold to determine talking by the DSP. */
69 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
70
71 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
72 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
73 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
74
75 struct softmix_remb_collector {
76         /*! The frame which will be given to each source stream */
77         struct ast_frame frame;
78         /*! The REMB to send to the source which is collecting REMB reports */
79         struct ast_rtp_rtcp_feedback feedback;
80         /*! The maximum bitrate (A single precision floating point is big enough) */
81         float bitrate;
82 };
83
84 struct softmix_stats {
85         /*! Each index represents a sample rate used above the internal rate. */
86         unsigned int sample_rates[16];
87         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
88         unsigned int num_channels[16];
89         /*! The number of channels above the internal sample rate */
90         unsigned int num_above_internal_rate;
91         /*! The number of channels above the maximum sample rate */
92         unsigned int num_above_maximum_rate;
93         /*! The number of channels at the internal sample rate */
94         unsigned int num_at_internal_rate;
95         /*! The absolute highest sample rate preferred by any channel in the bridge */
96         unsigned int highest_supported_rate;
97         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
98         unsigned int locked_rate;
99         /*! The maximum sample rate the bridge may use */
100         unsigned int maximum_rate;
101 };
102
103 struct softmix_translate_helper_entry {
104         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
105                                                                   and re-init if it was usable. */
106         struct ast_format *dst_format; /*!< The destination format for this helper */
107         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
108         struct ast_frame *out_frame; /*!< The output frame from the last translation */
109         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
110 };
111
112 struct softmix_translate_helper {
113         struct ast_format *slin_src; /*!< the source format expected for all the translators */
114         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
115 };
116
117 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
118 {
119         struct softmix_translate_helper_entry *entry;
120         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
121                 return NULL;
122         }
123         entry->dst_format = ao2_bump(dst);
124         /* initialize this to one so that the first time through the cleanup code after
125            allocation it won't be removed from the entry list */
126         entry->num_times_requested = 1;
127         return entry;
128 }
129
130 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
131 {
132         ao2_cleanup(entry->dst_format);
133
134         if (entry->trans_pvt) {
135                 ast_translator_free_path(entry->trans_pvt);
136         }
137         if (entry->out_frame) {
138                 ast_frfree(entry->out_frame);
139         }
140         ast_free(entry);
141         return NULL;
142 }
143
144 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
145 {
146         memset(trans_helper, 0, sizeof(*trans_helper));
147         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
148 }
149
150 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
151 {
152         struct softmix_translate_helper_entry *entry;
153
154         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
155                 softmix_translate_helper_free_entry(entry);
156         }
157 }
158
159 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
160 {
161         struct softmix_translate_helper_entry *entry;
162
163         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
164         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
165                 if (entry->trans_pvt) {
166                         ast_translator_free_path(entry->trans_pvt);
167                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
168                                 AST_LIST_REMOVE_CURRENT(entry);
169                                 entry = softmix_translate_helper_free_entry(entry);
170                         }
171                 }
172         }
173         AST_LIST_TRAVERSE_SAFE_END;
174 }
175
176 /*!
177  * \internal
178  * \brief Get the next available audio on the softmix channel's read stream
179  * and determine if it should be mixed out or not on the write stream.
180  *
181  * \retval pointer to buffer containing the exact number of samples requested on success.
182  * \retval NULL if no samples are present
183  */
184 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
185 {
186         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
187                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
188                 sc->have_audio = 1;
189                 return sc->our_buf;
190         }
191         sc->have_audio = 0;
192         return NULL;
193 }
194
195 /*!
196  * \internal
197  * \brief Process a softmix channel's write audio
198  *
199  * \details This function will remove the channel's talking from its own audio if present and
200  * possibly even do the channel's write translation for it depending on how many other
201  * channels use the same write format.
202  */
203 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
204         struct ast_format *raw_write_fmt,
205         struct softmix_channel *sc, unsigned int default_sample_size)
206 {
207         struct softmix_translate_helper_entry *entry = NULL;
208         int i;
209
210         /* If we provided any audio then take it out while in slinear format. */
211         if (sc->have_audio && !sc->binaural) {
212                 for (i = 0; i < sc->write_frame.samples; i++) {
213                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
214                 }
215                 /* check to see if any entries exist for the format. if not we'll want
216                    to remove it during cleanup */
217                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
218                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
219                                 ++entry->num_times_requested;
220                                 break;
221                         }
222                 }
223                 /* do not do any special write translate optimization if we had to make
224                  * a special mix for them to remove their own audio. */
225                 return;
226         } else if (sc->have_audio && sc->binaural > 0) {
227                 /*
228                  * Binaural audio requires special saturated substract since we have two
229                  * audio signals per channel now.
230                  */
231                 softmix_process_write_binaural_audio(sc, default_sample_size);
232                 return;
233         }
234
235         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
236            of translation paths and track the number of references for each type. Each one of the same
237            type should be able to use the same out_frame. Since the optimization is only necessary for
238            multiple channels (>=2) using the same codec make sure resources are allocated only when
239            needed and released when not (see also softmix_translate_helper_cleanup */
240         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
241                 if (sc->binaural != 0) {
242                         continue;
243                 }
244                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
245                         entry->num_times_requested++;
246                 } else {
247                         continue;
248                 }
249                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
250                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
251                 }
252                 if (entry->trans_pvt && !entry->out_frame) {
253                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
254                 }
255                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
256                                 && entry->out_frame->datalen < MAX_DATALEN) {
257                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
258                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
259                         sc->write_frame.datalen = entry->out_frame->datalen;
260                         sc->write_frame.samples = entry->out_frame->samples;
261                 }
262                 break;
263         }
264
265         /* add new entry into list if this format destination was not matched. */
266         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
267                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
268         }
269 }
270
271 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
272 {
273         struct softmix_translate_helper_entry *entry;
274
275         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
276                 /* if it hasn't been requested then remove it */
277                 if (!entry->num_times_requested) {
278                         AST_LIST_REMOVE_CURRENT(entry);
279                         softmix_translate_helper_free_entry(entry);
280                         continue;
281                 }
282
283                 if (entry->out_frame) {
284                         ast_frfree(entry->out_frame);
285                         entry->out_frame = NULL;
286                 }
287
288                 /* nothing is optimized for a single path reference, so there is
289                    no reason to continue to hold onto the codec */
290                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
291                         ast_translator_free_path(entry->trans_pvt);
292                         entry->trans_pvt = NULL;
293                 }
294
295                 /* for each iteration (a mixing run) in the bridge softmix thread the number
296                    of references to a given entry is recalculated, so reset the number of
297                    times requested */
298                 entry->num_times_requested = 0;
299         }
300         AST_LIST_TRAVERSE_SAFE_END;
301 }
302
303 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
304 {
305         struct softmix_channel *sc = bridge_channel->tech_pvt;
306         struct ast_format *slin_format;
307         int setup_fail;
308
309 #ifdef BINAURAL_RENDERING
310         if (interval != BINAURAL_MIXING_INTERVAL) {
311                 interval = BINAURAL_MIXING_INTERVAL;
312         }
313 #endif
314
315         /* The callers have already ensured that sc is never NULL. */
316         ast_assert(sc != NULL);
317
318         slin_format = ast_format_cache_get_slin_by_rate(rate);
319
320         ast_mutex_lock(&sc->lock);
321         if (reset) {
322                 ast_slinfactory_destroy(&sc->factory);
323                 ast_dsp_free(sc->dsp);
324         }
325
326         /* Setup write frame parameters */
327         sc->write_frame.frametype = AST_FRAME_VOICE;
328         /*
329          * NOTE: The write_frame format holds a reference because translation
330          * could be needed and the format changed to the translated format
331          * for the channel.  The translated format may not be a
332          * static cached format.
333          */
334         ao2_replace(sc->write_frame.subclass.format, slin_format);
335         sc->write_frame.data.ptr = sc->final_buf;
336         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
337         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
338
339         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
340         sc->rate = rate;
341
342         /* If the channel will contain binaural data we will set a identifier in the channel
343          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
344         if (set_binaural == 1) {
345                 sc->binaural = 1;
346         } else if (set_binaural == 0) {
347                 sc->binaural = 0;
348         }
349
350         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
351          * and doesn't have to be done if we just updating sample rates. */
352         if (binaural_pos_id != -1) {
353                 sc->binaural_pos = binaural_pos_id;
354         }
355         if (is_announcement != -1) {
356                 sc->is_announcement = is_announcement;
357         }
358
359         /*
360          * NOTE: The read_slin_format does not hold a reference because it
361          * will always be a signed linear format.
362          */
363         sc->read_slin_format = slin_format;
364
365         /* Setup smoother */
366         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
367
368         /* set new read and write formats on channel. */
369         ast_channel_lock(bridge_channel->chan);
370         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
371                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
372         ast_channel_unlock(bridge_channel->chan);
373
374         /* If channel contains binaural data we will set it here for the trans_pvt. */
375         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
376                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
377         } else if (set_binaural == 0) {
378                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
379         }
380
381         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
382         sc->dsp = ast_dsp_new_with_rate(rate);
383         if (setup_fail || !sc->dsp) {
384                 /* Bad news.  Could not setup the channel for softmix. */
385                 ast_mutex_unlock(&sc->lock);
386                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
387                 return;
388         }
389
390         /* we want to aggressively detect silence to avoid feedback */
391         if (bridge_channel->tech_args.talking_threshold) {
392                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
393         } else {
394                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
395         }
396
397         ast_mutex_unlock(&sc->lock);
398 }
399
400 /*!
401  * \internal
402  * \brief Poke the mixing thread in case it is waiting for an active channel.
403  * \since 12.0.0
404  *
405  * \param softmix_data Bridge mixing data.
406  *
407  * \return Nothing
408  */
409 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
410 {
411         ast_mutex_lock(&softmix_data->lock);
412         ast_cond_signal(&softmix_data->cond);
413         ast_mutex_unlock(&softmix_data->lock);
414 }
415
416 /*! \brief Function called when a channel is unsuspended from the bridge */
417 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
418 {
419 #ifdef BINAURAL_RENDERING
420         struct softmix_channel *sc = bridge_channel->tech_pvt;
421         if (sc->binaural) {
422                 /* Restore some usefull data if it was a binaural channel */
423                 struct ast_format *slin_format;
424
425                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
426                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
427         }
428 #endif
429         if (bridge->tech_pvt) {
430                 softmix_poke_thread(bridge->tech_pvt);
431         }
432 }
433
434 /*!
435  * \brief Determine if a stream is a video source stream.
436  *
437  * \param stream The stream to test
438  * \retval 1 The stream is a video source
439  * \retval 0 The stream is not a video source
440  */
441 static int is_video_source(const struct ast_stream *stream)
442 {
443         if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED
444                 && ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO
445                 && strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
446                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
447                 return 1;
448         }
449
450         return 0;
451 }
452
453 /*!
454  * \brief Determine if a stream is a video destination stream.
455  *
456  * A source channel name can be provided to narrow this to a destination stream
457  * for a particular source channel. Further, a source stream name can be provided
458  * to narrow this to a particular source stream's destination. However, empty strings
459  * can be provided to match any destination video stream, regardless of source channel
460  * or source stream.
461  *
462  * \param stream The stream to test
463  * \param source_channel_name The name of a source video channel to match
464  * \param source_channel_stream_position The position of the video on the source channel
465  * \retval 1 The stream is a video destination stream
466  * \retval 0 The stream is not a video destination stream
467  */
468 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
469         int source_channel_stream_position)
470 {
471         char *dest_video_name;
472         size_t dest_video_name_len;
473
474         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED
475                 || ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
476                 return 0;
477         }
478
479         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
480         if (!ast_strlen_zero(source_channel_name)) {
481                 dest_video_name_len += strlen(source_channel_name) + 1;
482                 if (source_channel_stream_position != -1) {
483                         dest_video_name_len += 11;
484                 }
485
486                 dest_video_name = ast_alloca(dest_video_name_len);
487                 if (source_channel_stream_position != -1) {
488                         /* We are looking for an exact stream position */
489                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%d",
490                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
491                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
492                                 source_channel_stream_position);
493                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
494                 }
495                 snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
496                         SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
497                         source_channel_name);
498         } else {
499                 dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
500         }
501
502         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
503 }
504
505 static int append_source_stream(struct ast_stream_topology *dest,
506         const char *channel_name, const char *sdp_label,
507         struct ast_stream *stream, int index)
508 {
509         char *stream_clone_name = NULL;
510         struct ast_stream *stream_clone;
511
512         /* We use the stream topology index for the stream to uniquely identify and recognize it.
513          * This is guaranteed to remain the same across renegotiation of the source channel and
514          * ensures that the stream name is unique.
515          */
516         if (ast_asprintf(&stream_clone_name, "%s%c%s%c%d", SOFTBRIDGE_VIDEO_DEST_PREFIX,
517                 SOFTBRIDGE_VIDEO_DEST_SEPARATOR, channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
518                 index) < 0) {
519                 return -1;
520         }
521
522         stream_clone = ast_stream_clone(stream, stream_clone_name);
523         ast_free(stream_clone_name);
524         if (!stream_clone) {
525                 return -1;
526         }
527
528         /* Sends an "a:label" attribute in the SDP for participant event correlation */
529         if (!ast_strlen_zero(sdp_label)) {
530                 ast_stream_set_metadata(stream_clone, "SDP:LABEL", sdp_label);
531         }
532
533         /* We will be sending them a stream and not expecting anything in return */
534         ast_stream_set_state(stream_clone, AST_STREAM_STATE_SENDONLY);
535
536         if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
537                 ast_stream_free(stream_clone);
538                 return -1;
539         }
540
541         return 0;
542 }
543
544
545 static int append_source_streams(struct ast_stream_topology *dest,
546         const char *channel_name, const char *sdp_label,
547         const struct ast_stream_topology *source)
548 {
549         int i;
550
551         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
552                 struct ast_stream *stream;
553
554                 stream = ast_stream_topology_get_stream(source, i);
555
556                 if (!is_video_source(stream)) {
557                         continue;
558                 }
559
560                 if (append_source_stream(dest, channel_name, sdp_label, stream, i)) {
561                         return -1;
562                 }
563         }
564
565         return 0;
566 }
567
568 static int append_all_streams(struct ast_stream_topology *dest,
569         const struct ast_stream_topology *source)
570 {
571         int i;
572         int dest_index = 0;
573
574         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
575                 struct ast_stream *clone;
576                 int added = 0;
577
578                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
579                 if (!clone) {
580                         return -1;
581                 }
582
583                 /* If we can reuse an existing removed stream then do so */
584                 while (dest_index < ast_stream_topology_get_count(dest)) {
585                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
586
587                         dest_index++;
588
589                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
590                                 /* This cannot fail because dest_index - 1 is less than the
591                                  * current count in dest. */
592                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
593                                 added = 1;
594                                 break;
595                         }
596                 }
597
598                 /* If no removed stream exists that we took the place of append the stream */
599                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
600                         ast_stream_free(clone);
601                         return -1;
602                 }
603         }
604
605         return 0;
606 }
607
608 /*!
609  * \brief Issue channel stream topology change requests.
610  *
611  * When in SFU mode, each participant needs to be able to
612  * send video directly to other participants in the bridge.
613  * This means that all participants need to have their topologies
614  * updated. The joiner needs to have destination streams for
615  * all current participants, and the current participants need
616  * to have destinations streams added for the joiner's sources.
617  *
618  * \param joiner The channel that is joining the softmix bridge
619  * \param participants The current participants in the softmix bridge
620  */
621 static void sfu_topologies_on_join(struct ast_bridge *bridge,
622         struct ast_bridge_channel *joiner)
623 {
624         struct ast_stream_topology *joiner_video = NULL;
625         struct ast_bridge_channels_list *participants = &bridge->channels;
626         struct ast_bridge_channel *participant;
627         int res;
628         struct softmix_channel *sc;
629
630         joiner_video = ast_stream_topology_alloc();
631         if (!joiner_video) {
632                 return;
633         }
634
635         sc = joiner->tech_pvt;
636
637         ast_channel_lock(joiner->chan);
638         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan),
639                 bridge->softmix.send_sdp_label ? ast_channel_uniqueid(joiner->chan) : NULL,
640                 ast_channel_get_stream_topology(joiner->chan));
641         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
642         ast_channel_unlock(joiner->chan);
643
644         if (res || !sc->topology) {
645                 goto cleanup;
646         }
647
648         AST_LIST_TRAVERSE(participants, participant, entry) {
649                 if (participant == joiner) {
650                         continue;
651                 }
652                 ast_channel_lock(participant->chan);
653                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
654                         bridge->softmix.send_sdp_label ? ast_channel_uniqueid(participant->chan) : NULL,
655                         ast_channel_get_stream_topology(participant->chan));
656                 ast_channel_unlock(participant->chan);
657                 if (res) {
658                         goto cleanup;
659                 }
660         }
661
662         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
663
664         AST_LIST_TRAVERSE(participants, participant, entry) {
665                 if (participant == joiner) {
666                         continue;
667                 }
668
669                 sc = participant->tech_pvt;
670                 if (append_all_streams(sc->topology, joiner_video)) {
671                         goto cleanup;
672                 }
673                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
674         }
675
676 cleanup:
677         ast_stream_topology_free(joiner_video);
678 }
679
680 /*! \brief Function called when a channel is joined into the bridge */
681 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
682 {
683         struct softmix_channel *sc;
684         struct softmix_bridge_data *softmix_data;
685         int set_binaural = 0;
686         /*
687          * If false, the channel will be convolved, but since it is a non stereo channel, output
688          * will be mono.
689          */
690         int skip_binaural_output = 1;
691         int pos_id;
692         int is_announcement = 0;
693         int samplerate_change;
694
695         softmix_data = bridge->tech_pvt;
696         if (!softmix_data) {
697                 return -1;
698         }
699
700         /* Create a new softmix_channel structure and allocate various things on it */
701         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
702                 return -1;
703         }
704
705         samplerate_change = softmix_data->internal_rate;
706         pos_id = -1;
707         if (bridge->softmix.binaural_active) {
708                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
709                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
710                         if (set_binaural) {
711                                 softmix_data->internal_rate = samplerate_change;
712                         }
713                         skip_binaural_output = 0;
714                 } else {
715                         is_announcement = 1;
716                 }
717                 if (set_binaural) {
718                         softmix_data->convolve.binaural_active = 1;
719                 }
720                 if (!skip_binaural_output)      {
721                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
722                         if (pos_id == -1) {
723                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
724                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
725                                                 ast_channel_name(bridge_channel->chan));
726                                 ast_free(sc);
727                                 return -1;
728                         }
729                 }
730         }
731
732         /* Can't forget the lock */
733         ast_mutex_init(&sc->lock);
734
735         /* Can't forget to record our pvt structure within the bridged channel structure */
736         bridge_channel->tech_pvt = sc;
737
738         set_softmix_bridge_data(softmix_data->internal_rate,
739                 softmix_data->internal_mixing_interval
740                         ? softmix_data->internal_mixing_interval
741                         : DEFAULT_SOFTMIX_INTERVAL,
742                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
743
744         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
745                 sfu_topologies_on_join(bridge, bridge_channel);
746         }
747
748         /* Complete any active hold before entering, or transitioning to softmix. */
749         if (ast_channel_hold_state(bridge_channel->chan) == AST_CONTROL_HOLD) {
750                 ast_debug(1, "Channel %s simulating UNHOLD for bridge softmix join.\n",
751                         ast_channel_name(bridge_channel->chan));
752                 ast_indicate(bridge_channel->chan, AST_CONTROL_UNHOLD);
753         }
754
755         softmix_poke_thread(softmix_data);
756         return 0;
757 }
758
759 static int remove_destination_streams(struct ast_stream_topology *topology,
760         const char *channel_name)
761 {
762         int i;
763         int stream_removed = 0;
764
765         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
766                 struct ast_stream *stream;
767
768                 stream = ast_stream_topology_get_stream(topology, i);
769
770                 if (is_video_dest(stream, channel_name, -1)) {
771                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
772                         stream_removed = 1;
773                 }
774         }
775         return stream_removed;
776 }
777
778 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
779 {
780         struct ast_bridge_channel *participant;
781         struct softmix_channel *sc;
782
783         AST_LIST_TRAVERSE(participants, participant, entry) {
784                 sc = participant->tech_pvt;
785                 if (!remove_destination_streams(sc->topology, ast_channel_name(leaver->chan))) {
786                         continue;
787                 }
788                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
789         }
790
791         sc = leaver->tech_pvt;
792         if (remove_destination_streams(sc->topology, "")) {
793                 ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
794         }
795
796         return 0;
797 }
798
799 /*! \brief Function called when a channel leaves the bridge */
800 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
801 {
802         struct softmix_channel *sc;
803         struct softmix_bridge_data *softmix_data;
804
805         softmix_data = bridge->tech_pvt;
806         sc = bridge_channel->tech_pvt;
807         if (!sc) {
808                 return;
809         }
810
811         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
812                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
813         }
814
815         if (bridge->softmix.binaural_active) {
816                 if (sc->binaural) {
817                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
818                                         softmix_data->default_sample_size);
819                 }
820         }
821
822         bridge_channel->tech_pvt = NULL;
823
824         ast_stream_topology_free(sc->topology);
825
826         ao2_cleanup(sc->remb_collector);
827
828         AST_VECTOR_FREE(&sc->video_sources);
829
830         /* Drop mutex lock */
831         ast_mutex_destroy(&sc->lock);
832
833         /* Drop the factory */
834         ast_slinfactory_destroy(&sc->factory);
835
836         /* Drop any formats on the frames */
837         ao2_cleanup(sc->write_frame.subclass.format);
838
839         /* Drop the DSP */
840         ast_dsp_free(sc->dsp);
841
842         /* Eep! drop ourselves */
843         ast_free(sc);
844 }
845
846 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
847 {
848         struct ast_bridge_channel *cur;
849
850         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
851                 if (cur->suspended) {
852                         continue;
853                 }
854                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
855                         ast_bridge_channel_queue_frame(cur, frame);
856                         break;
857                 }
858         }
859 }
860
861 /*!
862  * \internal
863  * \brief Determine what to do with a video frame.
864  * \since 12.0.0
865  *
866  * \param bridge Which bridge is getting the frame
867  * \param bridge_channel Which channel is writing the frame.
868  * \param frame What is being written.
869  *
870  * \return Nothing
871  */
872 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
873 {
874         struct softmix_channel *sc;
875         int video_src_priority;
876
877         /* Determine if the video frame should be distributed or not */
878         switch (bridge->softmix.video_mode.mode) {
879         case AST_BRIDGE_VIDEO_MODE_NONE:
880                 break;
881         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
882                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
883                 if (video_src_priority == 1) {
884                         /* Pass to me and everyone else. */
885                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
886                 }
887                 break;
888         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
889                 sc = bridge_channel->tech_pvt;
890                 ast_mutex_lock(&sc->lock);
891                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
892                         sc->video_talker.energy_average,
893                         frame->subclass.frame_ending);
894                 ast_mutex_unlock(&sc->lock);
895                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
896                 if (video_src_priority == 1) {
897                         int num_src = ast_bridge_number_video_src(bridge);
898                         int echo = num_src > 1 ? 0 : 1;
899
900                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
901                 } else if (video_src_priority == 2) {
902                         softmix_pass_video_top_priority(bridge, frame);
903                 }
904                 break;
905         case AST_BRIDGE_VIDEO_MODE_SFU:
906                 /* Nothing special to do here, the bridge channel stream map will ensure the
907                  * video goes everywhere it needs to
908                  */
909                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
910                 break;
911         }
912 }
913
914 /*!
915  * \internal
916  * \brief Determine what to do with a voice frame.
917  * \since 12.0.0
918  *
919  * \param bridge Which bridge is getting the frame
920  * \param bridge_channel Which channel is writing the frame.
921  * \param frame What is being written.
922  *
923  * \return Nothing
924  */
925 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
926 {
927         struct softmix_channel *sc = bridge_channel->tech_pvt;
928         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
929         int silent = 0;
930         int totalsilence = 0;
931         int cur_energy = 0;
932         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
933                 bridge_channel->tech_args.silence_threshold :
934                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
935         /*
936          * If update_talking is set to 0 or 1, tell the bridge that the channel
937          * has started or stopped talking.
938          */
939         char update_talking = -1;
940
941         /* Write the frame into the conference */
942         ast_mutex_lock(&sc->lock);
943
944         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
945                 /*
946                  * The incoming frame is not the expected format.  Update
947                  * the channel's translation path to get us slinear from
948                  * the new format for the next frame.
949                  *
950                  * There is the possibility that this frame is an old slinear
951                  * rate frame that was in flight when the softmix bridge
952                  * changed rates.  If so it will self correct on subsequent
953                  * frames.
954                  */
955                 ast_channel_lock(bridge_channel->chan);
956                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
957                         ast_channel_name(bridge_channel->chan),
958                         ast_format_get_name(frame->subclass.format),
959                         ast_format_get_name(sc->read_slin_format));
960                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
961                         sc->read_slin_format);
962                 ast_channel_unlock(bridge_channel->chan);
963         }
964
965         /* The channel will be leaving soon if there is no dsp. */
966         if (sc->dsp) {
967                 silent = ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
968         }
969
970         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
971                 int cur_slot = sc->video_talker.energy_history_cur_slot;
972
973                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
974                 sc->video_talker.energy_accum += cur_energy;
975                 sc->video_talker.energy_history[cur_slot] = cur_energy;
976                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
977                 sc->video_talker.energy_history_cur_slot++;
978                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
979                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
980                 }
981         }
982
983         if (totalsilence < silence_threshold) {
984                 if (!sc->talking && !silent) {
985                         /* Tell the write process we have audio to be mixed out */
986                         sc->talking = 1;
987                         update_talking = 1;
988                 }
989         } else {
990                 if (sc->talking) {
991                         sc->talking = 0;
992                         update_talking = 0;
993                 }
994         }
995
996         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
997          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
998          * the audio by flushing the buffer before adding new audio in. */
999         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
1000                 ast_slinfactory_flush(&sc->factory);
1001         }
1002
1003         if (sc->talking || !bridge_channel->tech_args.drop_silence) {
1004                 /* Add frame to the smoother for mixing with other channels. */
1005                 ast_slinfactory_feed(&sc->factory, frame);
1006         }
1007
1008         /* Alllll done */
1009         ast_mutex_unlock(&sc->lock);
1010
1011         if (update_talking != -1) {
1012                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
1013         }
1014 }
1015
1016 /*!
1017  * \internal
1018  * \brief Clear talking flag, stop contributing to mixing and notify handlers.
1019  * \since 13.21.0, 15.4.0
1020  *
1021  * \param bridge_channel Which channel's talking to clear
1022  *
1023  * \return Nothing
1024  */
1025 static void clear_talking(struct ast_bridge_channel *bridge_channel)
1026 {
1027         struct softmix_channel *sc = bridge_channel->tech_pvt;
1028
1029         if (sc->talking) {
1030                 ast_mutex_lock(&sc->lock);
1031                 ast_slinfactory_flush(&sc->factory);
1032                 sc->talking = 0;
1033                 ast_mutex_unlock(&sc->lock);
1034
1035                 /* Notify that we are no longer talking. */
1036                 ast_bridge_channel_notify_talking(bridge_channel, 0);
1037         }
1038 }
1039
1040 /*!
1041  * \internal
1042  * \brief Check for voice status updates.
1043  * \since 13.20.0
1044  *
1045  * \param bridge Which bridge we are in
1046  * \param bridge_channel Which channel we are checking
1047  *
1048  * \return Nothing
1049  */
1050 static void softmix_bridge_check_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1051 {
1052         if (bridge_channel->features->mute) {
1053                 /*
1054                  * We were muted while we were talking.
1055                  *
1056                  * Immediately stop contributing to mixing
1057                  * and report no longer talking.
1058                  */
1059                 clear_talking(bridge_channel);
1060         }
1061 }
1062
1063 static int remove_all_original_streams(struct ast_stream_topology *dest,
1064         const struct ast_stream_topology *source,
1065         const struct ast_stream_topology *original)
1066 {
1067         int i;
1068
1069         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
1070                 struct ast_stream *stream;
1071                 int original_index;
1072
1073                 stream = ast_stream_topology_get_stream(source, i);
1074
1075                 /* Mark the existing stream as removed so we get a new one, this will get
1076                  * reused on a subsequent renegotiation.
1077                  */
1078                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
1079                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
1080
1081                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
1082                                 struct ast_stream *removed;
1083
1084                                 /* Since the participant is still going to be in the bridge we
1085                                  * change the name so that routing does not attempt to route video
1086                                  * to this stream.
1087                                  */
1088                                 removed = ast_stream_clone(stream, "removed");
1089                                 if (!removed) {
1090                                         return -1;
1091                                 }
1092
1093                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
1094
1095                                 /* The destination topology can only ever contain the same, or more,
1096                                  * streams than the original so this is safe.
1097                                  */
1098                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
1099                                         ast_stream_free(removed);
1100                                         return -1;
1101                                 }
1102
1103                                 break;
1104                         }
1105                 }
1106         }
1107
1108         return 0;
1109 }
1110
1111 static void sfu_topologies_on_source_change(struct ast_bridge *bridge,
1112         struct ast_bridge_channel *source)
1113 {
1114         struct ast_stream_topology *source_video = NULL;
1115         struct ast_bridge_channels_list *participants = &bridge->channels;
1116         struct ast_bridge_channel *participant;
1117         int res;
1118
1119         source_video = ast_stream_topology_alloc();
1120         if (!source_video) {
1121                 return;
1122         }
1123
1124         ast_channel_lock(source->chan);
1125         res = append_source_streams(source_video, ast_channel_name(source->chan),
1126                 bridge->softmix.send_sdp_label ? ast_channel_uniqueid(source->chan) : NULL,
1127                 ast_channel_get_stream_topology(source->chan));
1128         ast_channel_unlock(source->chan);
1129         if (res) {
1130                 goto cleanup;
1131         }
1132
1133         AST_LIST_TRAVERSE(participants, participant, entry) {
1134                 struct ast_stream_topology *original_topology;
1135                 struct softmix_channel *sc;
1136
1137                 if (participant == source) {
1138                         continue;
1139                 }
1140
1141                 sc = participant->tech_pvt;
1142
1143                 original_topology = ast_stream_topology_clone(sc->topology);
1144                 if (!original_topology) {
1145                         goto cleanup;
1146                 }
1147
1148                 /* We add all the source streams back in, if any removed streams are already present they will
1149                  * get used first followed by appending new ones.
1150                  */
1151                 if (append_all_streams(sc->topology, source_video)) {
1152                         ast_stream_topology_free(original_topology);
1153                         goto cleanup;
1154                 }
1155
1156                 /* And the original existing streams get marked as removed. This causes the remote side to see
1157                  * a new stream for the source streams.
1158                  */
1159                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1160                         ast_stream_topology_free(original_topology);
1161                         goto cleanup;
1162                 }
1163
1164                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1165                 ast_stream_topology_free(original_topology);
1166         }
1167
1168 cleanup:
1169         ast_stream_topology_free(source_video);
1170 }
1171
1172 /*!
1173  * \internal
1174  * \brief Determine what to do with a text frame.
1175  * \since 13.22.0
1176  * \since 15.5.0
1177  *
1178  * \param bridge Which bridge is getting the frame
1179  * \param bridge_channel Which channel is writing the frame.
1180  * \param frame What is being written.
1181  *
1182  * \return Nothing
1183  */
1184 static void softmix_bridge_write_text(struct ast_bridge *bridge,
1185         struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1186 {
1187         if (DEBUG_ATLEAST(1)) {
1188                 struct ast_msg_data *msg = frame->data.ptr;
1189                 char frame_type[64];
1190
1191                 ast_frame_type2str(frame->frametype, frame_type, sizeof(frame_type));
1192
1193                 if (frame->frametype == AST_FRAME_TEXT_DATA) {
1194                         ast_log(LOG_DEBUG, "Received %s frame from '%s:%s': %s\n", frame_type,
1195                                 ast_msg_data_get_attribute(msg, AST_MSG_DATA_ATTR_FROM),
1196                                 ast_channel_name(bridge_channel->chan),
1197                                 ast_msg_data_get_attribute(msg, AST_MSG_DATA_ATTR_BODY));
1198                 } else {
1199                         ast_log(LOG_DEBUG, "Received %s frame from '%s': %.*s\n", frame_type,
1200                                 ast_channel_name(bridge_channel->chan), frame->datalen,
1201                                 (char *)frame->data.ptr);
1202                 }
1203         }
1204
1205         ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1206 }
1207
1208 /*!
1209  * \internal
1210  * \brief Determine what to do with a control frame.
1211  * \since 12.0.0
1212  *
1213  * \param bridge Which bridge is getting the frame
1214  * \param bridge_channel Which channel is writing the frame.
1215  * \param frame What is being written.
1216  *
1217  * \retval 0 Frame accepted into the bridge.
1218  * \retval -1 Frame needs to be deferred.
1219  */
1220 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1221 {
1222         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1223
1224         /*
1225          * XXX Softmix needs to use channel roles to determine what to
1226          * do with control frames.
1227          */
1228
1229         switch (frame->subclass.integer) {
1230         case AST_CONTROL_HOLD:
1231                 /*
1232                  * Doing anything for holds in a conference bridge could be considered a bit
1233                  * odd. That being said, in most cases one would probably want the talking
1234                  * flag cleared when 'hold' is pressed by the remote endpoint, so go ahead
1235                  * and do that here. However, that is all we'll do. Meaning if for some reason
1236                  * the endpoint continues to send audio frames despite pressing 'hold' talking
1237                  * will once again be detected for that channel.
1238                  */
1239                 clear_talking(bridge_channel);
1240                 break;
1241         case AST_CONTROL_VIDUPDATE:
1242                 if (!bridge->softmix.video_mode.video_update_discard ||
1243                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1244                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1245                         softmix_data->last_video_update = ast_tvnow();
1246                 }
1247                 break;
1248         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1249                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1250                         sfu_topologies_on_source_change(bridge, bridge_channel);
1251                 }
1252                 break;
1253         default:
1254                 break;
1255         }
1256
1257         return 0;
1258 }
1259
1260 /*!
1261  * \internal
1262  * \brief Determine what to do with an RTCP frame.
1263  * \since 15.4.0
1264  *
1265  * \param bridge Which bridge is getting the frame
1266  * \param bridge_channel Which channel is writing the frame.
1267  * \param frame What is being written.
1268  */
1269 static void softmix_bridge_write_rtcp(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1270 {
1271         struct ast_rtp_rtcp_feedback *feedback = frame->data.ptr;
1272         struct softmix_channel *sc = bridge_channel->tech_pvt;
1273
1274         /* We only care about REMB reports right now. In the future we may be able to use sender or
1275          * receiver reports to further tweak things, but not yet.
1276          */
1277         if (frame->subclass.integer != AST_RTP_RTCP_PSFB || feedback->fmt != AST_RTP_RTCP_FMT_REMB ||
1278                 bridge->softmix.video_mode.mode != AST_BRIDGE_VIDEO_MODE_SFU ||
1279                 !bridge->softmix.video_mode.mode_data.sfu_data.remb_send_interval) {
1280                 return;
1281         }
1282
1283         /* REMB is the total estimated maximum bitrate across all streams within the session, so we store
1284          * only the latest report and use it everywhere.
1285          */
1286         ast_mutex_lock(&sc->lock);
1287         sc->remb = feedback->remb;
1288         ast_mutex_unlock(&sc->lock);
1289
1290         return;
1291 }
1292
1293 /*!
1294  * \internal
1295  * \brief Determine what to do with a frame written into the bridge.
1296  * \since 12.0.0
1297  *
1298  * \param bridge Which bridge is getting the frame
1299  * \param bridge_channel Which channel is writing the frame.
1300  * \param frame What is being written.
1301  *
1302  * \retval 0 Frame accepted into the bridge.
1303  * \retval -1 Frame needs to be deferred.
1304  *
1305  * \note On entry, bridge is already locked.
1306  */
1307 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1308 {
1309         int res = 0;
1310
1311         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1312                 /* "Accept" the frame and discard it. */
1313                 return 0;
1314         }
1315
1316         /*
1317          * XXX Softmix needs to use channel roles to determine who gets
1318          * what frame.  Possible roles: announcer, recorder, agent,
1319          * supervisor.
1320          */
1321         switch (frame->frametype) {
1322         case AST_FRAME_NULL:
1323                 /* "Accept" the frame and discard it. */
1324                 softmix_bridge_check_voice(bridge, bridge_channel);
1325                 break;
1326         case AST_FRAME_DTMF_BEGIN:
1327         case AST_FRAME_DTMF_END:
1328                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1329                 break;
1330         case AST_FRAME_VOICE:
1331                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1332                 break;
1333         case AST_FRAME_VIDEO:
1334                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1335                 break;
1336         case AST_FRAME_TEXT:
1337         case AST_FRAME_TEXT_DATA:
1338                 softmix_bridge_write_text(bridge, bridge_channel, frame);
1339                 break;
1340         case AST_FRAME_CONTROL:
1341                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1342                 break;
1343         case AST_FRAME_RTCP:
1344                 softmix_bridge_write_rtcp(bridge, bridge_channel, frame);
1345                 break;
1346         case AST_FRAME_BRIDGE_ACTION:
1347                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1348                 break;
1349         case AST_FRAME_BRIDGE_ACTION_SYNC:
1350                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1351                 ast_assert(0);
1352         default:
1353                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1354                 /* "Accept" the frame and discard it. */
1355                 break;
1356         }
1357
1358         return res;
1359 }
1360
1361 static void remb_collect_report_all(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
1362         float bitrate)
1363 {
1364         if (!softmix_data->bitrate) {
1365                 softmix_data->bitrate = bitrate;
1366                 return;
1367         }
1368
1369         switch (bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior) {
1370         case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE_ALL:
1371                 softmix_data->bitrate = (softmix_data->bitrate + bitrate) / 2;
1372                 break;
1373         case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST_ALL:
1374                 if (bitrate < softmix_data->bitrate) {
1375                         softmix_data->bitrate = bitrate;
1376                 }
1377                 break;
1378         case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST_ALL:
1379                 if (bitrate > softmix_data->bitrate) {
1380                         softmix_data->bitrate = bitrate;
1381                 }
1382                 break;
1383         case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE:
1384         case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST:
1385         case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST:
1386                 /* These will never actually get hit due to being handled by remb_collect_report below */
1387                 break;
1388         }
1389 }
1390
1391 static void remb_collect_report(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
1392         struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
1393 {
1394         int i;
1395         float bitrate;
1396
1397         /* If there are no video sources that we are a receiver of then we have noone to
1398          * report REMB to.
1399          */
1400         if (!AST_VECTOR_SIZE(&sc->video_sources)) {
1401                 return;
1402         }
1403
1404         /* We evenly divide the available maximum bitrate across the video sources
1405          * to this receiver so each source gets an equal slice.
1406          */
1407         bitrate = (sc->remb.br_mantissa << sc->remb.br_exp) / AST_VECTOR_SIZE(&sc->video_sources);
1408
1409         /* If this receiver has no bitrate yet ignore it */
1410         if (!bitrate) {
1411                 return;
1412         }
1413
1414         /* If we are using the "all" variants then we should use the bridge bitrate to store information */
1415         if (bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior == AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE_ALL ||
1416                 bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior == AST_BRIDGE_VIDEO_SFU_REMB_LOWEST_ALL ||
1417                 bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior == AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST_ALL) {
1418                 remb_collect_report_all(bridge, softmix_data, bitrate);
1419                 return;
1420         }
1421
1422         for (i = 0; i < AST_VECTOR_SIZE(&sc->video_sources); ++i) {
1423                 struct softmix_remb_collector *collector;
1424
1425                 /* The collector will always exist if a video source is in our list */
1426                 collector = AST_VECTOR_GET(&softmix_data->remb_collectors, AST_VECTOR_GET(&sc->video_sources, i));
1427
1428                 if (!collector->bitrate) {
1429                         collector->bitrate = bitrate;
1430                         continue;
1431                 }
1432
1433                 switch (bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior) {
1434                 case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE:
1435                         collector->bitrate = (collector->bitrate + bitrate) / 2;
1436                         break;
1437                 case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST:
1438                         if (bitrate < collector->bitrate) {
1439                                 collector->bitrate = bitrate;
1440                         }
1441                         break;
1442                 case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST:
1443                         if (bitrate > collector->bitrate) {
1444                                 collector->bitrate = bitrate;
1445                         }
1446                         break;
1447                 case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE_ALL:
1448                 case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST_ALL:
1449                 case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST_ALL:
1450                         /* These will never actually get hit due to being handled by remb_collect_report_all above */
1451                         break;
1452                 }
1453         }
1454
1455         /* After the report is integrated we reset this to 0 in case they stop producing
1456          * REMB reports.
1457          */
1458         sc->remb.br_mantissa = 0;
1459         sc->remb.br_exp = 0;
1460 }
1461
1462 static void remb_send_report(struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data,
1463         struct softmix_channel *sc)
1464 {
1465         float bitrate = softmix_data->bitrate;
1466         int i;
1467         int exp;
1468
1469         if (!sc->remb_collector) {
1470                 return;
1471         }
1472
1473         /* If there is no bridge level bitrate fall back to collector level */
1474         if (!bitrate) {
1475                 bitrate = sc->remb_collector->bitrate;
1476                 sc->remb_collector->bitrate = 0;
1477         }
1478
1479         /* We always do this calculation as even when the bitrate is zero the browser
1480          * still prefers it to be accurate instead of lying.
1481          *
1482          * The mantissa only has 18 bits available, so make sure it fits. Adjust the
1483          * value and exponent for those values that don't.
1484          *
1485          * For example given the following:
1486          *
1487          * bitrate = 123456789.0
1488          * frexp(bitrate, &exp);
1489          *
1490          * 'exp' should now equal 27 (number of bits needed to represent the value). Since
1491          * the mantissa must fit into an 18-bit unsigned integer, and the given bitrate is
1492          * too large to fit, we must subtract 18 from the exponent in order to get the
1493          * number of times the bitrate will fit into that size integer.
1494          *
1495          * exp -= 18;
1496          *
1497          * 'exp' is now equal to 9. Now we can get the mantissa that fits into an 18-bit
1498          * unsigned integer by dividing the bitrate by 2^exp:
1499          *
1500          * mantissa = 123456789.0 / 2^9
1501          *
1502          * This makes the final mantissa equal to 241126 (implicitly cast), which is less
1503          * than 262143 (the max value that can be put into an unsigned 18-bit integer).
1504          * So now we have the following:
1505          *
1506          * exp = 9;
1507          * mantissa = 241126;
1508          *
1509          * If we multiply that back we should come up with something close to the original
1510          * bit rate:
1511          *
1512          * 241126 * 2^9 = 123456512
1513          *
1514          * Precision is lost due to the nature of floating point values. Easier to why from
1515          * the binary:
1516          *
1517          * 241126 * 2^9 = 241126 << 9 = 111010110111100110 << 9 = 111010110111100110000000000
1518          *
1519          * Precision on the "lower" end is lost due to zeros being shifted in. This loss is
1520          * both expected and acceptable.
1521          */
1522         frexp(bitrate, &exp);
1523         exp = exp > 18 ? exp - 18 : 0;
1524
1525         sc->remb_collector->feedback.remb.br_mantissa = bitrate / (1 << exp);
1526         sc->remb_collector->feedback.remb.br_exp = exp;
1527
1528         for (i = 0; i < AST_VECTOR_SIZE(&bridge_channel->stream_map.to_bridge); ++i) {
1529                 int bridge_num = AST_VECTOR_GET(&bridge_channel->stream_map.to_bridge, i);
1530
1531                 /* If this stream is not being provided to the bridge there can be no receivers of it
1532                  * so therefore no REMB reports.
1533                  */
1534                 if (bridge_num == -1) {
1535                         continue;
1536                 }
1537
1538                 /* We need to update the frame with this stream, or else it won't be
1539                  * properly routed. We don't use the actual channel stream identifier as
1540                  * the bridging core will do the translation from bridge stream identifier to
1541                  * channel stream identifier.
1542                  */
1543                 sc->remb_collector->frame.stream_num = bridge_num;
1544                 ast_bridge_channel_queue_frame(bridge_channel, &sc->remb_collector->frame);
1545         }
1546 }
1547
1548 static void gather_softmix_stats(struct softmix_stats *stats,
1549         const struct softmix_bridge_data *softmix_data,
1550         struct ast_bridge_channel *bridge_channel)
1551 {
1552         int channel_native_rate;
1553
1554         /* Gather stats about channel sample rates. */
1555         ast_channel_lock(bridge_channel->chan);
1556         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1557                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1558         ast_channel_unlock(bridge_channel->chan);
1559
1560         if (stats->highest_supported_rate < channel_native_rate) {
1561                 stats->highest_supported_rate = channel_native_rate;
1562         }
1563         if (stats->maximum_rate && stats->maximum_rate < channel_native_rate) {
1564                 stats->num_above_maximum_rate++;
1565         } else if (softmix_data->internal_rate < channel_native_rate) {
1566                 int i;
1567
1568                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1569                         if (stats->sample_rates[i] == channel_native_rate) {
1570                                 stats->num_channels[i]++;
1571                                 break;
1572                         } else if (!stats->sample_rates[i]) {
1573                                 stats->sample_rates[i] = channel_native_rate;
1574                                 stats->num_channels[i]++;
1575                                 break;
1576                         }
1577                 }
1578                 stats->num_above_internal_rate++;
1579         } else if (softmix_data->internal_rate == channel_native_rate) {
1580                 stats->num_at_internal_rate++;
1581         }
1582 }
1583
1584 /*!
1585  * \internal
1586  * \brief Analyse mixing statistics and change bridges internal rate
1587  * if necessary.
1588  *
1589  * \retval 0, no changes to internal rate
1590  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1591  */
1592 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1593                 struct softmix_bridge_data *softmix_data, int binaural_active)
1594 {
1595         int i;
1596
1597         if (binaural_active) {
1598                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1599         }
1600
1601         /*
1602          * Re-adjust the internal bridge sample rate if
1603          * 1. The bridge's internal sample rate is locked in at a sample
1604          *    rate other than the current sample rate being used.
1605          * 2. two or more channels support a higher sample rate
1606          * 3. no channels support the current sample rate or a higher rate
1607          */
1608         if (stats->locked_rate) {
1609                 /* if the rate is locked by the bridge, only update it if it differs
1610                  * from the current rate we are using. */
1611                 if (softmix_data->internal_rate != stats->locked_rate) {
1612                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1613                                 softmix_data->internal_rate, stats->locked_rate);
1614                         softmix_data->internal_rate = stats->locked_rate;
1615                         return 1;
1616                 }
1617         } else if (stats->num_above_maximum_rate) {
1618                 /* if the bridge has a maximum rate set and channels are above it only
1619                  * update if it differs from the current rate we are using. */
1620                 if (softmix_data->internal_rate != stats->maximum_rate) {
1621                         ast_debug(1, "Locking at new maximum rate.  Bridge changed from %u to %u.\n",
1622                                 softmix_data->internal_rate, stats->maximum_rate);
1623                         softmix_data->internal_rate = stats->maximum_rate;
1624                         return 1;
1625                 }
1626         } else if (stats->num_above_internal_rate >= 2) {
1627                 /* the highest rate is just used as a starting point */
1628                 unsigned int best_rate = stats->highest_supported_rate;
1629                 int best_index = -1;
1630
1631                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1632                         if (stats->num_channels[i]) {
1633                                 break;
1634                         }
1635                         if (2 <= stats->num_channels[i]) {
1636                                 /* Two or more channels support this rate. */
1637                                 if (best_index == -1
1638                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1639                                         /*
1640                                          * best_rate starts out being the first sample rate
1641                                          * greater than the internal sample rate that two or
1642                                          * more channels support.
1643                                          *
1644                                          * or
1645                                          *
1646                                          * There are multiple rates above the internal rate
1647                                          * and this rate is higher than the previous rate two
1648                                          * or more channels support.
1649                                          */
1650                                         best_rate = stats->sample_rates[i];
1651                                         best_index = i;
1652                                 }
1653                         } else if (best_index == -1) {
1654                                 /*
1655                                  * It is possible that multiple channels exist with native sample
1656                                  * rates above the internal sample rate, but none of those channels
1657                                  * have the same rate in common.  In this case, the lowest sample
1658                                  * rate among those channels is picked. Over time as additional
1659                                  * statistic runs are made the internal sample rate number will
1660                                  * adjust to the most optimal sample rate, but it may take multiple
1661                                  * iterations.
1662                                  */
1663                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1664                         }
1665                 }
1666
1667                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1668                         softmix_data->internal_rate, best_rate);
1669                 softmix_data->internal_rate = best_rate;
1670                 return 1;
1671         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1672                 /* In this case, the highest supported rate is actually lower than the internal rate */
1673                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1674                         softmix_data->internal_rate, stats->highest_supported_rate);
1675                 softmix_data->internal_rate = stats->highest_supported_rate;
1676                 return 1;
1677         }
1678         return 0;
1679 }
1680
1681 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1682                 unsigned int starting_num_entries, unsigned int binaural_active)
1683 {
1684         memset(mixing_array, 0, sizeof(*mixing_array));
1685         mixing_array->max_num_entries = starting_num_entries;
1686         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1687                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1688                 return -1;
1689         }
1690         if (binaural_active) {
1691                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1692                                 sizeof(struct convolve_channel_pair *)))) {
1693                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1694                         return -1;
1695                 }
1696         }
1697         return 0;
1698 }
1699
1700 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1701                 unsigned int binaural_active)
1702 {
1703         ast_free(mixing_array->buffers);
1704         if (binaural_active) {
1705                 ast_free(mixing_array->chan_pairs);
1706         }
1707 }
1708
1709 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1710                 unsigned int num_entries, unsigned int binaural_active)
1711 {
1712         int16_t **tmp;
1713
1714         /* give it some room to grow since memory is cheap but allocations can be expensive */
1715         mixing_array->max_num_entries = num_entries;
1716         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1717                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1718                 return -1;
1719         }
1720         mixing_array->buffers = tmp;
1721
1722         if (binaural_active) {
1723                 struct convolve_channel_pair **tmp2;
1724                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1725                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1726                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1727                         return -1;
1728                 }
1729                 mixing_array->chan_pairs = tmp2;
1730         }
1731         return 0;
1732 }
1733
1734 /*!
1735  * \brief Mixing loop.
1736  *
1737  * \retval 0 on success
1738  * \retval -1 on failure
1739  */
1740 static int softmix_mixing_loop(struct ast_bridge *bridge)
1741 {
1742         struct softmix_stats stats = { { 0 }, };
1743         struct softmix_mixing_array mixing_array;
1744         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1745         struct ast_timer *timer;
1746         struct softmix_translate_helper trans_helper;
1747         int16_t buf[MAX_DATALEN];
1748 #ifdef BINAURAL_RENDERING
1749         int16_t bin_buf[MAX_DATALEN];
1750         int16_t ann_buf[MAX_DATALEN];
1751 #endif
1752         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1753         int timingfd;
1754         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1755         unsigned int idx;
1756         unsigned int x;
1757         int res = -1;
1758
1759         timer = softmix_data->timer;
1760         timingfd = ast_timer_fd(timer);
1761         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1762         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1763
1764         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1765         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1766                         bridge->softmix.binaural_active)) {
1767                 goto softmix_cleanup;
1768         }
1769
1770         /*
1771          * XXX Softmix needs to use channel roles to determine who gets
1772          * what audio mixed.
1773          */
1774         while (!softmix_data->stop && bridge->num_active) {
1775                 struct ast_bridge_channel *bridge_channel;
1776                 int timeout = -1;
1777                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1778                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1779                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1780                 int remb_update = 0;
1781
1782                 if (softmix_datalen > MAX_DATALEN) {
1783                         /* This should NEVER happen, but if it does we need to know about it. Almost
1784                          * all the memcpys used during this process depend on this assumption.  Rather
1785                          * than checking this over and over again through out the code, this single
1786                          * verification is done on each iteration. */
1787                         ast_log(LOG_WARNING,
1788                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1789                                 bridge->uniqueid);
1790                         goto softmix_cleanup;
1791                 }
1792
1793                 /* Grow the mixing array buffer as participants are added. */
1794                 if (mixing_array.max_num_entries < bridge->num_channels
1795                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1796                                         bridge->softmix.binaural_active)) {
1797                         goto softmix_cleanup;
1798                 }
1799
1800                 /* init the number of buffers stored in the mixing array to 0.
1801                  * As buffers are added for mixing, this number is incremented. */
1802                 mixing_array.used_entries = 0;
1803
1804                 /* These variables help determine if a rate change is required */
1805                 if (!stat_iteration_counter) {
1806                         memset(&stats, 0, sizeof(stats));
1807                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1808                         stats.maximum_rate = bridge->softmix.maximum_sample_rate;
1809                 }
1810
1811                 /* If the sample rate has changed, update the translator helper */
1812                 if (update_all_rates) {
1813                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1814                 }
1815
1816 #ifdef BINAURAL_RENDERING
1817                 check_binaural_position_change(bridge, softmix_data);
1818 #endif
1819
1820                 /* If we need to do a REMB update to all video sources then do so */
1821                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU &&
1822                         bridge->softmix.video_mode.mode_data.sfu_data.remb_send_interval &&
1823                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_remb_update) > bridge->softmix.video_mode.mode_data.sfu_data.remb_send_interval) {
1824                         remb_update = 1;
1825                         softmix_data->last_remb_update = ast_tvnow();
1826                 }
1827
1828                 /* Go through pulling audio from each factory that has it available */
1829                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1830                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1831
1832                         if (!sc) {
1833                                 /* This channel failed to join successfully. */
1834                                 continue;
1835                         }
1836
1837                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1838                         if (update_all_rates) {
1839                                 set_softmix_bridge_data(softmix_data->internal_rate,
1840                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1841                         }
1842
1843                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1844                         if (!stat_iteration_counter) {
1845                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1846                         }
1847
1848                         /* if the channel is suspended, don't check for audio, but still gather stats */
1849                         if (bridge_channel->suspended) {
1850                                 continue;
1851                         }
1852
1853                         /* Try to get audio from the factory if available */
1854                         ast_mutex_lock(&sc->lock);
1855                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1856 #ifdef BINAURAL_RENDERING
1857                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1858                                                 ast_channel_name(bridge_channel->chan));
1859 #endif
1860                                 mixing_array.used_entries++;
1861                         }
1862                         if (remb_update) {
1863                                 remb_collect_report(bridge, bridge_channel, softmix_data, sc);
1864                         }
1865                         ast_mutex_unlock(&sc->lock);
1866                 }
1867
1868                 /* mix it like crazy (non binaural channels)*/
1869                 memset(buf, 0, softmix_datalen);
1870                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1871                         for (x = 0; x < softmix_samples; ++x) {
1872                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1873                         }
1874                 }
1875
1876 #ifdef BINAURAL_RENDERING
1877                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1878 #endif
1879
1880                 /* Next step go through removing the channel's own audio and creating a good frame... */
1881                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1882                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1883
1884                         if (!sc || bridge_channel->suspended) {
1885                                 /* This channel failed to join successfully or is suspended. */
1886                                 continue;
1887                         }
1888
1889                         ast_mutex_lock(&sc->lock);
1890
1891                         /* Make SLINEAR write frame from local buffer */
1892                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1893                                 "Replace softmix channel slin format");
1894 #ifdef BINAURAL_RENDERING
1895                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1896                                         && sc->binaural) {
1897                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1898                                                 softmix_samples, buf);
1899                         } else
1900 #endif
1901                         {
1902                                 sc->write_frame.datalen = softmix_datalen;
1903                                 sc->write_frame.samples = softmix_samples;
1904                                 memcpy(sc->final_buf, buf, softmix_datalen);
1905                         }
1906                         /* process the softmix channel's new write audio */
1907                         softmix_process_write_audio(&trans_helper,
1908                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1909                                         softmix_data->default_sample_size);
1910
1911                         ast_mutex_unlock(&sc->lock);
1912
1913                         /* A frame is now ready for the channel. */
1914                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1915
1916                         if (remb_update) {
1917                                 remb_send_report(bridge_channel, softmix_data, sc);
1918                         }
1919                 }
1920
1921                 if (remb_update) {
1922                         /* In case we are doing bridge level REMB reset the bitrate so we start fresh */
1923                         softmix_data->bitrate = 0;
1924                 }
1925
1926                 update_all_rates = 0;
1927                 if (!stat_iteration_counter) {
1928                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1929                                         bridge->softmix.binaural_active);
1930                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1931                 }
1932                 stat_iteration_counter--;
1933
1934                 ast_bridge_unlock(bridge);
1935                 /* cleanup any translation frame data from the previous mixing iteration. */
1936                 softmix_translate_helper_cleanup(&trans_helper);
1937                 /* Wait for the timing source to tell us to wake up and get things done */
1938                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1939                 if (ast_timer_ack(timer, 1) < 0) {
1940                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1941                                 bridge->uniqueid);
1942                         ast_bridge_lock(bridge);
1943                         goto softmix_cleanup;
1944                 }
1945                 ast_bridge_lock(bridge);
1946
1947                 /* make sure to detect mixing interval changes if they occur. */
1948                 if (bridge->softmix.internal_mixing_interval
1949                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1950                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1951                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1952                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1953                 }
1954         }
1955
1956         res = 0;
1957
1958 softmix_cleanup:
1959         softmix_translate_helper_destroy(&trans_helper);
1960         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1961         return res;
1962 }
1963
1964 /*!
1965  * \internal
1966  * \brief Mixing thread.
1967  * \since 12.0.0
1968  *
1969  * \note The thread does not have its own reference to the
1970  * bridge.  The lifetime of the thread is tied to the lifetime
1971  * of the mixing technology association with the bridge.
1972  */
1973 static void *softmix_mixing_thread(void *data)
1974 {
1975         struct softmix_bridge_data *softmix_data = data;
1976         struct ast_bridge *bridge = softmix_data->bridge;
1977
1978         ast_bridge_lock(bridge);
1979         if (bridge->callid) {
1980                 ast_callid_threadassoc_add(bridge->callid);
1981         }
1982
1983         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1984
1985         while (!softmix_data->stop) {
1986                 if (!bridge->num_active) {
1987                         /* Wait for something to happen to the bridge. */
1988                         ast_bridge_unlock(bridge);
1989                         ast_mutex_lock(&softmix_data->lock);
1990                         if (!softmix_data->stop) {
1991                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1992                         }
1993                         ast_mutex_unlock(&softmix_data->lock);
1994                         ast_bridge_lock(bridge);
1995                         continue;
1996                 }
1997
1998                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1999 #ifndef BINAURAL_RENDERING
2000                         ast_bridge_lock(bridge);
2001                         bridge->softmix.binaural_active = 0;
2002                         ast_bridge_unlock(bridge);
2003                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
2004                                         "compiled.\n", bridge->uniqueid);
2005 #else
2006                         /* Set and init binaural data if binaural is activated in the configuration. */
2007                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
2008                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
2009                                         softmix_data->internal_mixing_interval);
2010                         /* If init for binaural processing fails we will fall back to mono audio processing. */
2011                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
2012                                         == -1) {
2013                                 ast_bridge_lock(bridge);
2014                                 bridge->softmix.binaural_active = 0;
2015                                 ast_bridge_unlock(bridge);
2016                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
2017                                                 "binaural processing,  Will only process mono audio.\n",
2018                                                 bridge->uniqueid);
2019                         }
2020                         softmix_data->binaural_init = 1;
2021 #endif
2022                 }
2023
2024                 if (softmix_mixing_loop(bridge)) {
2025                         /*
2026                          * A mixing error occurred.  Sleep and try again later so we
2027                          * won't flood the logs.
2028                          */
2029                         ast_bridge_unlock(bridge);
2030                         sleep(1);
2031                         ast_bridge_lock(bridge);
2032                 }
2033         }
2034
2035         ast_bridge_unlock(bridge);
2036
2037         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
2038
2039         return NULL;
2040 }
2041
2042 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
2043 {
2044         if (softmix_data->timer) {
2045                 ast_timer_close(softmix_data->timer);
2046                 softmix_data->timer = NULL;
2047         }
2048         ast_mutex_destroy(&softmix_data->lock);
2049         ast_cond_destroy(&softmix_data->cond);
2050         AST_VECTOR_RESET(&softmix_data->remb_collectors, ao2_cleanup);
2051         AST_VECTOR_FREE(&softmix_data->remb_collectors);
2052         ast_free(softmix_data);
2053 }
2054
2055 /*! \brief Function called when a bridge is created */
2056 static int softmix_bridge_create(struct ast_bridge *bridge)
2057 {
2058         struct softmix_bridge_data *softmix_data;
2059
2060         softmix_data = ast_calloc(1, sizeof(*softmix_data));
2061         if (!softmix_data) {
2062                 return -1;
2063         }
2064         softmix_data->bridge = bridge;
2065         ast_mutex_init(&softmix_data->lock);
2066         ast_cond_init(&softmix_data->cond, NULL);
2067         softmix_data->timer = ast_timer_open();
2068         if (!softmix_data->timer) {
2069                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
2070                 softmix_bridge_data_destroy(softmix_data);
2071                 return -1;
2072         }
2073         /* start at minimum rate, let it grow from there */
2074         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
2075         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
2076
2077 #ifdef BINAURAL_RENDERING
2078         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
2079                         softmix_data->internal_mixing_interval);
2080 #endif
2081
2082         AST_VECTOR_INIT(&softmix_data->remb_collectors, 0);
2083
2084         bridge->tech_pvt = softmix_data;
2085
2086         /* Start the mixing thread. */
2087         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
2088                 softmix_data)) {
2089                 softmix_data->thread = AST_PTHREADT_NULL;
2090                 softmix_bridge_data_destroy(softmix_data);
2091                 bridge->tech_pvt = NULL;
2092                 return -1;
2093         }
2094
2095         return 0;
2096 }
2097
2098 /*!
2099  * \internal
2100  * \brief Request the softmix mixing thread stop.
2101  * \since 12.0.0
2102  *
2103  * \param bridge Which bridge is being stopped.
2104  *
2105  * \return Nothing
2106  */
2107 static void softmix_bridge_stop(struct ast_bridge *bridge)
2108 {
2109         struct softmix_bridge_data *softmix_data;
2110
2111         softmix_data = bridge->tech_pvt;
2112         if (!softmix_data) {
2113                 return;
2114         }
2115
2116         ast_mutex_lock(&softmix_data->lock);
2117         softmix_data->stop = 1;
2118         ast_mutex_unlock(&softmix_data->lock);
2119 }
2120
2121 /*! \brief Function called when a bridge is destroyed */
2122 static void softmix_bridge_destroy(struct ast_bridge *bridge)
2123 {
2124         struct softmix_bridge_data *softmix_data;
2125         pthread_t thread;
2126
2127         softmix_data = bridge->tech_pvt;
2128         if (!softmix_data) {
2129                 return;
2130         }
2131
2132         /* Stop the mixing thread. */
2133         ast_mutex_lock(&softmix_data->lock);
2134         softmix_data->stop = 1;
2135         ast_cond_signal(&softmix_data->cond);
2136         thread = softmix_data->thread;
2137         softmix_data->thread = AST_PTHREADT_NULL;
2138         ast_mutex_unlock(&softmix_data->lock);
2139         if (thread != AST_PTHREADT_NULL) {
2140                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
2141                 pthread_join(thread, NULL);
2142         }
2143 #ifdef BINAURAL_RENDERING
2144         free_convolve_data(&softmix_data->convolve);
2145 #endif
2146         softmix_bridge_data_destroy(softmix_data);
2147         bridge->tech_pvt = NULL;
2148 }
2149
2150 /*!
2151  * \brief Map a source stream to all of its destination streams.
2152  *
2153  * \param source_channel_name Name of channel where the source stream originates
2154  * \param bridge_stream_position The slot in the bridge where source video will come from
2155  * \param participants The bridge_channels in the bridge
2156  * \param source_channel_stream_position The position of the stream on the source channel
2157  */
2158 static void map_source_to_destinations(const char *source_channel_name,
2159         size_t bridge_stream_position, struct ast_bridge_channels_list *participants, int source_channel_stream_position)
2160 {
2161         struct ast_bridge_channel *participant;
2162
2163         AST_LIST_TRAVERSE(participants, participant, entry) {
2164                 int i;
2165                 struct ast_stream_topology *topology;
2166
2167                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
2168                         continue;
2169                 }
2170
2171                 ast_bridge_channel_lock(participant);
2172                 ast_channel_lock(participant->chan);
2173                 topology = ast_channel_get_stream_topology(participant->chan);
2174
2175                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2176                         struct ast_stream *stream;
2177
2178                         stream = ast_stream_topology_get_stream(topology, i);
2179                         if (is_video_dest(stream, source_channel_name, source_channel_stream_position)) {
2180                                 struct softmix_channel *sc = participant->tech_pvt;
2181
2182                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
2183                                 AST_VECTOR_APPEND(&sc->video_sources, bridge_stream_position);
2184                                 break;
2185                         }
2186                 }
2187                 ast_channel_unlock(participant->chan);
2188                 ast_bridge_channel_unlock(participant);
2189         }
2190 }
2191
2192 /*!
2193  * \brief Allocate a REMB collector
2194  *
2195  * \retval non-NULL success
2196  * \retval NULL failure
2197  */
2198 static struct softmix_remb_collector *remb_collector_alloc(void)
2199 {
2200         struct softmix_remb_collector *collector;
2201
2202         collector = ao2_alloc_options(sizeof(*collector), NULL, AO2_ALLOC_OPT_LOCK_NOLOCK);
2203         if (!collector) {
2204                 return NULL;
2205         }
2206
2207         collector->frame.frametype = AST_FRAME_RTCP;
2208         collector->frame.subclass.integer = AST_RTP_RTCP_PSFB;
2209         collector->feedback.fmt = AST_RTP_RTCP_FMT_REMB;
2210         collector->frame.data.ptr = &collector->feedback;
2211         collector->frame.datalen = sizeof(collector->feedback);
2212
2213         return collector;
2214 }
2215
2216 /*!
2217  * \brief Setup REMB collection for a particular bridge stream and channel.
2218  *
2219  * \param bridge The bridge
2220  * \param bridge_channel Channel that is collecting REMB information
2221  * \param bridge_stream_position The slot in the bridge where source video comes from
2222  */
2223 static void remb_enable_collection(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
2224         size_t bridge_stream_position)
2225 {
2226         struct softmix_channel *sc = bridge_channel->tech_pvt;
2227         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2228
2229         if (!sc->remb_collector) {
2230                 sc->remb_collector = remb_collector_alloc();
2231                 if (!sc->remb_collector) {
2232                         /* This is not fatal. Things will still continue to work but we won't
2233                          * produce a REMB report to the sender.
2234                          */
2235                         return;
2236                 }
2237         }
2238
2239         ao2_ref(sc->remb_collector, +1);
2240         if (AST_VECTOR_REPLACE(&softmix_data->remb_collectors, bridge_stream_position,
2241                 sc->remb_collector)) {
2242                 ao2_ref(sc->remb_collector, -1);
2243         }
2244 }
2245
2246 static void softmix_bridge_stream_sources_update(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
2247         struct softmix_channel *sc)
2248 {
2249         int index;
2250         struct ast_stream_topology *old_topology = sc->topology;
2251         struct ast_stream_topology *new_topology = ast_channel_get_stream_topology(bridge_channel->chan);
2252         int removed_streams[MAX(ast_stream_topology_get_count(sc->topology), ast_stream_topology_get_count(new_topology))];
2253         size_t removed_streams_count = 0;
2254         struct ast_stream_topology *added_streams;
2255         struct ast_bridge_channels_list *participants = &bridge->channels;
2256         struct ast_bridge_channel *participant;
2257
2258         added_streams = ast_stream_topology_alloc();
2259         if (!added_streams) {
2260                 return;
2261         }
2262
2263         /* We go through the old topology comparing it to the new topology to determine what streams
2264          * changed state. A state transition can result in the stream being considered a new source
2265          * (for example it was removed and is now present) or being removed (a stream became inactive).
2266          * Added streams are copied into a topology and added to each other participant while for
2267          * removed streams we merely store their position and mark them as removed later.
2268          */
2269         for (index = 0; index < ast_stream_topology_get_count(sc->topology) && index < ast_stream_topology_get_count(new_topology); ++index) {
2270                 struct ast_stream *old_stream = ast_stream_topology_get_stream(sc->topology, index);
2271                 struct ast_stream *new_stream = ast_stream_topology_get_stream(new_topology, index);
2272
2273                 /* Ignore all streams that don't carry video and streams that are strictly outgoing destination streams */
2274                 if ((ast_stream_get_type(old_stream) != AST_MEDIA_TYPE_VIDEO && ast_stream_get_type(new_stream) != AST_MEDIA_TYPE_VIDEO) ||
2275                         !strncmp(ast_stream_get_name(old_stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
2276                                 SOFTBRIDGE_VIDEO_DEST_LEN)) {
2277                         continue;
2278                 }
2279
2280                 if (ast_stream_get_type(old_stream) == AST_MEDIA_TYPE_VIDEO && ast_stream_get_type(new_stream) != AST_MEDIA_TYPE_VIDEO) {
2281                         /* If a stream renegotiates from video to non-video then we need to remove it as a source */
2282                         removed_streams[removed_streams_count++] = index;
2283                 } else if (ast_stream_get_type(old_stream) != AST_MEDIA_TYPE_VIDEO && ast_stream_get_type(new_stream) == AST_MEDIA_TYPE_VIDEO) {
2284                         if (ast_stream_get_state(new_stream) != AST_STREAM_STATE_REMOVED) {
2285                                 /* If a stream renegotiates from non-video to video in a non-removed state we need to add it as a source */
2286                                 if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2287                                                         bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2288                                                         new_stream, index)) {
2289                                         goto cleanup;
2290                                 }
2291                         }
2292                 } else if (ast_stream_get_state(old_stream) != AST_STREAM_STATE_REMOVED &&
2293                                 ast_stream_get_state(new_stream) != AST_STREAM_STATE_SENDRECV && ast_stream_get_state(new_stream) != AST_STREAM_STATE_RECVONLY) {
2294                         /* If a stream renegotiates and is removed then we remove it */
2295                         removed_streams[removed_streams_count++] = index;
2296                 } else if ((ast_stream_get_state(old_stream) == AST_STREAM_STATE_REMOVED || ast_stream_get_state(old_stream) == AST_STREAM_STATE_INACTIVE ||
2297                                 ast_stream_get_state(old_stream) == AST_STREAM_STATE_SENDONLY) &&
2298                                 ast_stream_get_state(new_stream) != AST_STREAM_STATE_INACTIVE && ast_stream_get_state(new_stream) != AST_STREAM_STATE_SENDONLY &&
2299                                 ast_stream_get_state(new_stream) != AST_STREAM_STATE_REMOVED) {
2300                         /* If a stream renegotiates and is added then we add it */
2301                         if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2302                                                 bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2303                                                 new_stream, index)) {
2304                                 goto cleanup;
2305                         }
2306                 }
2307         }
2308
2309         /* Any newly added streams that did not take the position of a removed stream
2310          * will be present at the end of the new topology. Since streams are never
2311          * removed from the topology but merely marked as removed we can pick up where we
2312          * left off when comparing the old and new topologies.
2313          */
2314         for (; index < ast_stream_topology_get_count(new_topology); ++index) {
2315                 struct ast_stream *stream = ast_stream_topology_get_stream(new_topology, index);
2316
2317                 if (!is_video_source(stream)) {
2318                         continue;
2319                 }
2320
2321                 if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2322                                         bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2323                                         stream, index)) {
2324                         goto cleanup;
2325                 }
2326         }
2327
2328         /*  We always update the stored topology if we can to reflect what is currently negotiated */
2329         sc->topology = ast_stream_topology_clone(new_topology);
2330         if (!sc->topology) {
2331                 sc->topology = old_topology;
2332         } else {
2333                 ast_stream_topology_free(old_topology);
2334         }
2335
2336         /* If there are no removed sources and no added sources we don't need to renegotiate the
2337          * other participants.
2338          */
2339         if (!removed_streams_count && !ast_stream_topology_get_count(added_streams)) {
2340                 goto cleanup;
2341         }
2342
2343         /* Go through each participant adding in the new streams and removing the old ones */
2344         AST_LIST_TRAVERSE(participants, participant, entry) {
2345                 if (participant == bridge_channel) {
2346                         continue;
2347                 }
2348
2349                 sc = participant->tech_pvt;
2350
2351                 /* We add in all the new streams first so that they do not take the place
2352                  * of any of our removed streams, allowing the remote side to reset the state
2353                  * for each removed stream. */
2354                 if (append_all_streams(sc->topology, added_streams)) {
2355                         goto cleanup;
2356                 }
2357
2358                 /* Then we go through and remove any ones that were removed */
2359                 for (index = 0; removed_streams_count && index < ast_stream_topology_get_count(sc->topology); ++index) {
2360                         struct ast_stream *stream = ast_stream_topology_get_stream(sc->topology, index);
2361                         int removed_stream;
2362
2363                         for (removed_stream = 0; removed_stream < removed_streams_count; ++removed_stream) {
2364                                 if (is_video_dest(stream, ast_channel_name(bridge_channel->chan), removed_streams[removed_stream])) {
2365                                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
2366                                 }
2367                         }
2368                 }
2369
2370                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
2371         }
2372
2373
2374 cleanup:
2375         ast_stream_topology_free(added_streams);
2376 }
2377
2378 /*!
2379  * \brief stream_topology_changed callback
2380  *
2381  * For most video modes, nothing beyond the ordinary is required.
2382  * For the SFU case, though, we need to completely remap the streams
2383  * in order to ensure video gets directed where it is expected to go.
2384  *
2385  * \param bridge The bridge
2386  * \param bridge_channel Channel whose topology has changed
2387  */
2388 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
2389 {
2390         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2391         struct softmix_channel *sc = bridge_channel->tech_pvt;
2392         struct ast_bridge_channel *participant;
2393         struct ast_vector_int media_types;
2394         int nths[AST_MEDIA_TYPE_END] = {0};
2395         int idx;
2396
2397         switch (bridge->softmix.video_mode.mode) {
2398         case AST_BRIDGE_VIDEO_MODE_NONE:
2399         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
2400         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
2401         default:
2402                 ast_bridge_channel_stream_map(bridge_channel);
2403                 return;
2404         case AST_BRIDGE_VIDEO_MODE_SFU:
2405                 break;
2406         }
2407
2408         ast_channel_lock(bridge_channel->chan);
2409         softmix_bridge_stream_sources_update(bridge, bridge_channel, sc);
2410         ast_channel_unlock(bridge_channel->chan);
2411
2412         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
2413
2414         /* The bridge stream identifiers may change, so reset the mapping for them.
2415          * When channels end up getting added back in they'll reuse their existing
2416          * collector and won't need to allocate a new one (unless they were just added).
2417          */
2418         for (idx = 0; idx < AST_VECTOR_SIZE(&softmix_data->remb_collectors); ++idx) {
2419                 ao2_cleanup(AST_VECTOR_GET(&softmix_data->remb_collectors, idx));
2420                 AST_VECTOR_REPLACE(&softmix_data->remb_collectors, idx, NULL);
2421         }
2422
2423         /* First traversal: re-initialize all of the participants' stream maps */
2424         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2425                 ast_bridge_channel_lock(participant);
2426
2427                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
2428                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
2429
2430                 sc = participant->tech_pvt;
2431                 AST_VECTOR_RESET(&sc->video_sources, AST_VECTOR_ELEM_CLEANUP_NOOP);
2432
2433                 ast_bridge_channel_unlock(participant);
2434         }
2435
2436         /* Second traversal: Map specific video channels from their source to their destinations.
2437          *
2438          * This is similar to what is done in ast_stream_topology_map(),
2439          * except that video channels are handled differently.  Each video
2440          * source has it's own unique index on the bridge.  This way, a
2441          * particular channel's source video can be distributed to the
2442          * appropriate destination streams on the other channels.
2443          */
2444         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2445                 int i;
2446                 struct ast_stream_topology *topology;
2447
2448                 ast_bridge_channel_lock(participant);
2449                 ast_channel_lock(participant->chan);
2450
2451                 topology = ao2_bump(ast_channel_get_stream_topology(participant->chan));
2452                 if (!topology) {
2453                         /* Oh, my, we are in trouble. */
2454                         ast_channel_unlock(participant->chan);
2455                         ast_bridge_channel_unlock(participant);
2456                         continue;
2457                 }
2458
2459                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2460                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
2461
2462                         if (is_video_source(stream)) {
2463                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
2464                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
2465                                 /*
2466                                  * There are cases where we need to bidirectionally send frames, such as for REMB reports
2467                                  * so we also map back to the channel.
2468                                  */
2469                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, i);
2470                                 remb_enable_collection(bridge, participant, AST_VECTOR_SIZE(&media_types) - 1);
2471                                 /*
2472                                  * Unlock the channel and participant to prevent
2473                                  * potential deadlock in map_source_to_destinations().
2474                                  */
2475                                 ast_channel_unlock(participant->chan);
2476                                 ast_bridge_channel_unlock(participant);
2477
2478                                 map_source_to_destinations(ast_channel_name(participant->chan),
2479                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels, i);
2480                                 ast_bridge_channel_lock(participant);
2481                                 ast_channel_lock(participant->chan);
2482                         } else if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO) {
2483                                 /* Video stream mapping occurs directly when a video source stream
2484                                  * is found on a channel. Video streams should otherwise remain
2485                                  * unmapped.
2486                                  */
2487                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
2488                         } else if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED) {
2489                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
2490                                  * be factored out in some way
2491                                  */
2492                                 enum ast_media_type type = ast_stream_get_type(stream);
2493                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
2494                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
2495
2496                                 if (index == -1) {
2497                                         AST_VECTOR_APPEND(&media_types, type);
2498                                         index = AST_VECTOR_SIZE(&media_types) - 1;
2499                                 }
2500
2501                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
2502                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
2503                         }
2504                 }
2505
2506                 ast_stream_topology_free(topology);
2507
2508                 ast_channel_unlock(participant->chan);
2509                 ast_bridge_channel_unlock(participant);
2510         }
2511
2512         AST_VECTOR_FREE(&media_types);
2513 }
2514
2515 static struct ast_bridge_technology softmix_bridge = {
2516         .name = "softmix",
2517         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
2518         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
2519         .create = softmix_bridge_create,
2520         .stop = softmix_bridge_stop,
2521         .destroy = softmix_bridge_destroy,
2522         .join = softmix_bridge_join,
2523         .leave = softmix_bridge_leave,
2524         .unsuspend = softmix_bridge_unsuspend,
2525         .write = softmix_bridge_write,
2526         .stream_topology_changed = softmix_bridge_stream_topology_changed,
2527 };
2528
2529 #ifdef TEST_FRAMEWORK
2530 struct stream_parameters {
2531         const char *name;
2532         const char *formats;
2533         enum ast_media_type type;
2534 };
2535
2536 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
2537 {
2538         struct ast_stream_topology *topology;
2539         size_t i;
2540
2541         topology = ast_stream_topology_alloc();
2542         if (!topology) {
2543                 return NULL;
2544         }
2545
2546         for (i = 0; i < num_streams; ++i) {
2547                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
2548                 struct ast_stream *stream;
2549
2550                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
2551                 if (!caps) {
2552                         goto fail;
2553                 }
2554                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
2555                         goto fail;
2556                 }
2557                 stream = ast_stream_alloc(params[i].name, params[i].type);
2558                 if (!stream) {
2559                         goto fail;
2560                 }
2561                 ast_stream_set_formats(stream, caps);
2562                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
2563                         ast_stream_free(stream);
2564                         goto fail;
2565                 }
2566         }
2567
2568         return topology;
2569
2570 fail:
2571         ast_stream_topology_free(topology);
2572         return NULL;
2573 }
2574
2575 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
2576         const struct stream_parameters *params)
2577 {
2578         const struct ast_format_cap *stream_caps;
2579         struct ast_format_cap *params_caps;
2580
2581         if (ast_stream_get_type(stream) != params->type) {
2582                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
2583                         ast_codec_media_type2str(params->type),
2584                         ast_codec_media_type2str(ast_stream_get_type(stream)));
2585                 return -1;
2586         }
2587         if (strcmp(ast_stream_get_name(stream), params->name)) {
2588                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
2589                         params->name, ast_stream_get_name(stream));
2590                 return -1;
2591         }
2592
2593         stream_caps = ast_stream_get_formats(stream);
2594         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
2595         if (!params_caps) {
2596                 ast_test_status_update(test, "Allocation error on capabilities\n");
2597                 return -1;
2598         }
2599         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
2600
2601         if (!ast_format_cap_identical(stream_caps, params_caps)) {
2602                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
2603                         ast_stream_get_name(stream));
2604                 ao2_cleanup(params_caps);
2605                 return -1;
2606         }
2607
2608         ao2_cleanup(params_caps);
2609         return 0;
2610 }
2611
2612 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
2613         const struct stream_parameters *params, size_t num_streams)
2614 {
2615         int i;
2616
2617         if (ast_stream_topology_get_count(topology) < num_streams) {
2618                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
2619                         ast_stream_topology_get_count(topology), num_streams);
2620                 return -1;
2621         }
2622
2623         for (i = 0; i < num_streams; ++i) {
2624                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
2625                         return -1;
2626                 }
2627         }
2628
2629         return 0;
2630 }
2631
2632 AST_TEST_DEFINE(sfu_append_source_streams)
2633 {
2634         enum ast_test_result_state res = AST_TEST_FAIL;
2635         static const struct stream_parameters bob_streams[] = {
2636                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2637                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2638         };
2639         static const struct stream_parameters alice_streams[] = {
2640                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
2641                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2642         };
2643         static const struct stream_parameters alice_dest_stream = {
2644                 "softbridge_dest_PJSIP/Bob-00000001_1", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2645         };
2646         static const struct stream_parameters bob_dest_stream = {
2647                 "softbridge_dest_PJSIP/Alice-00000000_1", "vp8", AST_MEDIA_TYPE_VIDEO,
2648         };
2649         struct ast_stream_topology *topology_alice = NULL;
2650         struct ast_stream_topology *topology_bob = NULL;
2651
2652         switch (cmd) {
2653         case TEST_INIT:
2654                 info->name = "sfu_append_source_streams";
2655                 info->category = "/bridges/bridge_softmix/";
2656                 info->summary = "Test appending of video streams";
2657                 info->description =
2658                         "This tests does stuff.";
2659                 return AST_TEST_NOT_RUN;
2660         case TEST_EXECUTE:
2661                 break;
2662         }
2663
2664         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2665         if (!topology_alice) {
2666                 goto end;
2667         }
2668
2669         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2670         if (!topology_bob) {
2671                 goto end;
2672         }
2673
2674         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", NULL, topology_bob)) {
2675                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2676                 goto end;
2677         }
2678
2679         if (ast_stream_topology_get_count(topology_alice) != 3) {
2680                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2681                         ast_stream_topology_get_count(topology_alice), 3);
2682                 goto end;
2683         }
2684
2685         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2686                 goto end;
2687         }
2688
2689         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2690                 goto end;
2691         }
2692
2693         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", NULL, topology_alice)) {
2694                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2695                 goto end;
2696         }
2697
2698         if (ast_stream_topology_get_count(topology_bob) != 3) {
2699                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2700                         ast_stream_topology_get_count(topology_bob), 3);
2701                 goto end;
2702         }
2703
2704         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2705                 goto end;
2706         }
2707
2708         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2709                 goto end;
2710         }
2711
2712         res = AST_TEST_PASS;
2713
2714 end:
2715         ast_stream_topology_free(topology_alice);
2716         ast_stream_topology_free(topology_bob);
2717         return res;
2718 }
2719
2720 AST_TEST_DEFINE(sfu_remove_destination_streams)
2721 {
2722         enum ast_test_result_state res = AST_TEST_FAIL;
2723         static const struct stream_parameters params[] = {
2724                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2725                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2726                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2727                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2728         };
2729         static const struct {
2730                 const char *channel_name;
2731                 int num_streams;
2732                 int params_index[4];
2733         } removal_results[] = {
2734                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2735                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2736                 { "", 4, { 0, 1, 2, 3 }, },
2737         };
2738         struct ast_stream_topology *orig = NULL;
2739         int i;
2740
2741         switch (cmd) {
2742         case TEST_INIT:
2743                 info->name = "sfu_remove_destination_streams";
2744                 info->category = "/bridges/bridge_softmix/";
2745                 info->summary = "Test removal of destination video streams";
2746                 info->description =
2747                         "This tests does stuff.";
2748                 return AST_TEST_NOT_RUN;
2749         case TEST_EXECUTE:
2750                 break;
2751         }
2752
2753         orig = build_topology(params, ARRAY_LEN(params));
2754         if (!orig) {
2755                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2756                 goto end;
2757         }
2758
2759         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2760                 int j;
2761
2762                 remove_destination_streams(orig, removal_results[i].channel_name);
2763
2764                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2765                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2766                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2767                         goto end;
2768                 }
2769
2770                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2771                         struct ast_stream *actual;
2772                         struct ast_stream *expected;
2773                         int orig_index;
2774
2775                         actual = ast_stream_topology_get_stream(orig, j);
2776
2777                         orig_index = removal_results[i].params_index[j];
2778                         expected = ast_stream_topology_get_stream(orig, orig_index);
2779
2780                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2781                                 ast_stream_get_formats(expected))) {
2782                                 struct ast_str *expected_str;
2783                                 struct ast_str *actual_str;
2784
2785                                 expected_str = ast_str_alloca(64);
2786                                 actual_str = ast_str_alloca(64);
2787
2788                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2789                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2790                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2791                                 goto end;
2792                         }
2793
2794                         if (is_video_dest(actual, removal_results[i].channel_name, -1) &&
2795                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2796                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2797                                 goto end;
2798                         }
2799                 }
2800         }
2801
2802         res = AST_TEST_PASS;
2803
2804 end:
2805         ast_stream_topology_free(orig);
2806         return res;
2807 }
2808
2809 #endif
2810
2811 static int unload_module(void)
2812 {
2813         ast_bridge_technology_unregister(&softmix_bridge);
2814         AST_TEST_UNREGISTER(sfu_append_source_streams);
2815         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2816         return 0;
2817 }
2818
2819 static int load_module(void)
2820 {
2821         if (ast_bridge_technology_register(&softmix_bridge)) {
2822                 unload_module();
2823                 return AST_MODULE_LOAD_DECLINE;
2824         }
2825         AST_TEST_REGISTER(sfu_append_source_streams);
2826         AST_TEST_REGISTER(sfu_remove_destination_streams);
2827         return AST_MODULE_LOAD_SUCCESS;
2828 }
2829
2830 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");