Merge "res_pjsip_mwi: potential double unref, and potential unwanted double link"
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include <math.h>
37
38 #include "asterisk/stream.h"
39 #include "asterisk/test.h"
40 #include "asterisk/vector.h"
41 #include "asterisk/message.h"
42 #include "bridge_softmix/include/bridge_softmix_internal.h"
43
44 /*! The minimum sample rate of the bridge. */
45 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
46
47 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
48 #define DEFAULT_SOFTMIX_INTERVAL 20
49
50 /*! \brief Size of the buffer used for sample manipulation */
51 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
52
53 /*! \brief Number of samples we are dealing with */
54 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
55
56 /*! \brief Number of mixing iterations to perform between gathering statistics. */
57 #define SOFTMIX_STAT_INTERVAL 100
58
59 /*!
60  * \brief Default time in ms of silence necessary to declare talking stopped by the bridge.
61  *
62  * \details
63  * This is the time at which a channel's own audio will stop getting
64  * mixed out of its own write audio stream because it is no longer talking.
65  */
66 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
67
68 /*! Default minimum average magnitude threshold to determine talking by the DSP. */
69 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
70
71 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
72 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
73 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
74
75 struct softmix_remb_collector {
76         /*! The frame which will be given to each source stream */
77         struct ast_frame frame;
78         /*! The REMB to send to the source which is collecting REMB reports */
79         struct ast_rtp_rtcp_feedback feedback;
80         /*! The maximum bitrate (A single precision floating point is big enough) */
81         float bitrate;
82 };
83
84 struct softmix_stats {
85         /*! Each index represents a sample rate used above the internal rate. */
86         unsigned int sample_rates[16];
87         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
88         unsigned int num_channels[16];
89         /*! The number of channels above the internal sample rate */
90         unsigned int num_above_internal_rate;
91         /*! The number of channels at the internal sample rate */
92         unsigned int num_at_internal_rate;
93         /*! The absolute highest sample rate preferred by any channel in the bridge */
94         unsigned int highest_supported_rate;
95         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
96         unsigned int locked_rate;
97 };
98
99 struct softmix_translate_helper_entry {
100         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
101                                                                   and re-init if it was usable. */
102         struct ast_format *dst_format; /*!< The destination format for this helper */
103         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
104         struct ast_frame *out_frame; /*!< The output frame from the last translation */
105         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
106 };
107
108 struct softmix_translate_helper {
109         struct ast_format *slin_src; /*!< the source format expected for all the translators */
110         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
111 };
112
113 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
114 {
115         struct softmix_translate_helper_entry *entry;
116         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
117                 return NULL;
118         }
119         entry->dst_format = ao2_bump(dst);
120         /* initialize this to one so that the first time through the cleanup code after
121            allocation it won't be removed from the entry list */
122         entry->num_times_requested = 1;
123         return entry;
124 }
125
126 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
127 {
128         ao2_cleanup(entry->dst_format);
129
130         if (entry->trans_pvt) {
131                 ast_translator_free_path(entry->trans_pvt);
132         }
133         if (entry->out_frame) {
134                 ast_frfree(entry->out_frame);
135         }
136         ast_free(entry);
137         return NULL;
138 }
139
140 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
141 {
142         memset(trans_helper, 0, sizeof(*trans_helper));
143         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
144 }
145
146 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
147 {
148         struct softmix_translate_helper_entry *entry;
149
150         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
151                 softmix_translate_helper_free_entry(entry);
152         }
153 }
154
155 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
156 {
157         struct softmix_translate_helper_entry *entry;
158
159         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
160         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
161                 if (entry->trans_pvt) {
162                         ast_translator_free_path(entry->trans_pvt);
163                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
164                                 AST_LIST_REMOVE_CURRENT(entry);
165                                 entry = softmix_translate_helper_free_entry(entry);
166                         }
167                 }
168         }
169         AST_LIST_TRAVERSE_SAFE_END;
170 }
171
172 /*!
173  * \internal
174  * \brief Get the next available audio on the softmix channel's read stream
175  * and determine if it should be mixed out or not on the write stream.
176  *
177  * \retval pointer to buffer containing the exact number of samples requested on success.
178  * \retval NULL if no samples are present
179  */
180 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
181 {
182         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
183                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
184                 sc->have_audio = 1;
185                 return sc->our_buf;
186         }
187         sc->have_audio = 0;
188         return NULL;
189 }
190
191 /*!
192  * \internal
193  * \brief Process a softmix channel's write audio
194  *
195  * \details This function will remove the channel's talking from its own audio if present and
196  * possibly even do the channel's write translation for it depending on how many other
197  * channels use the same write format.
198  */
199 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
200         struct ast_format *raw_write_fmt,
201         struct softmix_channel *sc, unsigned int default_sample_size)
202 {
203         struct softmix_translate_helper_entry *entry = NULL;
204         int i;
205
206         /* If we provided audio that was not determined to be silence,
207          * then take it out while in slinear format. */
208         if (sc->have_audio && sc->talking && !sc->binaural) {
209                 for (i = 0; i < sc->write_frame.samples; i++) {
210                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
211                 }
212                 /* check to see if any entries exist for the format. if not we'll want
213                    to remove it during cleanup */
214                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
215                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
216                                 ++entry->num_times_requested;
217                                 break;
218                         }
219                 }
220                 /* do not do any special write translate optimization if we had to make
221                  * a special mix for them to remove their own audio. */
222                 return;
223         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
224                 /*
225                  * Binaural audio requires special saturated substract since we have two
226                  * audio signals per channel now.
227                  */
228                 softmix_process_write_binaural_audio(sc, default_sample_size);
229                 return;
230         }
231
232         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
233            of translation paths and track the number of references for each type. Each one of the same
234            type should be able to use the same out_frame. Since the optimization is only necessary for
235            multiple channels (>=2) using the same codec make sure resources are allocated only when
236            needed and released when not (see also softmix_translate_helper_cleanup */
237         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
238                 if (sc->binaural != 0) {
239                         continue;
240                 }
241                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
242                         entry->num_times_requested++;
243                 } else {
244                         continue;
245                 }
246                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
247                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
248                 }
249                 if (entry->trans_pvt && !entry->out_frame) {
250                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
251                 }
252                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
253                                 && entry->out_frame->datalen < MAX_DATALEN) {
254                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
255                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
256                         sc->write_frame.datalen = entry->out_frame->datalen;
257                         sc->write_frame.samples = entry->out_frame->samples;
258                 }
259                 break;
260         }
261
262         /* add new entry into list if this format destination was not matched. */
263         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
264                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
265         }
266 }
267
268 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
269 {
270         struct softmix_translate_helper_entry *entry;
271
272         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
273                 /* if it hasn't been requested then remove it */
274                 if (!entry->num_times_requested) {
275                         AST_LIST_REMOVE_CURRENT(entry);
276                         softmix_translate_helper_free_entry(entry);
277                         continue;
278                 }
279
280                 if (entry->out_frame) {
281                         ast_frfree(entry->out_frame);
282                         entry->out_frame = NULL;
283                 }
284
285                 /* nothing is optimized for a single path reference, so there is
286                    no reason to continue to hold onto the codec */
287                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
288                         ast_translator_free_path(entry->trans_pvt);
289                         entry->trans_pvt = NULL;
290                 }
291
292                 /* for each iteration (a mixing run) in the bridge softmix thread the number
293                    of references to a given entry is recalculated, so reset the number of
294                    times requested */
295                 entry->num_times_requested = 0;
296         }
297         AST_LIST_TRAVERSE_SAFE_END;
298 }
299
300 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
301 {
302         struct softmix_channel *sc = bridge_channel->tech_pvt;
303         struct ast_format *slin_format;
304         int setup_fail;
305
306 #ifdef BINAURAL_RENDERING
307         if (interval != BINAURAL_MIXING_INTERVAL) {
308                 interval = BINAURAL_MIXING_INTERVAL;
309         }
310 #endif
311
312         /* The callers have already ensured that sc is never NULL. */
313         ast_assert(sc != NULL);
314
315         slin_format = ast_format_cache_get_slin_by_rate(rate);
316
317         ast_mutex_lock(&sc->lock);
318         if (reset) {
319                 ast_slinfactory_destroy(&sc->factory);
320                 ast_dsp_free(sc->dsp);
321         }
322
323         /* Setup write frame parameters */
324         sc->write_frame.frametype = AST_FRAME_VOICE;
325         /*
326          * NOTE: The write_frame format holds a reference because translation
327          * could be needed and the format changed to the translated format
328          * for the channel.  The translated format may not be a
329          * static cached format.
330          */
331         ao2_replace(sc->write_frame.subclass.format, slin_format);
332         sc->write_frame.data.ptr = sc->final_buf;
333         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
334         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
335
336         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
337         sc->rate = rate;
338
339         /* If the channel will contain binaural data we will set a identifier in the channel
340          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
341         if (set_binaural == 1) {
342                 sc->binaural = 1;
343         } else if (set_binaural == 0) {
344                 sc->binaural = 0;
345         }
346
347         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
348          * and doesn't have to be done if we just updating sample rates. */
349         if (binaural_pos_id != -1) {
350                 sc->binaural_pos = binaural_pos_id;
351         }
352         if (is_announcement != -1) {
353                 sc->is_announcement = is_announcement;
354         }
355
356         /*
357          * NOTE: The read_slin_format does not hold a reference because it
358          * will always be a signed linear format.
359          */
360         sc->read_slin_format = slin_format;
361
362         /* Setup smoother */
363         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
364
365         /* set new read and write formats on channel. */
366         ast_channel_lock(bridge_channel->chan);
367         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
368                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
369         ast_channel_unlock(bridge_channel->chan);
370
371         /* If channel contains binaural data we will set it here for the trans_pvt. */
372         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
373                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
374         } else if (set_binaural == 0) {
375                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
376         }
377
378         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
379         sc->dsp = ast_dsp_new_with_rate(rate);
380         if (setup_fail || !sc->dsp) {
381                 /* Bad news.  Could not setup the channel for softmix. */
382                 ast_mutex_unlock(&sc->lock);
383                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
384                 return;
385         }
386
387         /* we want to aggressively detect silence to avoid feedback */
388         if (bridge_channel->tech_args.talking_threshold) {
389                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
390         } else {
391                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
392         }
393
394         ast_mutex_unlock(&sc->lock);
395 }
396
397 /*!
398  * \internal
399  * \brief Poke the mixing thread in case it is waiting for an active channel.
400  * \since 12.0.0
401  *
402  * \param softmix_data Bridge mixing data.
403  *
404  * \return Nothing
405  */
406 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
407 {
408         ast_mutex_lock(&softmix_data->lock);
409         ast_cond_signal(&softmix_data->cond);
410         ast_mutex_unlock(&softmix_data->lock);
411 }
412
413 /*! \brief Function called when a channel is unsuspended from the bridge */
414 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
415 {
416 #ifdef BINAURAL_RENDERING
417         struct softmix_channel *sc = bridge_channel->tech_pvt;
418         if (sc->binaural) {
419                 /* Restore some usefull data if it was a binaural channel */
420                 struct ast_format *slin_format;
421
422                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
423                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
424         }
425 #endif
426         if (bridge->tech_pvt) {
427                 softmix_poke_thread(bridge->tech_pvt);
428         }
429 }
430
431 /*!
432  * \brief Determine if a stream is a video source stream.
433  *
434  * \param stream The stream to test
435  * \retval 1 The stream is a video source
436  * \retval 0 The stream is not a video source
437  */
438 static int is_video_source(const struct ast_stream *stream)
439 {
440         if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED
441                 && ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO
442                 && strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
443                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
444                 return 1;
445         }
446
447         return 0;
448 }
449
450 /*!
451  * \brief Determine if a stream is a video destination stream.
452  *
453  * A source channel name can be provided to narrow this to a destination stream
454  * for a particular source channel. Further, a source stream name can be provided
455  * to narrow this to a particular source stream's destination. However, empty strings
456  * can be provided to match any destination video stream, regardless of source channel
457  * or source stream.
458  *
459  * \param stream The stream to test
460  * \param source_channel_name The name of a source video channel to match
461  * \param source_stream_name The name of the source video stream to match
462  * \retval 1 The stream is a video destination stream
463  * \retval 0 The stream is not a video destination stream
464  */
465 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
466         const char *source_stream_name)
467 {
468         char *dest_video_name;
469         size_t dest_video_name_len;
470
471         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED
472                 || ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
473                 return 0;
474         }
475
476         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
477         if (!ast_strlen_zero(source_channel_name)) {
478                 dest_video_name_len += strlen(source_channel_name) + 1;
479                 if (!ast_strlen_zero(source_stream_name)) {
480                         dest_video_name_len += strlen(source_stream_name) + 1;
481                 }
482
483                 dest_video_name = ast_alloca(dest_video_name_len);
484                 if (!ast_strlen_zero(source_stream_name)) {
485                         /* We are looking for an exact stream name */
486                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
487                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
488                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
489                                 source_stream_name);
490                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
491                 }
492                 snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
493                         SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
494                         source_channel_name);
495         } else {
496                 dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
497         }
498
499         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
500 }
501
502 static int append_source_streams(struct ast_stream_topology *dest,
503         const char *channel_name, const char *sdp_label,
504         const struct ast_stream_topology *source)
505 {
506         int i;
507         const char *stream_identify;
508
509         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
510                 struct ast_stream *stream;
511                 struct ast_stream *stream_clone;
512                 char *stream_clone_name = NULL;
513
514                 stream = ast_stream_topology_get_stream(source, i);
515                 if (!is_video_source(stream)) {
516                         continue;
517                 }
518
519                 stream_identify = ast_stream_get_metadata(stream, "MSID:LABEL");
520                 if (!stream_identify) {
521                         stream_identify = ast_stream_get_name(stream);
522                 }
523
524                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
525                         channel_name, stream_identify) < 0) {
526                         return -1;
527                 }
528
529                 stream_clone = ast_stream_clone(stream, stream_clone_name);
530                 ast_free(stream_clone_name);
531                 if (!stream_clone) {
532                         return -1;
533                 }
534
535                 /* Sends an "a:label" attribute in the SDP for participant event correlation */
536                 if (!ast_strlen_zero(sdp_label)) {
537                         ast_stream_set_metadata(stream_clone, "SDP:LABEL", sdp_label);
538                 }
539
540                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
541                         ast_stream_free(stream_clone);
542                         return -1;
543                 }
544         }
545
546         return 0;
547 }
548
549 static int append_all_streams(struct ast_stream_topology *dest,
550         const struct ast_stream_topology *source)
551 {
552         int i;
553         int dest_index = 0;
554
555         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
556                 struct ast_stream *clone;
557                 int added = 0;
558
559                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
560                 if (!clone) {
561                         return -1;
562                 }
563
564                 /* If we can reuse an existing removed stream then do so */
565                 while (dest_index < ast_stream_topology_get_count(dest)) {
566                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
567
568                         dest_index++;
569
570                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
571                                 /* This cannot fail because dest_index - 1 is less than the
572                                  * current count in dest. */
573                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
574                                 added = 1;
575                                 break;
576                         }
577                 }
578
579                 /* If no removed stream exists that we took the place of append the stream */
580                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
581                         ast_stream_free(clone);
582                         return -1;
583                 }
584         }
585
586         return 0;
587 }
588
589 /*!
590  * \brief Issue channel stream topology change requests.
591  *
592  * When in SFU mode, each participant needs to be able to
593  * send video directly to other participants in the bridge.
594  * This means that all participants need to have their topologies
595  * updated. The joiner needs to have destination streams for
596  * all current participants, and the current participants need
597  * to have destinations streams added for the joiner's sources.
598  *
599  * \param joiner The channel that is joining the softmix bridge
600  * \param participants The current participants in the softmix bridge
601  */
602 static void sfu_topologies_on_join(struct ast_bridge *bridge,
603         struct ast_bridge_channel *joiner)
604 {
605         struct ast_stream_topology *joiner_video = NULL;
606         struct ast_bridge_channels_list *participants = &bridge->channels;
607         struct ast_bridge_channel *participant;
608         int res;
609         struct softmix_channel *sc;
610
611         joiner_video = ast_stream_topology_alloc();
612         if (!joiner_video) {
613                 return;
614         }
615
616         sc = joiner->tech_pvt;
617
618         ast_channel_lock(joiner->chan);
619         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan),
620                 bridge->softmix.send_sdp_label ? ast_channel_uniqueid(joiner->chan) : NULL,
621                 ast_channel_get_stream_topology(joiner->chan));
622         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
623         ast_channel_unlock(joiner->chan);
624
625         if (res || !sc->topology) {
626                 goto cleanup;
627         }
628
629         AST_LIST_TRAVERSE(participants, participant, entry) {
630                 if (participant == joiner) {
631                         continue;
632                 }
633                 ast_channel_lock(participant->chan);
634                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
635                         bridge->softmix.send_sdp_label ? ast_channel_uniqueid(participant->chan) : NULL,
636                         ast_channel_get_stream_topology(participant->chan));
637                 ast_channel_unlock(participant->chan);
638                 if (res) {
639                         goto cleanup;
640                 }
641         }
642
643         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
644
645         AST_LIST_TRAVERSE(participants, participant, entry) {
646                 if (participant == joiner) {
647                         continue;
648                 }
649
650                 sc = participant->tech_pvt;
651                 if (append_all_streams(sc->topology, joiner_video)) {
652                         goto cleanup;
653                 }
654                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
655         }
656
657 cleanup:
658         ast_stream_topology_free(joiner_video);
659 }
660
661 /*! \brief Function called when a channel is joined into the bridge */
662 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
663 {
664         struct softmix_channel *sc;
665         struct softmix_bridge_data *softmix_data;
666         int set_binaural = 0;
667         /*
668          * If false, the channel will be convolved, but since it is a non stereo channel, output
669          * will be mono.
670          */
671         int skip_binaural_output = 1;
672         int pos_id;
673         int is_announcement = 0;
674         int samplerate_change;
675
676         softmix_data = bridge->tech_pvt;
677         if (!softmix_data) {
678                 return -1;
679         }
680
681         /* Create a new softmix_channel structure and allocate various things on it */
682         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
683                 return -1;
684         }
685
686         samplerate_change = softmix_data->internal_rate;
687         pos_id = -1;
688         if (bridge->softmix.binaural_active) {
689                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
690                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
691                         if (set_binaural) {
692                                 softmix_data->internal_rate = samplerate_change;
693                         }
694                         skip_binaural_output = 0;
695                 } else {
696                         is_announcement = 1;
697                 }
698                 if (set_binaural) {
699                         softmix_data->convolve.binaural_active = 1;
700                 }
701                 if (!skip_binaural_output)      {
702                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
703                         if (pos_id == -1) {
704                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
705                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
706                                                 ast_channel_name(bridge_channel->chan));
707                                 ast_free(sc);
708                                 return -1;
709                         }
710                 }
711         }
712
713         /* Can't forget the lock */
714         ast_mutex_init(&sc->lock);
715
716         /* Can't forget to record our pvt structure within the bridged channel structure */
717         bridge_channel->tech_pvt = sc;
718
719         set_softmix_bridge_data(softmix_data->internal_rate,
720                 softmix_data->internal_mixing_interval
721                         ? softmix_data->internal_mixing_interval
722                         : DEFAULT_SOFTMIX_INTERVAL,
723                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
724
725         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
726                 sfu_topologies_on_join(bridge, bridge_channel);
727         }
728
729         softmix_poke_thread(softmix_data);
730         return 0;
731 }
732
733 static int remove_destination_streams(struct ast_stream_topology *topology,
734         const char *channel_name)
735 {
736         int i;
737         int stream_removed = 0;
738
739         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
740                 struct ast_stream *stream;
741
742                 stream = ast_stream_topology_get_stream(topology, i);
743
744                 if (is_video_dest(stream, channel_name, NULL)) {
745                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
746                         stream_removed = 1;
747                 }
748         }
749         return stream_removed;
750 }
751
752 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
753 {
754         struct ast_bridge_channel *participant;
755         struct softmix_channel *sc;
756
757         AST_LIST_TRAVERSE(participants, participant, entry) {
758                 sc = participant->tech_pvt;
759                 if (!remove_destination_streams(sc->topology, ast_channel_name(leaver->chan))) {
760                         continue;
761                 }
762                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
763         }
764
765         sc = leaver->tech_pvt;
766         if (remove_destination_streams(sc->topology, "")) {
767                 ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
768         }
769
770         return 0;
771 }
772
773 /*! \brief Function called when a channel leaves the bridge */
774 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
775 {
776         struct softmix_channel *sc;
777         struct softmix_bridge_data *softmix_data;
778
779         softmix_data = bridge->tech_pvt;
780         sc = bridge_channel->tech_pvt;
781         if (!sc) {
782                 return;
783         }
784
785         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
786                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
787         }
788
789         if (bridge->softmix.binaural_active) {
790                 if (sc->binaural) {
791                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
792                                         softmix_data->default_sample_size);
793                 }
794         }
795
796         bridge_channel->tech_pvt = NULL;
797
798         ast_stream_topology_free(sc->topology);
799
800         ao2_cleanup(sc->remb_collector);
801
802         AST_VECTOR_FREE(&sc->video_sources);
803
804         /* Drop mutex lock */
805         ast_mutex_destroy(&sc->lock);
806
807         /* Drop the factory */
808         ast_slinfactory_destroy(&sc->factory);
809
810         /* Drop any formats on the frames */
811         ao2_cleanup(sc->write_frame.subclass.format);
812
813         /* Drop the DSP */
814         ast_dsp_free(sc->dsp);
815
816         /* Eep! drop ourselves */
817         ast_free(sc);
818 }
819
820 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
821 {
822         struct ast_bridge_channel *cur;
823
824         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
825                 if (cur->suspended) {
826                         continue;
827                 }
828                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
829                         ast_bridge_channel_queue_frame(cur, frame);
830                         break;
831                 }
832         }
833 }
834
835 /*!
836  * \internal
837  * \brief Determine what to do with a video frame.
838  * \since 12.0.0
839  *
840  * \param bridge Which bridge is getting the frame
841  * \param bridge_channel Which channel is writing the frame.
842  * \param frame What is being written.
843  *
844  * \return Nothing
845  */
846 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
847 {
848         struct softmix_channel *sc;
849         int video_src_priority;
850
851         /* Determine if the video frame should be distributed or not */
852         switch (bridge->softmix.video_mode.mode) {
853         case AST_BRIDGE_VIDEO_MODE_NONE:
854                 break;
855         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
856                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
857                 if (video_src_priority == 1) {
858                         /* Pass to me and everyone else. */
859                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
860                 }
861                 break;
862         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
863                 sc = bridge_channel->tech_pvt;
864                 ast_mutex_lock(&sc->lock);
865                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
866                         sc->video_talker.energy_average,
867                         frame->subclass.frame_ending);
868                 ast_mutex_unlock(&sc->lock);
869                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
870                 if (video_src_priority == 1) {
871                         int num_src = ast_bridge_number_video_src(bridge);
872                         int echo = num_src > 1 ? 0 : 1;
873
874                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
875                 } else if (video_src_priority == 2) {
876                         softmix_pass_video_top_priority(bridge, frame);
877                 }
878                 break;
879         case AST_BRIDGE_VIDEO_MODE_SFU:
880                 /* Nothing special to do here, the bridge channel stream map will ensure the
881                  * video goes everywhere it needs to
882                  */
883                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
884                 break;
885         }
886 }
887
888 /*!
889  * \internal
890  * \brief Determine what to do with a voice frame.
891  * \since 12.0.0
892  *
893  * \param bridge Which bridge is getting the frame
894  * \param bridge_channel Which channel is writing the frame.
895  * \param frame What is being written.
896  *
897  * \return Nothing
898  */
899 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
900 {
901         struct softmix_channel *sc = bridge_channel->tech_pvt;
902         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
903         int silent = 0;
904         int totalsilence = 0;
905         int cur_energy = 0;
906         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
907                 bridge_channel->tech_args.silence_threshold :
908                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
909         /*
910          * If update_talking is set to 0 or 1, tell the bridge that the channel
911          * has started or stopped talking.
912          */
913         char update_talking = -1;
914
915         /* Write the frame into the conference */
916         ast_mutex_lock(&sc->lock);
917
918         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
919                 /*
920                  * The incoming frame is not the expected format.  Update
921                  * the channel's translation path to get us slinear from
922                  * the new format for the next frame.
923                  *
924                  * There is the possibility that this frame is an old slinear
925                  * rate frame that was in flight when the softmix bridge
926                  * changed rates.  If so it will self correct on subsequent
927                  * frames.
928                  */
929                 ast_channel_lock(bridge_channel->chan);
930                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
931                         ast_channel_name(bridge_channel->chan),
932                         ast_format_get_name(frame->subclass.format),
933                         ast_format_get_name(sc->read_slin_format));
934                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
935                         sc->read_slin_format);
936                 ast_channel_unlock(bridge_channel->chan);
937         }
938
939         /* The channel will be leaving soon if there is no dsp. */
940         if (sc->dsp) {
941                 silent = ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
942         }
943
944         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
945                 int cur_slot = sc->video_talker.energy_history_cur_slot;
946
947                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
948                 sc->video_talker.energy_accum += cur_energy;
949                 sc->video_talker.energy_history[cur_slot] = cur_energy;
950                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
951                 sc->video_talker.energy_history_cur_slot++;
952                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
953                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
954                 }
955         }
956
957         if (totalsilence < silence_threshold) {
958                 if (!sc->talking && !silent) {
959                         /* Tell the write process we have audio to be mixed out */
960                         sc->talking = 1;
961                         update_talking = 1;
962                 }
963         } else {
964                 if (sc->talking) {
965                         sc->talking = 0;
966                         update_talking = 0;
967                 }
968         }
969
970         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
971          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
972          * the audio by flushing the buffer before adding new audio in. */
973         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
974                 ast_slinfactory_flush(&sc->factory);
975         }
976
977         if (sc->talking || !bridge_channel->tech_args.drop_silence) {
978                 /* Add frame to the smoother for mixing with other channels. */
979                 ast_slinfactory_feed(&sc->factory, frame);
980         }
981
982         /* Alllll done */
983         ast_mutex_unlock(&sc->lock);
984
985         if (update_talking != -1) {
986                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
987         }
988 }
989
990 /*!
991  * \internal
992  * \brief Clear talking flag, stop contributing to mixing and notify handlers.
993  * \since 13.21.0, 15.4.0
994  *
995  * \param bridge_channel Which channel's talking to clear
996  *
997  * \return Nothing
998  */
999 static void clear_talking(struct ast_bridge_channel *bridge_channel)
1000 {
1001         struct softmix_channel *sc = bridge_channel->tech_pvt;
1002
1003         if (sc->talking) {
1004                 ast_mutex_lock(&sc->lock);
1005                 ast_slinfactory_flush(&sc->factory);
1006                 sc->talking = 0;
1007                 ast_mutex_unlock(&sc->lock);
1008
1009                 /* Notify that we are no longer talking. */
1010                 ast_bridge_channel_notify_talking(bridge_channel, 0);
1011         }
1012 }
1013
1014 /*!
1015  * \internal
1016  * \brief Check for voice status updates.
1017  * \since 13.20.0
1018  *
1019  * \param bridge Which bridge we are in
1020  * \param bridge_channel Which channel we are checking
1021  *
1022  * \return Nothing
1023  */
1024 static void softmix_bridge_check_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1025 {
1026         if (bridge_channel->features->mute) {
1027                 /*
1028                  * We were muted while we were talking.
1029                  *
1030                  * Immediately stop contributing to mixing
1031                  * and report no longer talking.
1032                  */
1033                 clear_talking(bridge_channel);
1034         }
1035 }
1036
1037 static int remove_all_original_streams(struct ast_stream_topology *dest,
1038         const struct ast_stream_topology *source,
1039         const struct ast_stream_topology *original)
1040 {
1041         int i;
1042
1043         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
1044                 struct ast_stream *stream;
1045                 int original_index;
1046
1047                 stream = ast_stream_topology_get_stream(source, i);
1048
1049                 /* Mark the existing stream as removed so we get a new one, this will get
1050                  * reused on a subsequent renegotiation.
1051                  */
1052                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
1053                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
1054
1055                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
1056                                 struct ast_stream *removed;
1057
1058                                 /* Since the participant is still going to be in the bridge we
1059                                  * change the name so that routing does not attempt to route video
1060                                  * to this stream.
1061                                  */
1062                                 removed = ast_stream_clone(stream, "removed");
1063                                 if (!removed) {
1064                                         return -1;
1065                                 }
1066
1067                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
1068
1069                                 /* The destination topology can only ever contain the same, or more,
1070                                  * streams than the original so this is safe.
1071                                  */
1072                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
1073                                         ast_stream_free(removed);
1074                                         return -1;
1075                                 }
1076
1077                                 break;
1078                         }
1079                 }
1080         }
1081
1082         return 0;
1083 }
1084
1085 static void sfu_topologies_on_source_change(struct ast_bridge *bridge,
1086         struct ast_bridge_channel *source)
1087 {
1088         struct ast_stream_topology *source_video = NULL;
1089         struct ast_bridge_channels_list *participants = &bridge->channels;
1090         struct ast_bridge_channel *participant;
1091         int res;
1092
1093         source_video = ast_stream_topology_alloc();
1094         if (!source_video) {
1095                 return;
1096         }
1097
1098         ast_channel_lock(source->chan);
1099         res = append_source_streams(source_video, ast_channel_name(source->chan),
1100                 bridge->softmix.send_sdp_label ? ast_channel_uniqueid(source->chan) : NULL,
1101                 ast_channel_get_stream_topology(source->chan));
1102         ast_channel_unlock(source->chan);
1103         if (res) {
1104                 goto cleanup;
1105         }
1106
1107         AST_LIST_TRAVERSE(participants, participant, entry) {
1108                 struct ast_stream_topology *original_topology;
1109                 struct softmix_channel *sc;
1110
1111                 if (participant == source) {
1112                         continue;
1113                 }
1114
1115                 sc = participant->tech_pvt;
1116
1117                 original_topology = ast_stream_topology_clone(sc->topology);
1118                 if (!original_topology) {
1119                         goto cleanup;
1120                 }
1121
1122                 /* We add all the source streams back in, if any removed streams are already present they will
1123                  * get used first followed by appending new ones.
1124                  */
1125                 if (append_all_streams(sc->topology, source_video)) {
1126                         ast_stream_topology_free(original_topology);
1127                         goto cleanup;
1128                 }
1129
1130                 /* And the original existing streams get marked as removed. This causes the remote side to see
1131                  * a new stream for the source streams.
1132                  */
1133                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1134                         ast_stream_topology_free(original_topology);
1135                         goto cleanup;
1136                 }
1137
1138                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1139                 ast_stream_topology_free(original_topology);
1140         }
1141
1142 cleanup:
1143         ast_stream_topology_free(source_video);
1144 }
1145
1146 /*!
1147  * \internal
1148  * \brief Determine what to do with a text frame.
1149  * \since 13.22.0
1150  * \since 15.5.0
1151  *
1152  * \param bridge Which bridge is getting the frame
1153  * \param bridge_channel Which channel is writing the frame.
1154  * \param frame What is being written.
1155  *
1156  * \return Nothing
1157  */
1158 static void softmix_bridge_write_text(struct ast_bridge *bridge,
1159         struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1160 {
1161         if (DEBUG_ATLEAST(1)) {
1162                 struct ast_msg_data *msg = frame->data.ptr;
1163                 char frame_type[64];
1164
1165                 ast_frame_type2str(frame->frametype, frame_type, sizeof(frame_type));
1166
1167                 if (frame->frametype == AST_FRAME_TEXT_DATA) {
1168                         ast_log(LOG_DEBUG, "Received %s frame from '%s:%s': %s\n", frame_type,
1169                                 ast_msg_data_get_attribute(msg, AST_MSG_DATA_ATTR_FROM),
1170                                 ast_channel_name(bridge_channel->chan),
1171                                 ast_msg_data_get_attribute(msg, AST_MSG_DATA_ATTR_BODY));
1172                 } else {
1173                         ast_log(LOG_DEBUG, "Received %s frame from '%s': %.*s\n", frame_type,
1174                                 ast_channel_name(bridge_channel->chan), frame->datalen,
1175                                 (char *)frame->data.ptr);
1176                 }
1177         }
1178
1179         ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1180 }
1181
1182 /*!
1183  * \internal
1184  * \brief Determine what to do with a control frame.
1185  * \since 12.0.0
1186  *
1187  * \param bridge Which bridge is getting the frame
1188  * \param bridge_channel Which channel is writing the frame.
1189  * \param frame What is being written.
1190  *
1191  * \retval 0 Frame accepted into the bridge.
1192  * \retval -1 Frame needs to be deferred.
1193  */
1194 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1195 {
1196         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1197
1198         /*
1199          * XXX Softmix needs to use channel roles to determine what to
1200          * do with control frames.
1201          */
1202
1203         switch (frame->subclass.integer) {
1204         case AST_CONTROL_HOLD:
1205                 /*
1206                  * Doing anything for holds in a conference bridge could be considered a bit
1207                  * odd. That being said, in most cases one would probably want the talking
1208                  * flag cleared when 'hold' is pressed by the remote endpoint, so go ahead
1209                  * and do that here. However, that is all we'll do. Meaning if for some reason
1210                  * the endpoint continues to send audio frames despite pressing 'hold' talking
1211                  * will once again be detected for that channel.
1212                  */
1213                 clear_talking(bridge_channel);
1214                 break;
1215         case AST_CONTROL_VIDUPDATE:
1216                 if (!bridge->softmix.video_mode.video_update_discard ||
1217                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1218                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1219                         softmix_data->last_video_update = ast_tvnow();
1220                 }
1221                 break;
1222         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1223                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1224                         sfu_topologies_on_source_change(bridge, bridge_channel);
1225                 }
1226                 break;
1227         default:
1228                 break;
1229         }
1230
1231         return 0;
1232 }
1233
1234 /*!
1235  * \internal
1236  * \brief Determine what to do with an RTCP frame.
1237  * \since 15.4.0
1238  *
1239  * \param bridge Which bridge is getting the frame
1240  * \param bridge_channel Which channel is writing the frame.
1241  * \param frame What is being written.
1242  */
1243 static void softmix_bridge_write_rtcp(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1244 {
1245         struct ast_rtp_rtcp_feedback *feedback = frame->data.ptr;
1246         struct softmix_channel *sc = bridge_channel->tech_pvt;
1247
1248         /* We only care about REMB reports right now. In the future we may be able to use sender or
1249          * receiver reports to further tweak things, but not yet.
1250          */
1251         if (frame->subclass.integer != AST_RTP_RTCP_PSFB || feedback->fmt != AST_RTP_RTCP_FMT_REMB ||
1252                 bridge->softmix.video_mode.mode != AST_BRIDGE_VIDEO_MODE_SFU ||
1253                 !bridge->softmix.video_mode.mode_data.sfu_data.remb_send_interval) {
1254                 return;
1255         }
1256
1257         /* REMB is the total estimated maximum bitrate across all streams within the session, so we store
1258          * only the latest report and use it everywhere.
1259          */
1260         ast_mutex_lock(&sc->lock);
1261         sc->remb = feedback->remb;
1262         ast_mutex_unlock(&sc->lock);
1263
1264         return;
1265 }
1266
1267 /*!
1268  * \internal
1269  * \brief Determine what to do with a frame written into the bridge.
1270  * \since 12.0.0
1271  *
1272  * \param bridge Which bridge is getting the frame
1273  * \param bridge_channel Which channel is writing the frame.
1274  * \param frame What is being written.
1275  *
1276  * \retval 0 Frame accepted into the bridge.
1277  * \retval -1 Frame needs to be deferred.
1278  *
1279  * \note On entry, bridge is already locked.
1280  */
1281 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1282 {
1283         int res = 0;
1284
1285         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1286                 /* "Accept" the frame and discard it. */
1287                 return 0;
1288         }
1289
1290         /*
1291          * XXX Softmix needs to use channel roles to determine who gets
1292          * what frame.  Possible roles: announcer, recorder, agent,
1293          * supervisor.
1294          */
1295         switch (frame->frametype) {
1296         case AST_FRAME_NULL:
1297                 /* "Accept" the frame and discard it. */
1298                 softmix_bridge_check_voice(bridge, bridge_channel);
1299                 break;
1300         case AST_FRAME_DTMF_BEGIN:
1301         case AST_FRAME_DTMF_END:
1302                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1303                 break;
1304         case AST_FRAME_VOICE:
1305                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1306                 break;
1307         case AST_FRAME_VIDEO:
1308                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1309                 break;
1310         case AST_FRAME_TEXT:
1311         case AST_FRAME_TEXT_DATA:
1312                 softmix_bridge_write_text(bridge, bridge_channel, frame);
1313                 break;
1314         case AST_FRAME_CONTROL:
1315                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1316                 break;
1317         case AST_FRAME_RTCP:
1318                 softmix_bridge_write_rtcp(bridge, bridge_channel, frame);
1319                 break;
1320         case AST_FRAME_BRIDGE_ACTION:
1321                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1322                 break;
1323         case AST_FRAME_BRIDGE_ACTION_SYNC:
1324                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1325                 ast_assert(0);
1326         default:
1327                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1328                 /* "Accept" the frame and discard it. */
1329                 break;
1330         }
1331
1332         return res;
1333 }
1334
1335 static void remb_collect_report_all(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
1336         float bitrate)
1337 {
1338         if (!softmix_data->bitrate) {
1339                 softmix_data->bitrate = bitrate;
1340                 return;
1341         }
1342
1343         switch (bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior) {
1344         case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE_ALL:
1345                 softmix_data->bitrate = (softmix_data->bitrate + bitrate) / 2;
1346                 break;
1347         case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST_ALL:
1348                 if (bitrate < softmix_data->bitrate) {
1349                         softmix_data->bitrate = bitrate;
1350                 }
1351                 break;
1352         case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST_ALL:
1353                 if (bitrate > softmix_data->bitrate) {
1354                         softmix_data->bitrate = bitrate;
1355                 }
1356                 break;
1357         case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE:
1358         case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST:
1359         case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST:
1360                 /* These will never actually get hit due to being handled by remb_collect_report below */
1361                 break;
1362         }
1363 }
1364
1365 static void remb_collect_report(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
1366         struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
1367 {
1368         int i;
1369         float bitrate;
1370
1371         /* If there are no video sources that we are a receiver of then we have noone to
1372          * report REMB to.
1373          */
1374         if (!AST_VECTOR_SIZE(&sc->video_sources)) {
1375                 return;
1376         }
1377
1378         /* We evenly divide the available maximum bitrate across the video sources
1379          * to this receiver so each source gets an equal slice.
1380          */
1381         bitrate = (sc->remb.br_mantissa << sc->remb.br_exp) / AST_VECTOR_SIZE(&sc->video_sources);
1382
1383         /* If this receiver has no bitrate yet ignore it */
1384         if (!bitrate) {
1385                 return;
1386         }
1387
1388         /* If we are using the "all" variants then we should use the bridge bitrate to store information */
1389         if (bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior == AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE_ALL ||
1390                 bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior == AST_BRIDGE_VIDEO_SFU_REMB_LOWEST_ALL ||
1391                 bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior == AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST_ALL) {
1392                 remb_collect_report_all(bridge, softmix_data, bitrate);
1393                 return;
1394         }
1395
1396         for (i = 0; i < AST_VECTOR_SIZE(&sc->video_sources); ++i) {
1397                 struct softmix_remb_collector *collector;
1398
1399                 /* The collector will always exist if a video source is in our list */
1400                 collector = AST_VECTOR_GET(&softmix_data->remb_collectors, AST_VECTOR_GET(&sc->video_sources, i));
1401
1402                 if (!collector->bitrate) {
1403                         collector->bitrate = bitrate;
1404                         continue;
1405                 }
1406
1407                 switch (bridge->softmix.video_mode.mode_data.sfu_data.remb_behavior) {
1408                 case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE:
1409                         collector->bitrate = (collector->bitrate + bitrate) / 2;
1410                         break;
1411                 case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST:
1412                         if (bitrate < collector->bitrate) {
1413                                 collector->bitrate = bitrate;
1414                         }
1415                         break;
1416                 case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST:
1417                         if (bitrate > collector->bitrate) {
1418                                 collector->bitrate = bitrate;
1419                         }
1420                         break;
1421                 case AST_BRIDGE_VIDEO_SFU_REMB_AVERAGE_ALL:
1422                 case AST_BRIDGE_VIDEO_SFU_REMB_LOWEST_ALL:
1423                 case AST_BRIDGE_VIDEO_SFU_REMB_HIGHEST_ALL:
1424                         /* These will never actually get hit due to being handled by remb_collect_report_all above */
1425                         break;
1426                 }
1427         }
1428
1429         /* After the report is integrated we reset this to 0 in case they stop producing
1430          * REMB reports.
1431          */
1432         sc->remb.br_mantissa = 0;
1433         sc->remb.br_exp = 0;
1434 }
1435
1436 static void remb_send_report(struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data,
1437         struct softmix_channel *sc)
1438 {
1439         float bitrate = softmix_data->bitrate;
1440         int i;
1441         int exp;
1442
1443         if (!sc->remb_collector) {
1444                 return;
1445         }
1446
1447         /* If there is no bridge level bitrate fall back to collector level */
1448         if (!bitrate) {
1449                 bitrate = sc->remb_collector->bitrate;
1450                 sc->remb_collector->bitrate = 0;
1451         }
1452
1453         /* We always do this calculation as even when the bitrate is zero the browser
1454          * still prefers it to be accurate instead of lying.
1455          *
1456          * The mantissa only has 18 bits available, so make sure it fits. Adjust the
1457          * value and exponent for those values that don't.
1458          *
1459          * For example given the following:
1460          *
1461          * bitrate = 123456789.0
1462          * frexp(bitrate, &exp);
1463          *
1464          * 'exp' should now equal 27 (number of bits needed to represent the value). Since
1465          * the mantissa must fit into an 18-bit unsigned integer, and the given bitrate is
1466          * too large to fit, we must subtract 18 from the exponent in order to get the
1467          * number of times the bitrate will fit into that size integer.
1468          *
1469          * exp -= 18;
1470          *
1471          * 'exp' is now equal to 9. Now we can get the mantissa that fits into an 18-bit
1472          * unsigned integer by dividing the bitrate by 2^exp:
1473          *
1474          * mantissa = 123456789.0 / 2^9
1475          *
1476          * This makes the final mantissa equal to 241126 (implicitly cast), which is less
1477          * than 262143 (the max value that can be put into an unsigned 18-bit integer).
1478          * So now we have the following:
1479          *
1480          * exp = 9;
1481          * mantissa = 241126;
1482          *
1483          * If we multiply that back we should come up with something close to the original
1484          * bit rate:
1485          *
1486          * 241126 * 2^9 = 123456512
1487          *
1488          * Precision is lost due to the nature of floating point values. Easier to why from
1489          * the binary:
1490          *
1491          * 241126 * 2^9 = 241126 << 9 = 111010110111100110 << 9 = 111010110111100110000000000
1492          *
1493          * Precision on the "lower" end is lost due to zeros being shifted in. This loss is
1494          * both expected and acceptable.
1495          */
1496         frexp(bitrate, &exp);
1497         exp = exp > 18 ? exp - 18 : 0;
1498
1499         sc->remb_collector->feedback.remb.br_mantissa = bitrate / (1 << exp);
1500         sc->remb_collector->feedback.remb.br_exp = exp;
1501
1502         for (i = 0; i < AST_VECTOR_SIZE(&bridge_channel->stream_map.to_bridge); ++i) {
1503                 int bridge_num = AST_VECTOR_GET(&bridge_channel->stream_map.to_bridge, i);
1504
1505                 /* If this stream is not being provided to the bridge there can be no receivers of it
1506                  * so therefore no REMB reports.
1507                  */
1508                 if (bridge_num == -1) {
1509                         continue;
1510                 }
1511
1512                 /* We need to update the frame with this stream, or else it won't be
1513                  * properly routed. We don't use the actual channel stream identifier as
1514                  * the bridging core will do the translation from bridge stream identifier to
1515                  * channel stream identifier.
1516                  */
1517                 sc->remb_collector->frame.stream_num = bridge_num;
1518                 ast_bridge_channel_queue_frame(bridge_channel, &sc->remb_collector->frame);
1519         }
1520 }
1521
1522 static void gather_softmix_stats(struct softmix_stats *stats,
1523         const struct softmix_bridge_data *softmix_data,
1524         struct ast_bridge_channel *bridge_channel)
1525 {
1526         int channel_native_rate;
1527
1528         /* Gather stats about channel sample rates. */
1529         ast_channel_lock(bridge_channel->chan);
1530         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1531                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1532         ast_channel_unlock(bridge_channel->chan);
1533
1534         if (stats->highest_supported_rate < channel_native_rate) {
1535                 stats->highest_supported_rate = channel_native_rate;
1536         }
1537         if (softmix_data->internal_rate < channel_native_rate) {
1538                 int i;
1539
1540                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1541                         if (stats->sample_rates[i] == channel_native_rate) {
1542                                 stats->num_channels[i]++;
1543                                 break;
1544                         } else if (!stats->sample_rates[i]) {
1545                                 stats->sample_rates[i] = channel_native_rate;
1546                                 stats->num_channels[i]++;
1547                                 break;
1548                         }
1549                 }
1550                 stats->num_above_internal_rate++;
1551         } else if (softmix_data->internal_rate == channel_native_rate) {
1552                 stats->num_at_internal_rate++;
1553         }
1554 }
1555
1556 /*!
1557  * \internal
1558  * \brief Analyse mixing statistics and change bridges internal rate
1559  * if necessary.
1560  *
1561  * \retval 0, no changes to internal rate
1562  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1563  */
1564 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1565                 struct softmix_bridge_data *softmix_data, int binaural_active)
1566 {
1567         int i;
1568
1569         if (binaural_active) {
1570                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1571         }
1572
1573         /*
1574          * Re-adjust the internal bridge sample rate if
1575          * 1. The bridge's internal sample rate is locked in at a sample
1576          *    rate other than the current sample rate being used.
1577          * 2. two or more channels support a higher sample rate
1578          * 3. no channels support the current sample rate or a higher rate
1579          */
1580         if (stats->locked_rate) {
1581                 /* if the rate is locked by the bridge, only update it if it differs
1582                  * from the current rate we are using. */
1583                 if (softmix_data->internal_rate != stats->locked_rate) {
1584                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1585                                 softmix_data->internal_rate, stats->locked_rate);
1586                         softmix_data->internal_rate = stats->locked_rate;
1587                         return 1;
1588                 }
1589         } else if (stats->num_above_internal_rate >= 2) {
1590                 /* the highest rate is just used as a starting point */
1591                 unsigned int best_rate = stats->highest_supported_rate;
1592                 int best_index = -1;
1593
1594                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1595                         if (stats->num_channels[i]) {
1596                                 break;
1597                         }
1598                         if (2 <= stats->num_channels[i]) {
1599                                 /* Two or more channels support this rate. */
1600                                 if (best_index == -1
1601                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1602                                         /*
1603                                          * best_rate starts out being the first sample rate
1604                                          * greater than the internal sample rate that two or
1605                                          * more channels support.
1606                                          *
1607                                          * or
1608                                          *
1609                                          * There are multiple rates above the internal rate
1610                                          * and this rate is higher than the previous rate two
1611                                          * or more channels support.
1612                                          */
1613                                         best_rate = stats->sample_rates[i];
1614                                         best_index = i;
1615                                 }
1616                         } else if (best_index == -1) {
1617                                 /*
1618                                  * It is possible that multiple channels exist with native sample
1619                                  * rates above the internal sample rate, but none of those channels
1620                                  * have the same rate in common.  In this case, the lowest sample
1621                                  * rate among those channels is picked. Over time as additional
1622                                  * statistic runs are made the internal sample rate number will
1623                                  * adjust to the most optimal sample rate, but it may take multiple
1624                                  * iterations.
1625                                  */
1626                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1627                         }
1628                 }
1629
1630                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1631                         softmix_data->internal_rate, best_rate);
1632                 softmix_data->internal_rate = best_rate;
1633                 return 1;
1634         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1635                 /* In this case, the highest supported rate is actually lower than the internal rate */
1636                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1637                         softmix_data->internal_rate, stats->highest_supported_rate);
1638                 softmix_data->internal_rate = stats->highest_supported_rate;
1639                 return 1;
1640         }
1641         return 0;
1642 }
1643
1644 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1645                 unsigned int starting_num_entries, unsigned int binaural_active)
1646 {
1647         memset(mixing_array, 0, sizeof(*mixing_array));
1648         mixing_array->max_num_entries = starting_num_entries;
1649         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1650                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1651                 return -1;
1652         }
1653         if (binaural_active) {
1654                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1655                                 sizeof(struct convolve_channel_pair *)))) {
1656                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1657                         return -1;
1658                 }
1659         }
1660         return 0;
1661 }
1662
1663 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1664                 unsigned int binaural_active)
1665 {
1666         ast_free(mixing_array->buffers);
1667         if (binaural_active) {
1668                 ast_free(mixing_array->chan_pairs);
1669         }
1670 }
1671
1672 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1673                 unsigned int num_entries, unsigned int binaural_active)
1674 {
1675         int16_t **tmp;
1676
1677         /* give it some room to grow since memory is cheap but allocations can be expensive */
1678         mixing_array->max_num_entries = num_entries;
1679         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1680                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1681                 return -1;
1682         }
1683         mixing_array->buffers = tmp;
1684
1685         if (binaural_active) {
1686                 struct convolve_channel_pair **tmp2;
1687                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1688                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1689                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1690                         return -1;
1691                 }
1692                 mixing_array->chan_pairs = tmp2;
1693         }
1694         return 0;
1695 }
1696
1697 /*!
1698  * \brief Mixing loop.
1699  *
1700  * \retval 0 on success
1701  * \retval -1 on failure
1702  */
1703 static int softmix_mixing_loop(struct ast_bridge *bridge)
1704 {
1705         struct softmix_stats stats = { { 0 }, };
1706         struct softmix_mixing_array mixing_array;
1707         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1708         struct ast_timer *timer;
1709         struct softmix_translate_helper trans_helper;
1710         int16_t buf[MAX_DATALEN];
1711 #ifdef BINAURAL_RENDERING
1712         int16_t bin_buf[MAX_DATALEN];
1713         int16_t ann_buf[MAX_DATALEN];
1714 #endif
1715         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1716         int timingfd;
1717         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1718         unsigned int idx;
1719         unsigned int x;
1720         int res = -1;
1721
1722         timer = softmix_data->timer;
1723         timingfd = ast_timer_fd(timer);
1724         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1725         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1726
1727         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1728         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1729                         bridge->softmix.binaural_active)) {
1730                 goto softmix_cleanup;
1731         }
1732
1733         /*
1734          * XXX Softmix needs to use channel roles to determine who gets
1735          * what audio mixed.
1736          */
1737         while (!softmix_data->stop && bridge->num_active) {
1738                 struct ast_bridge_channel *bridge_channel;
1739                 int timeout = -1;
1740                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1741                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1742                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1743                 int remb_update = 0;
1744
1745                 if (softmix_datalen > MAX_DATALEN) {
1746                         /* This should NEVER happen, but if it does we need to know about it. Almost
1747                          * all the memcpys used during this process depend on this assumption.  Rather
1748                          * than checking this over and over again through out the code, this single
1749                          * verification is done on each iteration. */
1750                         ast_log(LOG_WARNING,
1751                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1752                                 bridge->uniqueid);
1753                         goto softmix_cleanup;
1754                 }
1755
1756                 /* Grow the mixing array buffer as participants are added. */
1757                 if (mixing_array.max_num_entries < bridge->num_channels
1758                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1759                                         bridge->softmix.binaural_active)) {
1760                         goto softmix_cleanup;
1761                 }
1762
1763                 /* init the number of buffers stored in the mixing array to 0.
1764                  * As buffers are added for mixing, this number is incremented. */
1765                 mixing_array.used_entries = 0;
1766
1767                 /* These variables help determine if a rate change is required */
1768                 if (!stat_iteration_counter) {
1769                         memset(&stats, 0, sizeof(stats));
1770                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1771                 }
1772
1773                 /* If the sample rate has changed, update the translator helper */
1774                 if (update_all_rates) {
1775                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1776                 }
1777
1778 #ifdef BINAURAL_RENDERING
1779                 check_binaural_position_change(bridge, softmix_data);
1780 #endif
1781
1782                 /* If we need to do a REMB update to all video sources then do so */
1783                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU &&
1784                         bridge->softmix.video_mode.mode_data.sfu_data.remb_send_interval &&
1785                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_remb_update) > bridge->softmix.video_mode.mode_data.sfu_data.remb_send_interval) {
1786                         remb_update = 1;
1787                         softmix_data->last_remb_update = ast_tvnow();
1788                 }
1789
1790                 /* Go through pulling audio from each factory that has it available */
1791                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1792                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1793
1794                         if (!sc) {
1795                                 /* This channel failed to join successfully. */
1796                                 continue;
1797                         }
1798
1799                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1800                         if (update_all_rates) {
1801                                 set_softmix_bridge_data(softmix_data->internal_rate,
1802                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1803                         }
1804
1805                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1806                         if (!stat_iteration_counter) {
1807                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1808                         }
1809
1810                         /* if the channel is suspended, don't check for audio, but still gather stats */
1811                         if (bridge_channel->suspended) {
1812                                 continue;
1813                         }
1814
1815                         /* Try to get audio from the factory if available */
1816                         ast_mutex_lock(&sc->lock);
1817                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1818 #ifdef BINAURAL_RENDERING
1819                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1820                                                 ast_channel_name(bridge_channel->chan));
1821 #endif
1822                                 mixing_array.used_entries++;
1823                         }
1824                         if (remb_update) {
1825                                 remb_collect_report(bridge, bridge_channel, softmix_data, sc);
1826                         }
1827                         ast_mutex_unlock(&sc->lock);
1828                 }
1829
1830                 /* mix it like crazy (non binaural channels)*/
1831                 memset(buf, 0, softmix_datalen);
1832                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1833                         for (x = 0; x < softmix_samples; ++x) {
1834                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1835                         }
1836                 }
1837
1838 #ifdef BINAURAL_RENDERING
1839                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1840 #endif
1841
1842                 /* Next step go through removing the channel's own audio and creating a good frame... */
1843                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1844                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1845
1846                         if (!sc || bridge_channel->suspended) {
1847                                 /* This channel failed to join successfully or is suspended. */
1848                                 continue;
1849                         }
1850
1851                         ast_mutex_lock(&sc->lock);
1852
1853                         /* Make SLINEAR write frame from local buffer */
1854                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1855                                 "Replace softmix channel slin format");
1856 #ifdef BINAURAL_RENDERING
1857                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1858                                         && sc->binaural) {
1859                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1860                                                 softmix_samples, buf);
1861                         } else
1862 #endif
1863                         {
1864                                 sc->write_frame.datalen = softmix_datalen;
1865                                 sc->write_frame.samples = softmix_samples;
1866                                 memcpy(sc->final_buf, buf, softmix_datalen);
1867                         }
1868                         /* process the softmix channel's new write audio */
1869                         softmix_process_write_audio(&trans_helper,
1870                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1871                                         softmix_data->default_sample_size);
1872
1873                         ast_mutex_unlock(&sc->lock);
1874
1875                         /* A frame is now ready for the channel. */
1876                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1877
1878                         if (remb_update) {
1879                                 remb_send_report(bridge_channel, softmix_data, sc);
1880                         }
1881                 }
1882
1883                 if (remb_update) {
1884                         /* In case we are doing bridge level REMB reset the bitrate so we start fresh */
1885                         softmix_data->bitrate = 0;
1886                 }
1887
1888                 update_all_rates = 0;
1889                 if (!stat_iteration_counter) {
1890                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1891                                         bridge->softmix.binaural_active);
1892                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1893                 }
1894                 stat_iteration_counter--;
1895
1896                 ast_bridge_unlock(bridge);
1897                 /* cleanup any translation frame data from the previous mixing iteration. */
1898                 softmix_translate_helper_cleanup(&trans_helper);
1899                 /* Wait for the timing source to tell us to wake up and get things done */
1900                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1901                 if (ast_timer_ack(timer, 1) < 0) {
1902                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1903                                 bridge->uniqueid);
1904                         ast_bridge_lock(bridge);
1905                         goto softmix_cleanup;
1906                 }
1907                 ast_bridge_lock(bridge);
1908
1909                 /* make sure to detect mixing interval changes if they occur. */
1910                 if (bridge->softmix.internal_mixing_interval
1911                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1912                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1913                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1914                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1915                 }
1916         }
1917
1918         res = 0;
1919
1920 softmix_cleanup:
1921         softmix_translate_helper_destroy(&trans_helper);
1922         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1923         return res;
1924 }
1925
1926 /*!
1927  * \internal
1928  * \brief Mixing thread.
1929  * \since 12.0.0
1930  *
1931  * \note The thread does not have its own reference to the
1932  * bridge.  The lifetime of the thread is tied to the lifetime
1933  * of the mixing technology association with the bridge.
1934  */
1935 static void *softmix_mixing_thread(void *data)
1936 {
1937         struct softmix_bridge_data *softmix_data = data;
1938         struct ast_bridge *bridge = softmix_data->bridge;
1939
1940         ast_bridge_lock(bridge);
1941         if (bridge->callid) {
1942                 ast_callid_threadassoc_add(bridge->callid);
1943         }
1944
1945         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1946
1947         while (!softmix_data->stop) {
1948                 if (!bridge->num_active) {
1949                         /* Wait for something to happen to the bridge. */
1950                         ast_bridge_unlock(bridge);
1951                         ast_mutex_lock(&softmix_data->lock);
1952                         if (!softmix_data->stop) {
1953                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1954                         }
1955                         ast_mutex_unlock(&softmix_data->lock);
1956                         ast_bridge_lock(bridge);
1957                         continue;
1958                 }
1959
1960                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1961 #ifndef BINAURAL_RENDERING
1962                         ast_bridge_lock(bridge);
1963                         bridge->softmix.binaural_active = 0;
1964                         ast_bridge_unlock(bridge);
1965                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1966                                         "compiled.\n", bridge->uniqueid);
1967 #else
1968                         /* Set and init binaural data if binaural is activated in the configuration. */
1969                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1970                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1971                                         softmix_data->internal_mixing_interval);
1972                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1973                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1974                                         == -1) {
1975                                 ast_bridge_lock(bridge);
1976                                 bridge->softmix.binaural_active = 0;
1977                                 ast_bridge_unlock(bridge);
1978                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1979                                                 "binaural processing,  Will only process mono audio.\n",
1980                                                 bridge->uniqueid);
1981                         }
1982                         softmix_data->binaural_init = 1;
1983 #endif
1984                 }
1985
1986                 if (softmix_mixing_loop(bridge)) {
1987                         /*
1988                          * A mixing error occurred.  Sleep and try again later so we
1989                          * won't flood the logs.
1990                          */
1991                         ast_bridge_unlock(bridge);
1992                         sleep(1);
1993                         ast_bridge_lock(bridge);
1994                 }
1995         }
1996
1997         ast_bridge_unlock(bridge);
1998
1999         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
2000
2001         return NULL;
2002 }
2003
2004 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
2005 {
2006         if (softmix_data->timer) {
2007                 ast_timer_close(softmix_data->timer);
2008                 softmix_data->timer = NULL;
2009         }
2010         ast_mutex_destroy(&softmix_data->lock);
2011         ast_cond_destroy(&softmix_data->cond);
2012         AST_VECTOR_RESET(&softmix_data->remb_collectors, ao2_cleanup);
2013         AST_VECTOR_FREE(&softmix_data->remb_collectors);
2014         ast_free(softmix_data);
2015 }
2016
2017 /*! \brief Function called when a bridge is created */
2018 static int softmix_bridge_create(struct ast_bridge *bridge)
2019 {
2020         struct softmix_bridge_data *softmix_data;
2021
2022         softmix_data = ast_calloc(1, sizeof(*softmix_data));
2023         if (!softmix_data) {
2024                 return -1;
2025         }
2026         softmix_data->bridge = bridge;
2027         ast_mutex_init(&softmix_data->lock);
2028         ast_cond_init(&softmix_data->cond, NULL);
2029         softmix_data->timer = ast_timer_open();
2030         if (!softmix_data->timer) {
2031                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
2032                 softmix_bridge_data_destroy(softmix_data);
2033                 return -1;
2034         }
2035         /* start at minimum rate, let it grow from there */
2036         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
2037         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
2038
2039 #ifdef BINAURAL_RENDERING
2040         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
2041                         softmix_data->internal_mixing_interval);
2042 #endif
2043
2044         AST_VECTOR_INIT(&softmix_data->remb_collectors, 0);
2045
2046         bridge->tech_pvt = softmix_data;
2047
2048         /* Start the mixing thread. */
2049         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
2050                 softmix_data)) {
2051                 softmix_data->thread = AST_PTHREADT_NULL;
2052                 softmix_bridge_data_destroy(softmix_data);
2053                 bridge->tech_pvt = NULL;
2054                 return -1;
2055         }
2056
2057         return 0;
2058 }
2059
2060 /*!
2061  * \internal
2062  * \brief Request the softmix mixing thread stop.
2063  * \since 12.0.0
2064  *
2065  * \param bridge Which bridge is being stopped.
2066  *
2067  * \return Nothing
2068  */
2069 static void softmix_bridge_stop(struct ast_bridge *bridge)
2070 {
2071         struct softmix_bridge_data *softmix_data;
2072
2073         softmix_data = bridge->tech_pvt;
2074         if (!softmix_data) {
2075                 return;
2076         }
2077
2078         ast_mutex_lock(&softmix_data->lock);
2079         softmix_data->stop = 1;
2080         ast_mutex_unlock(&softmix_data->lock);
2081 }
2082
2083 /*! \brief Function called when a bridge is destroyed */
2084 static void softmix_bridge_destroy(struct ast_bridge *bridge)
2085 {
2086         struct softmix_bridge_data *softmix_data;
2087         pthread_t thread;
2088
2089         softmix_data = bridge->tech_pvt;
2090         if (!softmix_data) {
2091                 return;
2092         }
2093
2094         /* Stop the mixing thread. */
2095         ast_mutex_lock(&softmix_data->lock);
2096         softmix_data->stop = 1;
2097         ast_cond_signal(&softmix_data->cond);
2098         thread = softmix_data->thread;
2099         softmix_data->thread = AST_PTHREADT_NULL;
2100         ast_mutex_unlock(&softmix_data->lock);
2101         if (thread != AST_PTHREADT_NULL) {
2102                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
2103                 pthread_join(thread, NULL);
2104         }
2105 #ifdef BINAURAL_RENDERING
2106         free_convolve_data(&softmix_data->convolve);
2107 #endif
2108         softmix_bridge_data_destroy(softmix_data);
2109         bridge->tech_pvt = NULL;
2110 }
2111
2112 /*!
2113  * \brief Map a source stream to all of its destination streams.
2114  *
2115  * \param source_stream_name Name of the source stream
2116  * \param source_channel_name Name of channel where the source stream originates
2117  * \param bridge_stream_position The slot in the bridge where source video will come from
2118  * \param participants The bridge_channels in the bridge
2119  */
2120 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
2121         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
2122 {
2123         struct ast_bridge_channel *participant;
2124
2125         AST_LIST_TRAVERSE(participants, participant, entry) {
2126                 int i;
2127                 struct ast_stream_topology *topology;
2128
2129                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
2130                         continue;
2131                 }
2132
2133                 ast_bridge_channel_lock(participant);
2134                 ast_channel_lock(participant->chan);
2135                 topology = ast_channel_get_stream_topology(participant->chan);
2136
2137                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2138                         struct ast_stream *stream;
2139
2140                         stream = ast_stream_topology_get_stream(topology, i);
2141                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
2142                                 struct softmix_channel *sc = participant->tech_pvt;
2143
2144                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
2145                                 AST_VECTOR_APPEND(&sc->video_sources, bridge_stream_position);
2146                                 break;
2147                         }
2148                 }
2149                 ast_channel_unlock(participant->chan);
2150                 ast_bridge_channel_unlock(participant);
2151         }
2152 }
2153
2154 /*!
2155  * \brief Allocate a REMB collector
2156  *
2157  * \retval non-NULL success
2158  * \retval NULL failure
2159  */
2160 static struct softmix_remb_collector *remb_collector_alloc(void)
2161 {
2162         struct softmix_remb_collector *collector;
2163
2164         collector = ao2_alloc_options(sizeof(*collector), NULL, AO2_ALLOC_OPT_LOCK_NOLOCK);
2165         if (!collector) {
2166                 return NULL;
2167         }
2168
2169         collector->frame.frametype = AST_FRAME_RTCP;
2170         collector->frame.subclass.integer = AST_RTP_RTCP_PSFB;
2171         collector->feedback.fmt = AST_RTP_RTCP_FMT_REMB;
2172         collector->frame.data.ptr = &collector->feedback;
2173         collector->frame.datalen = sizeof(collector->feedback);
2174
2175         return collector;
2176 }
2177
2178 /*!
2179  * \brief Setup REMB collection for a particular bridge stream and channel.
2180  *
2181  * \param bridge The bridge
2182  * \param bridge_channel Channel that is collecting REMB information
2183  * \param bridge_stream_position The slot in the bridge where source video comes from
2184  */
2185 static void remb_enable_collection(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
2186         size_t bridge_stream_position)
2187 {
2188         struct softmix_channel *sc = bridge_channel->tech_pvt;
2189         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2190
2191         if (!sc->remb_collector) {
2192                 sc->remb_collector = remb_collector_alloc();
2193                 if (!sc->remb_collector) {
2194                         /* This is not fatal. Things will still continue to work but we won't
2195                          * produce a REMB report to the sender.
2196                          */
2197                         return;
2198                 }
2199         }
2200
2201         ao2_ref(sc->remb_collector, +1);
2202         if (AST_VECTOR_REPLACE(&softmix_data->remb_collectors, bridge_stream_position,
2203                 sc->remb_collector)) {
2204                 ao2_ref(sc->remb_collector, -1);
2205         }
2206 }
2207
2208 /*!
2209  * \brief stream_topology_changed callback
2210  *
2211  * For most video modes, nothing beyond the ordinary is required.
2212  * For the SFU case, though, we need to completely remap the streams
2213  * in order to ensure video gets directed where it is expected to go.
2214  *
2215  * \param bridge The bridge
2216  * \param bridge_channel Channel whose topology has changed
2217  */
2218 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
2219 {
2220         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2221         struct softmix_channel *sc;
2222         struct ast_bridge_channel *participant;
2223         struct ast_vector_int media_types;
2224         int nths[AST_MEDIA_TYPE_END] = {0};
2225         int idx;
2226
2227         switch (bridge->softmix.video_mode.mode) {
2228         case AST_BRIDGE_VIDEO_MODE_NONE:
2229         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
2230         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
2231         default:
2232                 ast_bridge_channel_stream_map(bridge_channel);
2233                 return;
2234         case AST_BRIDGE_VIDEO_MODE_SFU:
2235                 break;
2236         }
2237
2238         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
2239
2240         /* The bridge stream identifiers may change, so reset the mapping for them.
2241          * When channels end up getting added back in they'll reuse their existing
2242          * collector and won't need to allocate a new one (unless they were just added).
2243          */
2244         for (idx = 0; idx < AST_VECTOR_SIZE(&softmix_data->remb_collectors); ++idx) {
2245                 ao2_cleanup(AST_VECTOR_GET(&softmix_data->remb_collectors, idx));
2246                 AST_VECTOR_REPLACE(&softmix_data->remb_collectors, idx, NULL);
2247         }
2248
2249         /* First traversal: re-initialize all of the participants' stream maps */
2250         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2251                 ast_bridge_channel_lock(participant);
2252
2253                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
2254                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
2255
2256                 sc = participant->tech_pvt;
2257                 AST_VECTOR_RESET(&sc->video_sources, AST_VECTOR_ELEM_CLEANUP_NOOP);
2258
2259                 ast_bridge_channel_unlock(participant);
2260         }
2261
2262         /* Second traversal: Map specific video channels from their source to their destinations.
2263          *
2264          * This is similar to what is done in ast_stream_topology_map(),
2265          * except that video channels are handled differently.  Each video
2266          * source has it's own unique index on the bridge.  This way, a
2267          * particular channel's source video can be distributed to the
2268          * appropriate destination streams on the other channels.
2269          */
2270         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2271                 int i;
2272                 struct ast_stream_topology *topology;
2273
2274                 ast_bridge_channel_lock(participant);
2275                 ast_channel_lock(participant->chan);
2276
2277                 topology = ao2_bump(ast_channel_get_stream_topology(participant->chan));
2278                 if (!topology) {
2279                         /* Oh, my, we are in trouble. */
2280                         ast_channel_unlock(participant->chan);
2281                         ast_bridge_channel_unlock(participant);
2282                         continue;
2283                 }
2284
2285                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2286                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
2287                         const char *stream_identify;
2288
2289                         if (is_video_source(stream)) {
2290                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
2291                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
2292                                 /*
2293                                  * There are cases where we need to bidirectionally send frames, such as for REMB reports
2294                                  * so we also map back to the channel.
2295                                  */
2296                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, i);
2297                                 remb_enable_collection(bridge, participant, AST_VECTOR_SIZE(&media_types) - 1);
2298                                 /*
2299                                  * Unlock the channel and participant to prevent
2300                                  * potential deadlock in map_source_to_destinations().
2301                                  */
2302                                 ast_channel_unlock(participant->chan);
2303                                 ast_bridge_channel_unlock(participant);
2304
2305                                 stream_identify = ast_stream_get_metadata(stream, "MSID:LABEL");
2306                                 if (!stream_identify) {
2307                                         stream_identify = ast_stream_get_name(stream);
2308                                 }
2309                                 map_source_to_destinations(stream_identify, ast_channel_name(participant->chan),
2310                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
2311                                 ast_bridge_channel_lock(participant);
2312                                 ast_channel_lock(participant->chan);
2313                         } else if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO) {
2314                                 /* Video stream mapping occurs directly when a video source stream
2315                                  * is found on a channel. Video streams should otherwise remain
2316                                  * unmapped.
2317                                  */
2318                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
2319                         } else if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED) {
2320                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
2321                                  * be factored out in some way
2322                                  */
2323                                 enum ast_media_type type = ast_stream_get_type(stream);
2324                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
2325                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
2326
2327                                 if (index == -1) {
2328                                         AST_VECTOR_APPEND(&media_types, type);
2329                                         index = AST_VECTOR_SIZE(&media_types) - 1;
2330                                 }
2331
2332                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
2333                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
2334                         }
2335                 }
2336
2337                 ast_stream_topology_free(topology);
2338
2339                 ast_channel_unlock(participant->chan);
2340                 ast_bridge_channel_unlock(participant);
2341         }
2342
2343         AST_VECTOR_FREE(&media_types);
2344 }
2345
2346 static struct ast_bridge_technology softmix_bridge = {
2347         .name = "softmix",
2348         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
2349         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
2350         .create = softmix_bridge_create,
2351         .stop = softmix_bridge_stop,
2352         .destroy = softmix_bridge_destroy,
2353         .join = softmix_bridge_join,
2354         .leave = softmix_bridge_leave,
2355         .unsuspend = softmix_bridge_unsuspend,
2356         .write = softmix_bridge_write,
2357         .stream_topology_changed = softmix_bridge_stream_topology_changed,
2358 };
2359
2360 #ifdef TEST_FRAMEWORK
2361 struct stream_parameters {
2362         const char *name;
2363         const char *formats;
2364         enum ast_media_type type;
2365 };
2366
2367 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
2368 {
2369         struct ast_stream_topology *topology;
2370         size_t i;
2371
2372         topology = ast_stream_topology_alloc();
2373         if (!topology) {
2374                 return NULL;
2375         }
2376
2377         for (i = 0; i < num_streams; ++i) {
2378                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
2379                 struct ast_stream *stream;
2380
2381                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
2382                 if (!caps) {
2383                         goto fail;
2384                 }
2385                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
2386                         goto fail;
2387                 }
2388                 stream = ast_stream_alloc(params[i].name, params[i].type);
2389                 if (!stream) {
2390                         goto fail;
2391                 }
2392                 ast_stream_set_formats(stream, caps);
2393                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
2394                         ast_stream_free(stream);
2395                         goto fail;
2396                 }
2397         }
2398
2399         return topology;
2400
2401 fail:
2402         ast_stream_topology_free(topology);
2403         return NULL;
2404 }
2405
2406 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
2407         const struct stream_parameters *params)
2408 {
2409         struct ast_format_cap *stream_caps;
2410         struct ast_format_cap *params_caps;
2411
2412         if (ast_stream_get_type(stream) != params->type) {
2413                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
2414                         ast_codec_media_type2str(params->type),
2415                         ast_codec_media_type2str(ast_stream_get_type(stream)));
2416                 return -1;
2417         }
2418         if (strcmp(ast_stream_get_name(stream), params->name)) {
2419                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
2420                         params->name, ast_stream_get_name(stream));
2421                 return -1;
2422         }
2423
2424         stream_caps = ast_stream_get_formats(stream);
2425         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
2426         if (!params_caps) {
2427                 ast_test_status_update(test, "Allocation error on capabilities\n");
2428                 return -1;
2429         }
2430         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
2431
2432         if (!ast_format_cap_identical(stream_caps, params_caps)) {
2433                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
2434                         ast_stream_get_name(stream));
2435                 ao2_cleanup(params_caps);
2436                 return -1;
2437         }
2438
2439         ao2_cleanup(params_caps);
2440         return 0;
2441 }
2442
2443 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
2444         const struct stream_parameters *params, size_t num_streams)
2445 {
2446         int i;
2447
2448         if (ast_stream_topology_get_count(topology) < num_streams) {
2449                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
2450                         ast_stream_topology_get_count(topology), num_streams);
2451                 return -1;
2452         }
2453
2454         for (i = 0; i < num_streams; ++i) {
2455                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
2456                         return -1;
2457                 }
2458         }
2459
2460         return 0;
2461 }
2462
2463 AST_TEST_DEFINE(sfu_append_source_streams)
2464 {
2465         enum ast_test_result_state res = AST_TEST_FAIL;
2466         static const struct stream_parameters bob_streams[] = {
2467                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2468                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2469         };
2470         static const struct stream_parameters alice_streams[] = {
2471                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
2472                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2473         };
2474         static const struct stream_parameters alice_dest_stream = {
2475                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2476         };
2477         static const struct stream_parameters bob_dest_stream = {
2478                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "vp8", AST_MEDIA_TYPE_VIDEO,
2479         };
2480         struct ast_stream_topology *topology_alice = NULL;
2481         struct ast_stream_topology *topology_bob = NULL;
2482
2483         switch (cmd) {
2484         case TEST_INIT:
2485                 info->name = "sfu_append_source_streams";
2486                 info->category = "/bridges/bridge_softmix/";
2487                 info->summary = "Test appending of video streams";
2488                 info->description =
2489                         "This tests does stuff.";
2490                 return AST_TEST_NOT_RUN;
2491         case TEST_EXECUTE:
2492                 break;
2493         }
2494
2495         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2496         if (!topology_alice) {
2497                 goto end;
2498         }
2499
2500         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2501         if (!topology_bob) {
2502                 goto end;
2503         }
2504
2505         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", NULL, topology_bob)) {
2506                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2507                 goto end;
2508         }
2509
2510         if (ast_stream_topology_get_count(topology_alice) != 3) {
2511                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2512                         ast_stream_topology_get_count(topology_alice), 3);
2513                 goto end;
2514         }
2515
2516         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2517                 goto end;
2518         }
2519
2520         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2521                 goto end;
2522         }
2523
2524         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", NULL, topology_alice)) {
2525                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2526                 goto end;
2527         }
2528
2529         if (ast_stream_topology_get_count(topology_bob) != 3) {
2530                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2531                         ast_stream_topology_get_count(topology_bob), 3);
2532                 goto end;
2533         }
2534
2535         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2536                 goto end;
2537         }
2538
2539         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2540                 goto end;
2541         }
2542
2543         res = AST_TEST_PASS;
2544
2545 end:
2546         ast_stream_topology_free(topology_alice);
2547         ast_stream_topology_free(topology_bob);
2548         return res;
2549 }
2550
2551 AST_TEST_DEFINE(sfu_remove_destination_streams)
2552 {
2553         enum ast_test_result_state res = AST_TEST_FAIL;
2554         static const struct stream_parameters params[] = {
2555                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2556                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2557                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2558                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2559         };
2560         static const struct {
2561                 const char *channel_name;
2562                 int num_streams;
2563                 int params_index[4];
2564         } removal_results[] = {
2565                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2566                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2567                 { "", 4, { 0, 1, 2, 3 }, },
2568         };
2569         struct ast_stream_topology *orig = NULL;
2570         int i;
2571
2572         switch (cmd) {
2573         case TEST_INIT:
2574                 info->name = "sfu_remove_destination_streams";
2575                 info->category = "/bridges/bridge_softmix/";
2576                 info->summary = "Test removal of destination video streams";
2577                 info->description =
2578                         "This tests does stuff.";
2579                 return AST_TEST_NOT_RUN;
2580         case TEST_EXECUTE:
2581                 break;
2582         }
2583
2584         orig = build_topology(params, ARRAY_LEN(params));
2585         if (!orig) {
2586                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2587                 goto end;
2588         }
2589
2590         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2591                 int j;
2592
2593                 remove_destination_streams(orig, removal_results[i].channel_name);
2594
2595                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2596                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2597                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2598                         goto end;
2599                 }
2600
2601                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2602                         struct ast_stream *actual;
2603                         struct ast_stream *expected;
2604                         int orig_index;
2605
2606                         actual = ast_stream_topology_get_stream(orig, j);
2607
2608                         orig_index = removal_results[i].params_index[j];
2609                         expected = ast_stream_topology_get_stream(orig, orig_index);
2610
2611                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2612                                 ast_stream_get_formats(expected))) {
2613                                 struct ast_str *expected_str;
2614                                 struct ast_str *actual_str;
2615
2616                                 expected_str = ast_str_alloca(64);
2617                                 actual_str = ast_str_alloca(64);
2618
2619                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2620                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2621                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2622                                 goto end;
2623                         }
2624
2625                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2626                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2627                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2628                                 goto end;
2629                         }
2630                 }
2631         }
2632
2633         res = AST_TEST_PASS;
2634
2635 end:
2636         ast_stream_topology_free(orig);
2637         return res;
2638 }
2639
2640 #endif
2641
2642 static int unload_module(void)
2643 {
2644         ast_bridge_technology_unregister(&softmix_bridge);
2645         AST_TEST_UNREGISTER(sfu_append_source_streams);
2646         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2647         return 0;
2648 }
2649
2650 static int load_module(void)
2651 {
2652         if (ast_bridge_technology_register(&softmix_bridge)) {
2653                 unload_module();
2654                 return AST_MODULE_LOAD_DECLINE;
2655         }
2656         AST_TEST_REGISTER(sfu_append_source_streams);
2657         AST_TEST_REGISTER(sfu_remove_destination_streams);
2658         return AST_MODULE_LOAD_SUCCESS;
2659 }
2660
2661 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");