Merge "bridge_softmix.c: Don't match dead streams."
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                                                   and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED
422                 && ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO
423                 && strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
424                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
425                 return 1;
426         }
427
428         return 0;
429 }
430
431 /*!
432  * \brief Determine if a stream is a video destination stream.
433  *
434  * A source channel name can be provided to narrow this to a destination stream
435  * for a particular source channel. Further, a source stream name can be provided
436  * to narrow this to a particular source stream's destination. However, empty strings
437  * can be provided to match any destination video stream, regardless of source channel
438  * or source stream.
439  *
440  * \param stream The stream to test
441  * \param source_channel_name The name of a source video channel to match
442  * \param source_stream_name The name of the source video stream to match
443  * \retval 1 The stream is a video destination stream
444  * \retval 0 The stream is not a video destination stream
445  */
446 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
447         const char *source_stream_name)
448 {
449         char *dest_video_name;
450         size_t dest_video_name_len;
451
452         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED
453                 || ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
454                 return 0;
455         }
456
457         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
458         if (!ast_strlen_zero(source_channel_name)) {
459                 dest_video_name_len += strlen(source_channel_name) + 1;
460                 if (!ast_strlen_zero(source_stream_name)) {
461                         dest_video_name_len += strlen(source_stream_name) + 1;
462                 }
463
464                 dest_video_name = ast_alloca(dest_video_name_len);
465                 if (!ast_strlen_zero(source_stream_name)) {
466                         /* We are looking for an exact stream name */
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 }
473                 snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                         SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                         source_channel_name);
476         } else {
477                 dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
478         }
479
480         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
481 }
482
483 static int append_source_streams(struct ast_stream_topology *dest,
484         const char *channel_name,
485         const struct ast_stream_topology *source)
486 {
487         int i;
488
489         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
490                 struct ast_stream *stream;
491                 struct ast_stream *stream_clone;
492                 char *stream_clone_name = NULL;
493
494                 stream = ast_stream_topology_get_stream(source, i);
495                 if (!is_video_source(stream)) {
496                         continue;
497                 }
498
499                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
500                         channel_name, ast_stream_get_name(stream)) < 0) {
501                         return -1;
502                 }
503
504                 stream_clone = ast_stream_clone(stream, stream_clone_name);
505                 ast_free(stream_clone_name);
506                 if (!stream_clone) {
507                         return -1;
508                 }
509                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
510                         ast_stream_free(stream_clone);
511                         return -1;
512                 }
513         }
514
515         return 0;
516 }
517
518 static int append_all_streams(struct ast_stream_topology *dest,
519         const struct ast_stream_topology *source)
520 {
521         int i;
522         int dest_index = 0;
523
524         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
525                 struct ast_stream *clone;
526                 int added = 0;
527
528                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
529                 if (!clone) {
530                         return -1;
531                 }
532
533                 /* If we can reuse an existing removed stream then do so */
534                 while (dest_index < ast_stream_topology_get_count(dest)) {
535                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
536
537                         dest_index++;
538
539                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
540                                 /* This cannot fail because dest_index - 1 is less than the
541                                  * current count in dest. */
542                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
543                                 added = 1;
544                                 break;
545                         }
546                 }
547
548                 /* If no removed stream exists that we took the place of append the stream */
549                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
550                         ast_stream_free(clone);
551                         return -1;
552                 }
553         }
554
555         return 0;
556 }
557
558 /*!
559  * \brief Issue channel stream topology change requests.
560  *
561  * When in SFU mode, each participant needs to be able to
562  * send video directly to other participants in the bridge.
563  * This means that all participants need to have their topologies
564  * updated. The joiner needs to have destination streams for
565  * all current participants, and the current participants need
566  * to have destinations streams added for the joiner's sources.
567  *
568  * \param joiner The channel that is joining the softmix bridge
569  * \param participants The current participants in the softmix bridge
570  */
571 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
572 {
573         struct ast_stream_topology *joiner_video = NULL;
574         struct ast_bridge_channel *participant;
575         int res;
576         struct softmix_channel *sc;
577
578         joiner_video = ast_stream_topology_alloc();
579         if (!joiner_video) {
580                 return;
581         }
582
583         sc = joiner->tech_pvt;
584
585         ast_channel_lock(joiner->chan);
586         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
587         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
588         ast_channel_unlock(joiner->chan);
589
590         if (res || !sc->topology) {
591                 goto cleanup;
592         }
593
594         AST_LIST_TRAVERSE(participants, participant, entry) {
595                 if (participant == joiner) {
596                         continue;
597                 }
598                 ast_channel_lock(participant->chan);
599                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
600                                 ast_channel_get_stream_topology(participant->chan));
601                 ast_channel_unlock(participant->chan);
602                 if (res) {
603                         goto cleanup;
604                 }
605         }
606
607         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
608
609         AST_LIST_TRAVERSE(participants, participant, entry) {
610                 if (participant == joiner) {
611                         continue;
612                 }
613
614                 sc = participant->tech_pvt;
615                 if (append_all_streams(sc->topology, joiner_video)) {
616                         goto cleanup;
617                 }
618                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
619         }
620
621 cleanup:
622         ast_stream_topology_free(joiner_video);
623 }
624
625 /*! \brief Function called when a channel is joined into the bridge */
626 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
627 {
628         struct softmix_channel *sc;
629         struct softmix_bridge_data *softmix_data;
630         int set_binaural = 0;
631         /*
632          * If false, the channel will be convolved, but since it is a non stereo channel, output
633          * will be mono.
634          */
635         int skip_binaural_output = 1;
636         int pos_id;
637         int is_announcement = 0;
638         int samplerate_change;
639
640         softmix_data = bridge->tech_pvt;
641         if (!softmix_data) {
642                 return -1;
643         }
644
645         /* Create a new softmix_channel structure and allocate various things on it */
646         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
647                 return -1;
648         }
649
650         samplerate_change = softmix_data->internal_rate;
651         pos_id = -1;
652         if (bridge->softmix.binaural_active) {
653                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
654                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
655                         if (set_binaural) {
656                                 softmix_data->internal_rate = samplerate_change;
657                         }
658                         skip_binaural_output = 0;
659                 } else {
660                         is_announcement = 1;
661                 }
662                 if (set_binaural) {
663                         softmix_data->convolve.binaural_active = 1;
664                 }
665                 if (!skip_binaural_output)      {
666                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
667                         if (pos_id == -1) {
668                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
669                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
670                                                 ast_channel_name(bridge_channel->chan));
671                                 ast_free(sc);
672                                 return -1;
673                         }
674                 }
675         }
676
677         /* Can't forget the lock */
678         ast_mutex_init(&sc->lock);
679
680         /* Can't forget to record our pvt structure within the bridged channel structure */
681         bridge_channel->tech_pvt = sc;
682
683         set_softmix_bridge_data(softmix_data->internal_rate,
684                 softmix_data->internal_mixing_interval
685                         ? softmix_data->internal_mixing_interval
686                         : DEFAULT_SOFTMIX_INTERVAL,
687                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
688
689         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
690                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
691         }
692
693         softmix_poke_thread(softmix_data);
694         return 0;
695 }
696
697 static int remove_destination_streams(struct ast_stream_topology *topology,
698         const char *channel_name)
699 {
700         int i;
701         int stream_removed = 0;
702
703         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
704                 struct ast_stream *stream;
705
706                 stream = ast_stream_topology_get_stream(topology, i);
707
708                 if (is_video_dest(stream, channel_name, NULL)) {
709                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
710                         stream_removed = 1;
711                 }
712         }
713         return stream_removed;
714 }
715
716 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
717 {
718         struct ast_bridge_channel *participant;
719         struct softmix_channel *sc;
720
721         AST_LIST_TRAVERSE(participants, participant, entry) {
722                 sc = participant->tech_pvt;
723                 if (!remove_destination_streams(sc->topology, ast_channel_name(leaver->chan))) {
724                         continue;
725                 }
726                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
727         }
728
729         sc = leaver->tech_pvt;
730         if (remove_destination_streams(sc->topology, "")) {
731                 ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
732         }
733
734         return 0;
735 }
736
737 /*! \brief Function called when a channel leaves the bridge */
738 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
739 {
740         struct softmix_channel *sc;
741         struct softmix_bridge_data *softmix_data;
742
743         softmix_data = bridge->tech_pvt;
744         sc = bridge_channel->tech_pvt;
745         if (!sc) {
746                 return;
747         }
748
749         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
750                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
751         }
752
753         if (bridge->softmix.binaural_active) {
754                 if (sc->binaural) {
755                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
756                                         softmix_data->default_sample_size);
757                 }
758         }
759
760         bridge_channel->tech_pvt = NULL;
761
762         ast_stream_topology_free(sc->topology);
763
764         /* Drop mutex lock */
765         ast_mutex_destroy(&sc->lock);
766
767         /* Drop the factory */
768         ast_slinfactory_destroy(&sc->factory);
769
770         /* Drop any formats on the frames */
771         ao2_cleanup(sc->write_frame.subclass.format);
772
773         /* Drop the DSP */
774         ast_dsp_free(sc->dsp);
775
776         /* Eep! drop ourselves */
777         ast_free(sc);
778 }
779
780 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
781 {
782         struct ast_bridge_channel *cur;
783
784         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
785                 if (cur->suspended) {
786                         continue;
787                 }
788                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
789                         ast_bridge_channel_queue_frame(cur, frame);
790                         break;
791                 }
792         }
793 }
794
795 /*!
796  * \internal
797  * \brief Determine what to do with a video frame.
798  * \since 12.0.0
799  *
800  * \param bridge Which bridge is getting the frame
801  * \param bridge_channel Which channel is writing the frame.
802  * \param frame What is being written.
803  *
804  * \return Nothing
805  */
806 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
807 {
808         struct softmix_channel *sc;
809         int video_src_priority;
810
811         /* Determine if the video frame should be distributed or not */
812         switch (bridge->softmix.video_mode.mode) {
813         case AST_BRIDGE_VIDEO_MODE_NONE:
814                 break;
815         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
816                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
817                 if (video_src_priority == 1) {
818                         /* Pass to me and everyone else. */
819                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
820                 }
821                 break;
822         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
823                 sc = bridge_channel->tech_pvt;
824                 ast_mutex_lock(&sc->lock);
825                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
826                         sc->video_talker.energy_average,
827                         frame->subclass.frame_ending);
828                 ast_mutex_unlock(&sc->lock);
829                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
830                 if (video_src_priority == 1) {
831                         int num_src = ast_bridge_number_video_src(bridge);
832                         int echo = num_src > 1 ? 0 : 1;
833
834                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
835                 } else if (video_src_priority == 2) {
836                         softmix_pass_video_top_priority(bridge, frame);
837                 }
838                 break;
839         case AST_BRIDGE_VIDEO_MODE_SFU:
840                 /* Nothing special to do here, the bridge channel stream map will ensure the
841                  * video goes everywhere it needs to
842                  */
843                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
844                 break;
845         }
846 }
847
848 /*!
849  * \internal
850  * \brief Determine what to do with a voice frame.
851  * \since 12.0.0
852  *
853  * \param bridge Which bridge is getting the frame
854  * \param bridge_channel Which channel is writing the frame.
855  * \param frame What is being written.
856  *
857  * \return Nothing
858  */
859 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
860 {
861         struct softmix_channel *sc = bridge_channel->tech_pvt;
862         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
863         int totalsilence = 0;
864         int cur_energy = 0;
865         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
866                 bridge_channel->tech_args.silence_threshold :
867                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
868         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
869
870         /* Write the frame into the conference */
871         ast_mutex_lock(&sc->lock);
872
873         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
874                 /*
875                  * The incoming frame is not the expected format.  Update
876                  * the channel's translation path to get us slinear from
877                  * the new format for the next frame.
878                  *
879                  * There is the possibility that this frame is an old slinear
880                  * rate frame that was in flight when the softmix bridge
881                  * changed rates.  If so it will self correct on subsequent
882                  * frames.
883                  */
884                 ast_channel_lock(bridge_channel->chan);
885                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
886                         ast_channel_name(bridge_channel->chan),
887                         ast_format_get_name(frame->subclass.format),
888                         ast_format_get_name(sc->read_slin_format));
889                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
890                         sc->read_slin_format);
891                 ast_channel_unlock(bridge_channel->chan);
892         }
893
894         /* The channel will be leaving soon if there is no dsp. */
895         if (sc->dsp) {
896                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
897         }
898
899         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
900                 int cur_slot = sc->video_talker.energy_history_cur_slot;
901
902                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
903                 sc->video_talker.energy_accum += cur_energy;
904                 sc->video_talker.energy_history[cur_slot] = cur_energy;
905                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
906                 sc->video_talker.energy_history_cur_slot++;
907                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
908                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
909                 }
910         }
911
912         if (totalsilence < silence_threshold) {
913                 if (!sc->talking) {
914                         update_talking = 1;
915                 }
916                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
917         } else {
918                 if (sc->talking) {
919                         update_talking = 0;
920                 }
921                 sc->talking = 0;
922         }
923
924         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
925          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
926          * the audio by flushing the buffer before adding new audio in. */
927         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
928                 ast_slinfactory_flush(&sc->factory);
929         }
930
931         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
932          * is not determined to be talking. */
933         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
934                 ast_slinfactory_feed(&sc->factory, frame);
935         }
936
937         /* Alllll done */
938         ast_mutex_unlock(&sc->lock);
939
940         if (update_talking != -1) {
941                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
942         }
943 }
944
945 static int remove_all_original_streams(struct ast_stream_topology *dest,
946         const struct ast_stream_topology *source,
947         const struct ast_stream_topology *original)
948 {
949         int i;
950
951         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
952                 struct ast_stream *stream;
953                 int original_index;
954
955                 stream = ast_stream_topology_get_stream(source, i);
956
957                 /* Mark the existing stream as removed so we get a new one, this will get
958                  * reused on a subsequent renegotiation.
959                  */
960                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
961                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
962
963                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
964                                 struct ast_stream *removed;
965
966                                 /* Since the participant is still going to be in the bridge we
967                                  * change the name so that routing does not attempt to route video
968                                  * to this stream.
969                                  */
970                                 removed = ast_stream_clone(stream, "removed");
971                                 if (!removed) {
972                                         return -1;
973                                 }
974
975                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
976
977                                 /* The destination topology can only ever contain the same, or more,
978                                  * streams than the original so this is safe.
979                                  */
980                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
981                                         ast_stream_free(removed);
982                                         return -1;
983                                 }
984
985                                 break;
986                         }
987                 }
988         }
989
990         return 0;
991 }
992
993 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
994 {
995         struct ast_stream_topology *source_video = NULL;
996         struct ast_bridge_channel *participant;
997         int res;
998
999         source_video = ast_stream_topology_alloc();
1000         if (!source_video) {
1001                 return;
1002         }
1003
1004         ast_channel_lock(source->chan);
1005         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
1006         ast_channel_unlock(source->chan);
1007         if (res) {
1008                 goto cleanup;
1009         }
1010
1011         AST_LIST_TRAVERSE(participants, participant, entry) {
1012                 struct ast_stream_topology *original_topology;
1013                 struct softmix_channel *sc;
1014
1015                 if (participant == source) {
1016                         continue;
1017                 }
1018
1019                 sc = participant->tech_pvt;
1020
1021                 original_topology = ast_stream_topology_clone(sc->topology);
1022                 if (!original_topology) {
1023                         goto cleanup;
1024                 }
1025
1026                 /* We add all the source streams back in, if any removed streams are already present they will
1027                  * get used first followed by appending new ones.
1028                  */
1029                 if (append_all_streams(sc->topology, source_video)) {
1030                         ast_stream_topology_free(original_topology);
1031                         goto cleanup;
1032                 }
1033
1034                 /* And the original existing streams get marked as removed. This causes the remote side to see
1035                  * a new stream for the source streams.
1036                  */
1037                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1038                         ast_stream_topology_free(original_topology);
1039                         goto cleanup;
1040                 }
1041
1042                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1043                 ast_stream_topology_free(original_topology);
1044         }
1045
1046 cleanup:
1047         ast_stream_topology_free(source_video);
1048 }
1049
1050 /*!
1051  * \internal
1052  * \brief Determine what to do with a control frame.
1053  * \since 12.0.0
1054  *
1055  * \param bridge Which bridge is getting the frame
1056  * \param bridge_channel Which channel is writing the frame.
1057  * \param frame What is being written.
1058  *
1059  * \retval 0 Frame accepted into the bridge.
1060  * \retval -1 Frame needs to be deferred.
1061  */
1062 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1063 {
1064         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1065
1066         /*
1067          * XXX Softmix needs to use channel roles to determine what to
1068          * do with control frames.
1069          */
1070
1071         switch (frame->subclass.integer) {
1072         case AST_CONTROL_VIDUPDATE:
1073                 if (!bridge->softmix.video_mode.video_update_discard ||
1074                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1075                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1076                         softmix_data->last_video_update = ast_tvnow();
1077                 }
1078                 break;
1079         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1080                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1081                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1082                 }
1083                 break;
1084         default:
1085                 break;
1086         }
1087
1088         return 0;
1089 }
1090
1091 /*!
1092  * \internal
1093  * \brief Determine what to do with a frame written into the bridge.
1094  * \since 12.0.0
1095  *
1096  * \param bridge Which bridge is getting the frame
1097  * \param bridge_channel Which channel is writing the frame.
1098  * \param frame What is being written.
1099  *
1100  * \retval 0 Frame accepted into the bridge.
1101  * \retval -1 Frame needs to be deferred.
1102  *
1103  * \note On entry, bridge is already locked.
1104  */
1105 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1106 {
1107         int res = 0;
1108
1109         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1110                 /* "Accept" the frame and discard it. */
1111                 return 0;
1112         }
1113
1114         /*
1115          * XXX Softmix needs to use channel roles to determine who gets
1116          * what frame.  Possible roles: announcer, recorder, agent,
1117          * supervisor.
1118          */
1119         switch (frame->frametype) {
1120         case AST_FRAME_NULL:
1121                 /* "Accept" the frame and discard it. */
1122                 break;
1123         case AST_FRAME_DTMF_BEGIN:
1124         case AST_FRAME_DTMF_END:
1125                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1126                 break;
1127         case AST_FRAME_VOICE:
1128                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1129                 break;
1130         case AST_FRAME_VIDEO:
1131                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1132                 break;
1133         case AST_FRAME_CONTROL:
1134                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1135                 break;
1136         case AST_FRAME_BRIDGE_ACTION:
1137                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1138                 break;
1139         case AST_FRAME_BRIDGE_ACTION_SYNC:
1140                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1141                 ast_assert(0);
1142         default:
1143                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1144                 /* "Accept" the frame and discard it. */
1145                 break;
1146         }
1147
1148         return res;
1149 }
1150
1151 static void gather_softmix_stats(struct softmix_stats *stats,
1152         const struct softmix_bridge_data *softmix_data,
1153         struct ast_bridge_channel *bridge_channel)
1154 {
1155         int channel_native_rate;
1156
1157         /* Gather stats about channel sample rates. */
1158         ast_channel_lock(bridge_channel->chan);
1159         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1160                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1161         ast_channel_unlock(bridge_channel->chan);
1162
1163         if (stats->highest_supported_rate < channel_native_rate) {
1164                 stats->highest_supported_rate = channel_native_rate;
1165         }
1166         if (softmix_data->internal_rate < channel_native_rate) {
1167                 int i;
1168
1169                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1170                         if (stats->sample_rates[i] == channel_native_rate) {
1171                                 stats->num_channels[i]++;
1172                                 break;
1173                         } else if (!stats->sample_rates[i]) {
1174                                 stats->sample_rates[i] = channel_native_rate;
1175                                 stats->num_channels[i]++;
1176                                 break;
1177                         }
1178                 }
1179                 stats->num_above_internal_rate++;
1180         } else if (softmix_data->internal_rate == channel_native_rate) {
1181                 stats->num_at_internal_rate++;
1182         }
1183 }
1184
1185 /*!
1186  * \internal
1187  * \brief Analyse mixing statistics and change bridges internal rate
1188  * if necessary.
1189  *
1190  * \retval 0, no changes to internal rate
1191  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1192  */
1193 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1194                 struct softmix_bridge_data *softmix_data, int binaural_active)
1195 {
1196         int i;
1197
1198         if (binaural_active) {
1199                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1200         }
1201
1202         /*
1203          * Re-adjust the internal bridge sample rate if
1204          * 1. The bridge's internal sample rate is locked in at a sample
1205          *    rate other than the current sample rate being used.
1206          * 2. two or more channels support a higher sample rate
1207          * 3. no channels support the current sample rate or a higher rate
1208          */
1209         if (stats->locked_rate) {
1210                 /* if the rate is locked by the bridge, only update it if it differs
1211                  * from the current rate we are using. */
1212                 if (softmix_data->internal_rate != stats->locked_rate) {
1213                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1214                                 softmix_data->internal_rate, stats->locked_rate);
1215                         softmix_data->internal_rate = stats->locked_rate;
1216                         return 1;
1217                 }
1218         } else if (stats->num_above_internal_rate >= 2) {
1219                 /* the highest rate is just used as a starting point */
1220                 unsigned int best_rate = stats->highest_supported_rate;
1221                 int best_index = -1;
1222
1223                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1224                         if (stats->num_channels[i]) {
1225                                 break;
1226                         }
1227                         if (2 <= stats->num_channels[i]) {
1228                                 /* Two or more channels support this rate. */
1229                                 if (best_index == -1
1230                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1231                                         /*
1232                                          * best_rate starts out being the first sample rate
1233                                          * greater than the internal sample rate that two or
1234                                          * more channels support.
1235                                          *
1236                                          * or
1237                                          *
1238                                          * There are multiple rates above the internal rate
1239                                          * and this rate is higher than the previous rate two
1240                                          * or more channels support.
1241                                          */
1242                                         best_rate = stats->sample_rates[i];
1243                                         best_index = i;
1244                                 }
1245                         } else if (best_index == -1) {
1246                                 /*
1247                                  * It is possible that multiple channels exist with native sample
1248                                  * rates above the internal sample rate, but none of those channels
1249                                  * have the same rate in common.  In this case, the lowest sample
1250                                  * rate among those channels is picked. Over time as additional
1251                                  * statistic runs are made the internal sample rate number will
1252                                  * adjust to the most optimal sample rate, but it may take multiple
1253                                  * iterations.
1254                                  */
1255                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1256                         }
1257                 }
1258
1259                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1260                         softmix_data->internal_rate, best_rate);
1261                 softmix_data->internal_rate = best_rate;
1262                 return 1;
1263         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1264                 /* In this case, the highest supported rate is actually lower than the internal rate */
1265                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1266                         softmix_data->internal_rate, stats->highest_supported_rate);
1267                 softmix_data->internal_rate = stats->highest_supported_rate;
1268                 return 1;
1269         }
1270         return 0;
1271 }
1272
1273 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1274                 unsigned int starting_num_entries, unsigned int binaural_active)
1275 {
1276         memset(mixing_array, 0, sizeof(*mixing_array));
1277         mixing_array->max_num_entries = starting_num_entries;
1278         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1279                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1280                 return -1;
1281         }
1282         if (binaural_active) {
1283                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1284                                 sizeof(struct convolve_channel_pair *)))) {
1285                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1286                         return -1;
1287                 }
1288         }
1289         return 0;
1290 }
1291
1292 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1293                 unsigned int binaural_active)
1294 {
1295         ast_free(mixing_array->buffers);
1296         if (binaural_active) {
1297                 ast_free(mixing_array->chan_pairs);
1298         }
1299 }
1300
1301 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1302                 unsigned int num_entries, unsigned int binaural_active)
1303 {
1304         int16_t **tmp;
1305
1306         /* give it some room to grow since memory is cheap but allocations can be expensive */
1307         mixing_array->max_num_entries = num_entries;
1308         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1309                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1310                 return -1;
1311         }
1312         mixing_array->buffers = tmp;
1313
1314         if (binaural_active) {
1315                 struct convolve_channel_pair **tmp2;
1316                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1317                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1318                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1319                         return -1;
1320                 }
1321                 mixing_array->chan_pairs = tmp2;
1322         }
1323         return 0;
1324 }
1325
1326 /*!
1327  * \brief Mixing loop.
1328  *
1329  * \retval 0 on success
1330  * \retval -1 on failure
1331  */
1332 static int softmix_mixing_loop(struct ast_bridge *bridge)
1333 {
1334         struct softmix_stats stats = { { 0 }, };
1335         struct softmix_mixing_array mixing_array;
1336         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1337         struct ast_timer *timer;
1338         struct softmix_translate_helper trans_helper;
1339         int16_t buf[MAX_DATALEN];
1340 #ifdef BINAURAL_RENDERING
1341         int16_t bin_buf[MAX_DATALEN];
1342         int16_t ann_buf[MAX_DATALEN];
1343 #endif
1344         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1345         int timingfd;
1346         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1347         unsigned int idx;
1348         unsigned int x;
1349         int res = -1;
1350
1351         timer = softmix_data->timer;
1352         timingfd = ast_timer_fd(timer);
1353         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1354         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1355
1356         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1357         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1358                         bridge->softmix.binaural_active)) {
1359                 goto softmix_cleanup;
1360         }
1361
1362         /*
1363          * XXX Softmix needs to use channel roles to determine who gets
1364          * what audio mixed.
1365          */
1366         while (!softmix_data->stop && bridge->num_active) {
1367                 struct ast_bridge_channel *bridge_channel;
1368                 int timeout = -1;
1369                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1370                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1371                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1372
1373                 if (softmix_datalen > MAX_DATALEN) {
1374                         /* This should NEVER happen, but if it does we need to know about it. Almost
1375                          * all the memcpys used during this process depend on this assumption.  Rather
1376                          * than checking this over and over again through out the code, this single
1377                          * verification is done on each iteration. */
1378                         ast_log(LOG_WARNING,
1379                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1380                                 bridge->uniqueid);
1381                         goto softmix_cleanup;
1382                 }
1383
1384                 /* Grow the mixing array buffer as participants are added. */
1385                 if (mixing_array.max_num_entries < bridge->num_channels
1386                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1387                                         bridge->softmix.binaural_active)) {
1388                         goto softmix_cleanup;
1389                 }
1390
1391                 /* init the number of buffers stored in the mixing array to 0.
1392                  * As buffers are added for mixing, this number is incremented. */
1393                 mixing_array.used_entries = 0;
1394
1395                 /* These variables help determine if a rate change is required */
1396                 if (!stat_iteration_counter) {
1397                         memset(&stats, 0, sizeof(stats));
1398                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1399                 }
1400
1401                 /* If the sample rate has changed, update the translator helper */
1402                 if (update_all_rates) {
1403                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1404                 }
1405
1406 #ifdef BINAURAL_RENDERING
1407                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1408 #endif
1409
1410                 /* Go through pulling audio from each factory that has it available */
1411                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1412                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1413
1414                         if (!sc) {
1415                                 /* This channel failed to join successfully. */
1416                                 continue;
1417                         }
1418
1419                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1420                         if (update_all_rates) {
1421                                 set_softmix_bridge_data(softmix_data->internal_rate,
1422                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1423                         }
1424
1425                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1426                         if (!stat_iteration_counter) {
1427                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1428                         }
1429
1430                         /* if the channel is suspended, don't check for audio, but still gather stats */
1431                         if (bridge_channel->suspended) {
1432                                 continue;
1433                         }
1434
1435                         /* Try to get audio from the factory if available */
1436                         ast_mutex_lock(&sc->lock);
1437                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1438 #ifdef BINAURAL_RENDERING
1439                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1440                                                 ast_channel_name(bridge_channel->chan));
1441 #endif
1442                                 mixing_array.used_entries++;
1443                         }
1444                         ast_mutex_unlock(&sc->lock);
1445                 }
1446
1447                 /* mix it like crazy (non binaural channels)*/
1448                 memset(buf, 0, softmix_datalen);
1449                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1450                         for (x = 0; x < softmix_samples; ++x) {
1451                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1452                         }
1453                 }
1454
1455 #ifdef BINAURAL_RENDERING
1456                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1457 #endif
1458
1459                 /* Next step go through removing the channel's own audio and creating a good frame... */
1460                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1461                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1462
1463                         if (!sc || bridge_channel->suspended) {
1464                                 /* This channel failed to join successfully or is suspended. */
1465                                 continue;
1466                         }
1467
1468                         ast_mutex_lock(&sc->lock);
1469
1470                         /* Make SLINEAR write frame from local buffer */
1471                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1472                                 "Replace softmix channel slin format");
1473 #ifdef BINAURAL_RENDERING
1474                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1475                                         && sc->binaural) {
1476                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1477                                                 softmix_samples, buf);
1478                         } else
1479 #endif
1480                         {
1481                                 sc->write_frame.datalen = softmix_datalen;
1482                                 sc->write_frame.samples = softmix_samples;
1483                                 memcpy(sc->final_buf, buf, softmix_datalen);
1484                         }
1485                         /* process the softmix channel's new write audio */
1486                         softmix_process_write_audio(&trans_helper,
1487                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1488                                         softmix_data->default_sample_size);
1489
1490                         ast_mutex_unlock(&sc->lock);
1491
1492                         /* A frame is now ready for the channel. */
1493                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1494                 }
1495
1496                 update_all_rates = 0;
1497                 if (!stat_iteration_counter) {
1498                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1499                                         bridge->softmix.binaural_active);
1500                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1501                 }
1502                 stat_iteration_counter--;
1503
1504                 ast_bridge_unlock(bridge);
1505                 /* cleanup any translation frame data from the previous mixing iteration. */
1506                 softmix_translate_helper_cleanup(&trans_helper);
1507                 /* Wait for the timing source to tell us to wake up and get things done */
1508                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1509                 if (ast_timer_ack(timer, 1) < 0) {
1510                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1511                                 bridge->uniqueid);
1512                         ast_bridge_lock(bridge);
1513                         goto softmix_cleanup;
1514                 }
1515                 ast_bridge_lock(bridge);
1516
1517                 /* make sure to detect mixing interval changes if they occur. */
1518                 if (bridge->softmix.internal_mixing_interval
1519                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1520                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1521                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1522                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1523                 }
1524         }
1525
1526         res = 0;
1527
1528 softmix_cleanup:
1529         softmix_translate_helper_destroy(&trans_helper);
1530         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1531         return res;
1532 }
1533
1534 /*!
1535  * \internal
1536  * \brief Mixing thread.
1537  * \since 12.0.0
1538  *
1539  * \note The thread does not have its own reference to the
1540  * bridge.  The lifetime of the thread is tied to the lifetime
1541  * of the mixing technology association with the bridge.
1542  */
1543 static void *softmix_mixing_thread(void *data)
1544 {
1545         struct softmix_bridge_data *softmix_data = data;
1546         struct ast_bridge *bridge = softmix_data->bridge;
1547
1548         ast_bridge_lock(bridge);
1549         if (bridge->callid) {
1550                 ast_callid_threadassoc_add(bridge->callid);
1551         }
1552
1553         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1554
1555         while (!softmix_data->stop) {
1556                 if (!bridge->num_active) {
1557                         /* Wait for something to happen to the bridge. */
1558                         ast_bridge_unlock(bridge);
1559                         ast_mutex_lock(&softmix_data->lock);
1560                         if (!softmix_data->stop) {
1561                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1562                         }
1563                         ast_mutex_unlock(&softmix_data->lock);
1564                         ast_bridge_lock(bridge);
1565                         continue;
1566                 }
1567
1568                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1569 #ifndef BINAURAL_RENDERING
1570                         ast_bridge_lock(bridge);
1571                         bridge->softmix.binaural_active = 0;
1572                         ast_bridge_unlock(bridge);
1573                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1574                                         "compiled.\n", bridge->uniqueid);
1575 #else
1576                         /* Set and init binaural data if binaural is activated in the configuration. */
1577                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1578                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1579                                         softmix_data->internal_mixing_interval);
1580                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1581                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1582                                         == -1) {
1583                                 ast_bridge_lock(bridge);
1584                                 bridge->softmix.binaural_active = 0;
1585                                 ast_bridge_unlock(bridge);
1586                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1587                                                 "binaural processing,  Will only process mono audio.\n",
1588                                                 bridge->uniqueid);
1589                         }
1590                         softmix_data->binaural_init = 1;
1591 #endif
1592                 }
1593
1594                 if (softmix_mixing_loop(bridge)) {
1595                         /*
1596                          * A mixing error occurred.  Sleep and try again later so we
1597                          * won't flood the logs.
1598                          */
1599                         ast_bridge_unlock(bridge);
1600                         sleep(1);
1601                         ast_bridge_lock(bridge);
1602                 }
1603         }
1604
1605         ast_bridge_unlock(bridge);
1606
1607         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1608
1609         return NULL;
1610 }
1611
1612 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1613 {
1614         if (softmix_data->timer) {
1615                 ast_timer_close(softmix_data->timer);
1616                 softmix_data->timer = NULL;
1617         }
1618         ast_mutex_destroy(&softmix_data->lock);
1619         ast_cond_destroy(&softmix_data->cond);
1620         ast_free(softmix_data);
1621 }
1622
1623 /*! \brief Function called when a bridge is created */
1624 static int softmix_bridge_create(struct ast_bridge *bridge)
1625 {
1626         struct softmix_bridge_data *softmix_data;
1627
1628         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1629         if (!softmix_data) {
1630                 return -1;
1631         }
1632         softmix_data->bridge = bridge;
1633         ast_mutex_init(&softmix_data->lock);
1634         ast_cond_init(&softmix_data->cond, NULL);
1635         softmix_data->timer = ast_timer_open();
1636         if (!softmix_data->timer) {
1637                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1638                 softmix_bridge_data_destroy(softmix_data);
1639                 return -1;
1640         }
1641         /* start at minimum rate, let it grow from there */
1642         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1643         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1644
1645 #ifdef BINAURAL_RENDERING
1646         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1647                         softmix_data->internal_mixing_interval);
1648 #endif
1649
1650         bridge->tech_pvt = softmix_data;
1651
1652         /* Start the mixing thread. */
1653         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1654                 softmix_data)) {
1655                 softmix_data->thread = AST_PTHREADT_NULL;
1656                 softmix_bridge_data_destroy(softmix_data);
1657                 bridge->tech_pvt = NULL;
1658                 return -1;
1659         }
1660
1661         return 0;
1662 }
1663
1664 /*!
1665  * \internal
1666  * \brief Request the softmix mixing thread stop.
1667  * \since 12.0.0
1668  *
1669  * \param bridge Which bridge is being stopped.
1670  *
1671  * \return Nothing
1672  */
1673 static void softmix_bridge_stop(struct ast_bridge *bridge)
1674 {
1675         struct softmix_bridge_data *softmix_data;
1676
1677         softmix_data = bridge->tech_pvt;
1678         if (!softmix_data) {
1679                 return;
1680         }
1681
1682         ast_mutex_lock(&softmix_data->lock);
1683         softmix_data->stop = 1;
1684         ast_mutex_unlock(&softmix_data->lock);
1685 }
1686
1687 /*! \brief Function called when a bridge is destroyed */
1688 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1689 {
1690         struct softmix_bridge_data *softmix_data;
1691         pthread_t thread;
1692
1693         softmix_data = bridge->tech_pvt;
1694         if (!softmix_data) {
1695                 return;
1696         }
1697
1698         /* Stop the mixing thread. */
1699         ast_mutex_lock(&softmix_data->lock);
1700         softmix_data->stop = 1;
1701         ast_cond_signal(&softmix_data->cond);
1702         thread = softmix_data->thread;
1703         softmix_data->thread = AST_PTHREADT_NULL;
1704         ast_mutex_unlock(&softmix_data->lock);
1705         if (thread != AST_PTHREADT_NULL) {
1706                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1707                 pthread_join(thread, NULL);
1708         }
1709 #ifdef BINAURAL_RENDERING
1710         free_convolve_data(&softmix_data->convolve);
1711 #endif
1712         softmix_bridge_data_destroy(softmix_data);
1713         bridge->tech_pvt = NULL;
1714 }
1715
1716 /*!
1717  * \brief Map a source stream to all of its destination streams.
1718  *
1719  * \param source_stream_name Name of the source stream
1720  * \param source_channel_name Name of channel where the source stream originates
1721  * \param bridge_stream_position The slot in the bridge where source video will come from
1722  * \param participants The bridge_channels in the bridge
1723  */
1724 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1725         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1726 {
1727         struct ast_bridge_channel *participant;
1728
1729         AST_LIST_TRAVERSE(participants, participant, entry) {
1730                 int i;
1731                 struct ast_stream_topology *topology;
1732
1733                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1734                         continue;
1735                 }
1736
1737                 ast_bridge_channel_lock(participant);
1738                 ast_channel_lock(participant->chan);
1739                 topology = ast_channel_get_stream_topology(participant->chan);
1740
1741                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1742                         struct ast_stream *stream;
1743
1744                         stream = ast_stream_topology_get_stream(topology, i);
1745                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1746                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1747                                 break;
1748                         }
1749                 }
1750                 ast_channel_unlock(participant->chan);
1751                 ast_bridge_channel_unlock(participant);
1752         }
1753 }
1754
1755 /*!
1756  * \brief stream_topology_changed callback
1757  *
1758  * For most video modes, nothing beyond the ordinary is required.
1759  * For the SFU case, though, we need to completely remap the streams
1760  * in order to ensure video gets directed where it is expected to go.
1761  *
1762  * \param bridge The bridge
1763  * \param bridge_channel Channel whose topology has changed
1764  */
1765 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1766 {
1767         struct ast_bridge_channel *participant;
1768         struct ast_vector_int media_types;
1769         int nths[AST_MEDIA_TYPE_END] = {0};
1770
1771         switch (bridge->softmix.video_mode.mode) {
1772         case AST_BRIDGE_VIDEO_MODE_NONE:
1773         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1774         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1775         default:
1776                 ast_bridge_channel_stream_map(bridge_channel);
1777                 return;
1778         case AST_BRIDGE_VIDEO_MODE_SFU:
1779                 break;
1780         }
1781
1782         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1783
1784         /* First traversal: re-initialize all of the participants' stream maps */
1785         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1786                 ast_bridge_channel_lock(participant);
1787                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1788                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1789                 ast_bridge_channel_unlock(participant);
1790         }
1791
1792         /* Second traversal: Map specific video channels from their source to their destinations.
1793          *
1794          * This is similar to what is done in ast_stream_topology_map(),
1795          * except that video channels are handled differently.  Each video
1796          * source has it's own unique index on the bridge.  This way, a
1797          * particular channel's source video can be distributed to the
1798          * appropriate destination streams on the other channels.
1799          */
1800         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1801                 int i;
1802                 struct ast_stream_topology *topology;
1803
1804                 ast_bridge_channel_lock(participant);
1805                 ast_channel_lock(participant->chan);
1806
1807                 topology = ast_channel_get_stream_topology(participant->chan);
1808                 if (topology) {
1809                         /*
1810                          * Sigh.  We have to clone to avoid deadlock in
1811                          * map_source_to_destinations() because topology
1812                          * is not an ao2 object.
1813                          */
1814                         topology = ast_stream_topology_clone(topology);
1815                 }
1816                 if (!topology) {
1817                         /* Oh, my, we are in trouble. */
1818                         ast_channel_unlock(participant->chan);
1819                         ast_bridge_channel_unlock(participant);
1820                         continue;
1821                 }
1822
1823                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1824                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1825
1826                         if (is_video_source(stream)) {
1827                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1828                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1829                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1830                                 /*
1831                                  * Unlock the channel and participant to prevent
1832                                  * potential deadlock in map_source_to_destinations().
1833                                  */
1834                                 ast_channel_unlock(participant->chan);
1835                                 ast_bridge_channel_unlock(participant);
1836                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1837                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1838                                 ast_bridge_channel_lock(participant);
1839                                 ast_channel_lock(participant->chan);
1840                         } else if (is_video_dest(stream, NULL, NULL)) {
1841                                 /* We expect to never read media from video destination channels, but just
1842                                  * in case, we should set their to_bridge value to -1.
1843                                  */
1844                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1845                         } else {
1846                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1847                                  * be factored out in some way
1848                                  */
1849                                 enum ast_media_type type = ast_stream_get_type(stream);
1850                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1851                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1852
1853                                 if (index == -1) {
1854                                         AST_VECTOR_APPEND(&media_types, type);
1855                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1856                                 }
1857
1858                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1859                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1860                         }
1861                 }
1862
1863                 ast_stream_topology_free(topology);
1864
1865                 ast_channel_unlock(participant->chan);
1866                 ast_bridge_channel_unlock(participant);
1867         }
1868 }
1869
1870 static struct ast_bridge_technology softmix_bridge = {
1871         .name = "softmix",
1872         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1873         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1874         .create = softmix_bridge_create,
1875         .stop = softmix_bridge_stop,
1876         .destroy = softmix_bridge_destroy,
1877         .join = softmix_bridge_join,
1878         .leave = softmix_bridge_leave,
1879         .unsuspend = softmix_bridge_unsuspend,
1880         .write = softmix_bridge_write,
1881         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1882 };
1883
1884 #ifdef TEST_FRAMEWORK
1885 struct stream_parameters {
1886         const char *name;
1887         const char *formats;
1888         enum ast_media_type type;
1889 };
1890
1891 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1892 {
1893         struct ast_stream_topology *topology;
1894         size_t i;
1895
1896         topology = ast_stream_topology_alloc();
1897         if (!topology) {
1898                 return NULL;
1899         }
1900
1901         for (i = 0; i < num_streams; ++i) {
1902                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1903                 struct ast_stream *stream;
1904
1905                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1906                 if (!caps) {
1907                         goto fail;
1908                 }
1909                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1910                         goto fail;
1911                 }
1912                 stream = ast_stream_alloc(params[i].name, params[i].type);
1913                 if (!stream) {
1914                         goto fail;
1915                 }
1916                 ast_stream_set_formats(stream, caps);
1917                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1918                         ast_stream_free(stream);
1919                         goto fail;
1920                 }
1921         }
1922
1923         return topology;
1924
1925 fail:
1926         ast_stream_topology_free(topology);
1927         return NULL;
1928 }
1929
1930 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1931         const struct stream_parameters *params)
1932 {
1933         struct ast_format_cap *stream_caps;
1934         struct ast_format_cap *params_caps;
1935
1936         if (ast_stream_get_type(stream) != params->type) {
1937                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1938                         ast_codec_media_type2str(params->type),
1939                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1940                 return -1;
1941         }
1942         if (strcmp(ast_stream_get_name(stream), params->name)) {
1943                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1944                         params->name, ast_stream_get_name(stream));
1945                 return -1;
1946         }
1947
1948         stream_caps = ast_stream_get_formats(stream);
1949         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1950         if (!params_caps) {
1951                 ast_test_status_update(test, "Allocation error on capabilities\n");
1952                 return -1;
1953         }
1954         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
1955
1956         if (ast_format_cap_identical(stream_caps, params_caps)) {
1957                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
1958                         ast_stream_get_name(stream));
1959                 ao2_cleanup(params_caps);
1960                 return -1;
1961         }
1962
1963         ao2_cleanup(params_caps);
1964         return 0;
1965 }
1966
1967 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
1968         const struct stream_parameters *params, size_t num_streams)
1969 {
1970         int i;
1971
1972         if (ast_stream_topology_get_count(topology) < num_streams) {
1973                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
1974                         ast_stream_topology_get_count(topology), num_streams);
1975                 return -1;
1976         }
1977
1978         for (i = 0; i < ARRAY_LEN(params); ++i) {
1979                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
1980                         return -1;
1981                 }
1982         }
1983
1984         return 0;
1985 }
1986
1987 AST_TEST_DEFINE(sfu_append_source_streams)
1988 {
1989         enum ast_test_result_state res = AST_TEST_FAIL;
1990         static const struct stream_parameters bob_streams[] = {
1991                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1992                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1993         };
1994         static const struct stream_parameters alice_streams[] = {
1995                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
1996                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
1997         };
1998         static const struct stream_parameters alice_dest_stream = {
1999                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
2000         };
2001         static const struct stream_parameters bob_dest_stream = {
2002                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2003         };
2004         struct ast_stream_topology *topology_alice = NULL;
2005         struct ast_stream_topology *topology_bob = NULL;
2006
2007         switch (cmd) {
2008         case TEST_INIT:
2009                 info->name = "sfu_append_source_streams";
2010                 info->category = "/bridges/bridge_softmix/";
2011                 info->summary = "Test appending of video streams";
2012                 info->description =
2013                         "This tests does stuff.";
2014                 return AST_TEST_NOT_RUN;
2015         case TEST_EXECUTE:
2016                 break;
2017         }
2018
2019         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2020         if (!topology_alice) {
2021                 goto end;
2022         }
2023
2024         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2025         if (!topology_bob) {
2026                 goto end;
2027         }
2028
2029         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2030                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2031                 goto end;
2032         }
2033
2034         if (ast_stream_topology_get_count(topology_alice) != 3) {
2035                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2036                         ast_stream_topology_get_count(topology_alice), 3);
2037                 goto end;
2038         }
2039
2040         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2041                 goto end;
2042         }
2043
2044         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2045                 goto end;
2046         }
2047
2048         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2049                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2050                 goto end;
2051         }
2052
2053         if (ast_stream_topology_get_count(topology_bob) != 3) {
2054                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2055                         ast_stream_topology_get_count(topology_bob), 3);
2056                 goto end;
2057         }
2058
2059         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2060                 goto end;
2061         }
2062
2063         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2064                 goto end;
2065         }
2066
2067         res = AST_TEST_PASS;
2068
2069 end:
2070         ast_stream_topology_free(topology_alice);
2071         ast_stream_topology_free(topology_bob);
2072         return res;
2073 }
2074
2075 AST_TEST_DEFINE(sfu_remove_destination_streams)
2076 {
2077         enum ast_test_result_state res = AST_TEST_FAIL;
2078         static const struct stream_parameters params[] = {
2079                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2080                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2081                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2082                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2083         };
2084         static const struct {
2085                 const char *channel_name;
2086                 int num_streams;
2087                 int params_index[4];
2088         } removal_results[] = {
2089                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2090                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2091                 { "", 4, { 0, 1, 2, 3 }, },
2092         };
2093         struct ast_stream_topology *orig = NULL;
2094         int i;
2095
2096         switch (cmd) {
2097         case TEST_INIT:
2098                 info->name = "sfu_remove_destination_streams";
2099                 info->category = "/bridges/bridge_softmix/";
2100                 info->summary = "Test removal of destination video streams";
2101                 info->description =
2102                         "This tests does stuff.";
2103                 return AST_TEST_NOT_RUN;
2104         case TEST_EXECUTE:
2105                 break;
2106         }
2107
2108         orig = build_topology(params, ARRAY_LEN(params));
2109         if (!orig) {
2110                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2111                 goto end;
2112         }
2113
2114         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2115                 int j;
2116
2117                 remove_destination_streams(orig, removal_results[i].channel_name);
2118
2119                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2120                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2121                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2122                         goto end;
2123                 }
2124
2125                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2126                         struct ast_stream *actual;
2127                         struct ast_stream *expected;
2128                         int orig_index;
2129
2130                         actual = ast_stream_topology_get_stream(orig, j);
2131
2132                         orig_index = removal_results[i].params_index[j];
2133                         expected = ast_stream_topology_get_stream(orig, orig_index);
2134
2135                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2136                                 ast_stream_get_formats(expected))) {
2137                                 struct ast_str *expected_str;
2138                                 struct ast_str *actual_str;
2139
2140                                 expected_str = ast_str_alloca(64);
2141                                 actual_str = ast_str_alloca(64);
2142
2143                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2144                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2145                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2146                                 goto end;
2147                         }
2148
2149                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2150                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2151                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2152                                 goto end;
2153                         }
2154                 }
2155         }
2156
2157         res = AST_TEST_PASS;
2158
2159 end:
2160         ast_stream_topology_free(orig);
2161         return res;
2162 }
2163
2164 #endif
2165
2166 static int unload_module(void)
2167 {
2168         ast_bridge_technology_unregister(&softmix_bridge);
2169         AST_TEST_UNREGISTER(sfu_append_source_streams);
2170         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2171         return 0;
2172 }
2173
2174 static int load_module(void)
2175 {
2176         if (ast_bridge_technology_register(&softmix_bridge)) {
2177                 unload_module();
2178                 return AST_MODULE_LOAD_DECLINE;
2179         }
2180         AST_TEST_REGISTER(sfu_append_source_streams);
2181         AST_TEST_REGISTER(sfu_remove_destination_streams);
2182         return AST_MODULE_LOAD_SUCCESS;
2183 }
2184
2185 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");