bridge: Change participant SFU streams when source streams change.
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                                                   and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO &&
422                 strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
423                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
424                 return 1;
425         }
426
427         return 0;
428 }
429
430 /*!
431  * \brief Determine if a stream is a video destination stream.
432  *
433  * A source channel name can be provided to narrow this to a destination stream
434  * for a particular source channel. Further, a source stream name can be provided
435  * to narrow this to a particular source stream's destination. However, empty strings
436  * can be provided to match any destination video stream, regardless of source channel
437  * or source stream.
438  *
439  * \param stream The stream to test
440  * \param source_channel_name The name of a source video channel to match
441  * \param source_stream_name The name of the source video stream to match
442  * \retval 1 The stream is a video destination stream
443  * \retval 0 The stream is not a video destination stream
444  */
445 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
446         const char *source_stream_name)
447 {
448         char *dest_video_name;
449         size_t dest_video_name_len;
450
451         if (ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
452                 return 0;
453         }
454
455         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
456
457         if (!ast_strlen_zero(source_channel_name)) {
458                 dest_video_name_len += strlen(source_channel_name) + 1;
459                 if (!ast_strlen_zero(source_stream_name)) {
460                         dest_video_name_len += strlen(source_stream_name) + 1;
461                 }
462         }
463         dest_video_name = ast_alloca(dest_video_name_len);
464
465         if (!ast_strlen_zero(source_channel_name)) {
466                 if (!ast_strlen_zero(source_stream_name)) {
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 } else {
473                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                                 source_channel_name);
476                         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
477                 }
478         } else {
479                 snprintf(dest_video_name, dest_video_name_len, "%s",
480                         SOFTBRIDGE_VIDEO_DEST_PREFIX);
481                 return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
482         }
483
484         return 0;
485 }
486
487 static int append_source_streams(struct ast_stream_topology *dest,
488         const char *channel_name,
489         const struct ast_stream_topology *source)
490 {
491         int i;
492
493         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
494                 struct ast_stream *stream;
495                 struct ast_stream *stream_clone;
496                 char *stream_clone_name = NULL;
497
498                 stream = ast_stream_topology_get_stream(source, i);
499                 if (!is_video_source(stream)) {
500                         continue;
501                 }
502
503                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
504                         channel_name, ast_stream_get_name(stream)) < 0) {
505                         ast_free(stream_clone_name);
506                         return -1;
507                 }
508
509                 stream_clone = ast_stream_clone(stream, stream_clone_name);
510                 ast_free(stream_clone_name);
511                 if (!stream_clone) {
512                         return -1;
513                 }
514                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
515                         ast_stream_free(stream_clone);
516                         return -1;
517                 }
518         }
519
520         return 0;
521 }
522
523 static int append_all_streams(struct ast_stream_topology *dest,
524         const struct ast_stream_topology *source)
525 {
526         int i;
527         int dest_index = 0;
528
529         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
530                 struct ast_stream *clone;
531                 int added = 0;
532
533                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
534                 if (!clone) {
535                         return -1;
536                 }
537
538                 /* If we can reuse an existing removed stream then do so */
539                 while (dest_index < ast_stream_topology_get_count(dest)) {
540                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
541
542                         dest_index++;
543
544                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
545                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
546                                 added = 1;
547                                 break;
548                         }
549                 }
550
551                 /* If no removed stream exists that we took the place of append the stream */
552                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
553                         ast_stream_free(clone);
554                         return -1;
555                 }
556         }
557
558         return 0;
559 }
560
561 /*!
562  * \brief Issue channel stream topology change requests.
563  *
564  * When in SFU mode, each participant needs to be able to
565  * send video directly to other participants in the bridge.
566  * This means that all participants need to have their topologies
567  * updated. The joiner needs to have destination streams for
568  * all current participants, and the current participants need
569  * to have destinations streams added for the joiner's sources.
570  *
571  * \param joiner The channel that is joining the softmix bridge
572  * \param participants The current participants in the softmix bridge
573  */
574 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
575 {
576         struct ast_stream_topology *joiner_topology = NULL;
577         struct ast_stream_topology *joiner_video = NULL;
578         struct ast_stream_topology *existing_video = NULL;
579         struct ast_bridge_channel *participant;
580         int res;
581
582         joiner_video = ast_stream_topology_alloc();
583         if (!joiner_video) {
584                 return;
585         }
586
587         ast_channel_lock(joiner->chan);
588         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
589         ast_channel_unlock(joiner->chan);
590
591         if (res) {
592                 goto cleanup;
593         }
594
595         existing_video = ast_stream_topology_alloc();
596         if (!existing_video) {
597                 goto cleanup;
598         }
599
600         AST_LIST_TRAVERSE(participants, participant, entry) {
601                 if (participant == joiner) {
602                         continue;
603                 }
604                 ast_channel_lock(participant->chan);
605                 res = append_source_streams(existing_video, ast_channel_name(participant->chan),
606                                 ast_channel_get_stream_topology(participant->chan));
607                 ast_channel_unlock(participant->chan);
608                 if (res) {
609                         goto cleanup;
610                 }
611         }
612
613         ast_channel_lock(joiner->chan);
614         joiner_topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
615         ast_channel_unlock(joiner->chan);
616         if (!joiner_topology) {
617                 goto cleanup;
618         }
619         if (append_all_streams(joiner_topology, existing_video)) {
620                 goto cleanup;
621         }
622         ast_channel_request_stream_topology_change(joiner->chan, joiner_topology, NULL);
623
624         AST_LIST_TRAVERSE(participants, participant, entry) {
625                 struct ast_stream_topology *participant_topology;
626
627                 if (participant == joiner) {
628                         continue;
629                 }
630                 ast_channel_lock(participant->chan);
631                 participant_topology = ast_stream_topology_clone(ast_channel_get_stream_topology(participant->chan));
632                 ast_channel_unlock(participant->chan);
633                 if (!participant_topology) {
634                         goto cleanup;
635                 }
636                 if (append_all_streams(participant_topology, joiner_video)) {
637                         ast_stream_topology_free(participant_topology);
638                         goto cleanup;
639                 }
640                 ast_channel_request_stream_topology_change(participant->chan, participant_topology, NULL);
641                 ast_stream_topology_free(participant_topology);
642         }
643
644 cleanup:
645         ast_stream_topology_free(joiner_video);
646         ast_stream_topology_free(existing_video);
647         ast_stream_topology_free(joiner_topology);
648 }
649
650 /*! \brief Function called when a channel is joined into the bridge */
651 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
652 {
653         struct softmix_channel *sc;
654         struct softmix_bridge_data *softmix_data;
655         int set_binaural = 0;
656         /*
657          * If false, the channel will be convolved, but since it is a non stereo channel, output
658          * will be mono.
659          */
660         int skip_binaural_output = 1;
661         int pos_id;
662         int is_announcement = 0;
663         int samplerate_change;
664
665         softmix_data = bridge->tech_pvt;
666         if (!softmix_data) {
667                 return -1;
668         }
669
670         /* Create a new softmix_channel structure and allocate various things on it */
671         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
672                 return -1;
673         }
674
675         samplerate_change = softmix_data->internal_rate;
676         pos_id = -1;
677         if (bridge->softmix.binaural_active) {
678                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
679                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
680                         if (set_binaural) {
681                                 softmix_data->internal_rate = samplerate_change;
682                         }
683                         skip_binaural_output = 0;
684                 } else {
685                         is_announcement = 1;
686                 }
687                 if (set_binaural) {
688                         softmix_data->convolve.binaural_active = 1;
689                 }
690                 if (!skip_binaural_output)      {
691                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
692                         if (pos_id == -1) {
693                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
694                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
695                                                 ast_channel_name(bridge_channel->chan));
696                                 return -1;
697                         }
698                 }
699         }
700
701         /* Can't forget the lock */
702         ast_mutex_init(&sc->lock);
703
704         /* Can't forget to record our pvt structure within the bridged channel structure */
705         bridge_channel->tech_pvt = sc;
706
707         set_softmix_bridge_data(softmix_data->internal_rate,
708                 softmix_data->internal_mixing_interval
709                         ? softmix_data->internal_mixing_interval
710                         : DEFAULT_SOFTMIX_INTERVAL,
711                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
712
713         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
714                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
715         }
716
717         softmix_poke_thread(softmix_data);
718         return 0;
719 }
720
721 static int remove_destination_streams(struct ast_stream_topology *dest,
722         const char *channel_name,
723         const struct ast_stream_topology *source)
724 {
725         int i;
726
727         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
728                 struct ast_stream *stream;
729                 struct ast_stream *stream_clone;
730
731                 stream = ast_stream_topology_get_stream(source, i);
732
733                 stream_clone = ast_stream_clone(stream, NULL);
734                 if (!stream_clone) {
735                         continue;
736                 }
737
738                 if (is_video_dest(stream, channel_name, NULL)) {
739                         ast_stream_set_state(stream_clone, AST_STREAM_STATE_REMOVED);
740                 }
741
742                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
743                         ast_stream_free(stream_clone);
744                 }
745         }
746
747         return 0;
748 }
749
750 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
751 {
752         struct ast_stream_topology *leaver_topology;
753         struct ast_bridge_channel *participant;
754
755         leaver_topology = ast_stream_topology_alloc();
756         if (!leaver_topology) {
757                 return -1;
758         }
759
760         AST_LIST_TRAVERSE(participants, participant, entry) {
761                 struct ast_stream_topology *participant_topology;
762
763                 participant_topology = ast_stream_topology_alloc();
764                 if (!participant_topology) {
765                         continue;
766                 }
767
768                 ast_channel_lock(participant->chan);
769                 remove_destination_streams(participant_topology, ast_channel_name(leaver->chan), ast_channel_get_stream_topology(participant->chan));
770                 ast_channel_unlock(participant->chan);
771                 ast_channel_request_stream_topology_change(participant->chan, participant_topology, NULL);
772                 ast_stream_topology_free(participant_topology);
773         }
774
775         ast_channel_lock(leaver->chan);
776         remove_destination_streams(leaver_topology, "", ast_channel_get_stream_topology(leaver->chan));
777         ast_channel_unlock(leaver->chan);
778         ast_channel_request_stream_topology_change(leaver->chan, leaver_topology, NULL);
779         ast_stream_topology_free(leaver_topology);
780
781         return 0;
782 }
783
784 /*! \brief Function called when a channel leaves the bridge */
785 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
786 {
787         struct softmix_channel *sc;
788         struct softmix_bridge_data *softmix_data;
789
790         softmix_data = bridge->tech_pvt;
791         sc = bridge_channel->tech_pvt;
792         if (!sc) {
793                 return;
794         }
795
796         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
797                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
798         }
799
800         if (bridge->softmix.binaural_active) {
801                 if (sc->binaural) {
802                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
803                                         softmix_data->default_sample_size);
804                 }
805         }
806
807         bridge_channel->tech_pvt = NULL;
808
809         /* Drop mutex lock */
810         ast_mutex_destroy(&sc->lock);
811
812         /* Drop the factory */
813         ast_slinfactory_destroy(&sc->factory);
814
815         /* Drop any formats on the frames */
816         ao2_cleanup(sc->write_frame.subclass.format);
817
818         /* Drop the DSP */
819         ast_dsp_free(sc->dsp);
820
821         /* Eep! drop ourselves */
822         ast_free(sc);
823 }
824
825 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
826 {
827         struct ast_bridge_channel *cur;
828
829         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
830                 if (cur->suspended) {
831                         continue;
832                 }
833                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
834                         ast_bridge_channel_queue_frame(cur, frame);
835                         break;
836                 }
837         }
838 }
839
840 /*!
841  * \internal
842  * \brief Determine what to do with a video frame.
843  * \since 12.0.0
844  *
845  * \param bridge Which bridge is getting the frame
846  * \param bridge_channel Which channel is writing the frame.
847  * \param frame What is being written.
848  *
849  * \return Nothing
850  */
851 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
852 {
853         struct softmix_channel *sc;
854         int video_src_priority;
855
856         /* Determine if the video frame should be distributed or not */
857         switch (bridge->softmix.video_mode.mode) {
858         case AST_BRIDGE_VIDEO_MODE_NONE:
859                 break;
860         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
861                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
862                 if (video_src_priority == 1) {
863                         /* Pass to me and everyone else. */
864                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
865                 }
866                 break;
867         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
868                 sc = bridge_channel->tech_pvt;
869                 ast_mutex_lock(&sc->lock);
870                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
871                         sc->video_talker.energy_average,
872                         frame->subclass.frame_ending);
873                 ast_mutex_unlock(&sc->lock);
874                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
875                 if (video_src_priority == 1) {
876                         int num_src = ast_bridge_number_video_src(bridge);
877                         int echo = num_src > 1 ? 0 : 1;
878
879                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
880                 } else if (video_src_priority == 2) {
881                         softmix_pass_video_top_priority(bridge, frame);
882                 }
883                 break;
884         case AST_BRIDGE_VIDEO_MODE_SFU:
885                 /* Nothing special to do here, the bridge channel stream map will ensure the
886                  * video goes everywhere it needs to
887                  */
888                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
889                 break;
890         }
891 }
892
893 /*!
894  * \internal
895  * \brief Determine what to do with a voice frame.
896  * \since 12.0.0
897  *
898  * \param bridge Which bridge is getting the frame
899  * \param bridge_channel Which channel is writing the frame.
900  * \param frame What is being written.
901  *
902  * \return Nothing
903  */
904 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
905 {
906         struct softmix_channel *sc = bridge_channel->tech_pvt;
907         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
908         int totalsilence = 0;
909         int cur_energy = 0;
910         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
911                 bridge_channel->tech_args.silence_threshold :
912                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
913         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
914
915         /* Write the frame into the conference */
916         ast_mutex_lock(&sc->lock);
917
918         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
919                 /*
920                  * The incoming frame is not the expected format.  Update
921                  * the channel's translation path to get us slinear from
922                  * the new format for the next frame.
923                  *
924                  * There is the possibility that this frame is an old slinear
925                  * rate frame that was in flight when the softmix bridge
926                  * changed rates.  If so it will self correct on subsequent
927                  * frames.
928                  */
929                 ast_channel_lock(bridge_channel->chan);
930                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
931                         ast_channel_name(bridge_channel->chan),
932                         ast_format_get_name(frame->subclass.format),
933                         ast_format_get_name(sc->read_slin_format));
934                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
935                         sc->read_slin_format);
936                 ast_channel_unlock(bridge_channel->chan);
937         }
938
939         /* The channel will be leaving soon if there is no dsp. */
940         if (sc->dsp) {
941                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
942         }
943
944         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
945                 int cur_slot = sc->video_talker.energy_history_cur_slot;
946
947                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
948                 sc->video_talker.energy_accum += cur_energy;
949                 sc->video_talker.energy_history[cur_slot] = cur_energy;
950                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
951                 sc->video_talker.energy_history_cur_slot++;
952                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
953                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
954                 }
955         }
956
957         if (totalsilence < silence_threshold) {
958                 if (!sc->talking) {
959                         update_talking = 1;
960                 }
961                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
962         } else {
963                 if (sc->talking) {
964                         update_talking = 0;
965                 }
966                 sc->talking = 0;
967         }
968
969         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
970          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
971          * the audio by flushing the buffer before adding new audio in. */
972         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
973                 ast_slinfactory_flush(&sc->factory);
974         }
975
976         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
977          * is not determined to be talking. */
978         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
979                 ast_slinfactory_feed(&sc->factory, frame);
980         }
981
982         /* Alllll done */
983         ast_mutex_unlock(&sc->lock);
984
985         if (update_talking != -1) {
986                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
987         }
988 }
989
990 static int remove_all_original_streams(struct ast_stream_topology *dest,
991         const struct ast_stream_topology *source,
992         const struct ast_stream_topology *original)
993 {
994         int i;
995
996         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
997                 struct ast_stream *stream;
998                 int original_index;
999
1000                 stream = ast_stream_topology_get_stream(source, i);
1001
1002                 /* Mark the existing stream as removed so we get a new one, this will get
1003                  * reused on a subsequent renegotiation.
1004                  */
1005                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
1006                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
1007
1008                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
1009                                 struct ast_stream *removed;
1010
1011                                 /* Since the participant is still going to be in the bridge we
1012                                  * change the name so that routing does not attempt to route video
1013                                  * to this stream.
1014                                  */
1015                                 removed = ast_stream_clone(stream, "removed");
1016                                 if (!removed) {
1017                                         return -1;
1018                                 }
1019
1020                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
1021
1022                                 /* The destination topology can only ever contain the same, or more,
1023                                  * streams than the original so this is safe.
1024                                  */
1025                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
1026                                         ast_stream_free(removed);
1027                                         return -1;
1028                                 }
1029
1030                                 break;
1031                         }
1032                 }
1033         }
1034
1035         return 0;
1036 }
1037
1038 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
1039 {
1040         struct ast_stream_topology *source_video = NULL;
1041         struct ast_bridge_channel *participant;
1042         int res;
1043
1044         source_video = ast_stream_topology_alloc();
1045         if (!source_video) {
1046                 return;
1047         }
1048
1049         ast_channel_lock(source->chan);
1050         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
1051         ast_channel_unlock(source->chan);
1052         if (res) {
1053                 goto cleanup;
1054         }
1055
1056         AST_LIST_TRAVERSE(participants, participant, entry) {
1057                 struct ast_stream_topology *original_topology;
1058                 struct ast_stream_topology *participant_topology;
1059
1060                 if (participant == source) {
1061                         continue;
1062                 }
1063
1064                 ast_channel_lock(participant->chan);
1065                 original_topology = ast_stream_topology_clone(ast_channel_get_stream_topology(participant->chan));
1066                 ast_channel_unlock(participant->chan);
1067                 if (!original_topology) {
1068                         goto cleanup;
1069                 }
1070
1071                 participant_topology = ast_stream_topology_clone(original_topology);
1072                 if (!participant_topology) {
1073                         ast_stream_topology_free(original_topology);
1074                         goto cleanup;
1075                 }
1076
1077                 /* We add all the source streams back in, if any removed streams are already present they will
1078                  * get used first followed by appending new ones.
1079                  */
1080                 if (append_all_streams(participant_topology, source_video)) {
1081                         ast_stream_topology_free(participant_topology);
1082                         ast_stream_topology_free(original_topology);
1083                         goto cleanup;
1084                 }
1085
1086                 /* And the original existing streams get marked as removed. This causes the remote side to see
1087                  * a new stream for the source streams.
1088                  */
1089                 if (remove_all_original_streams(participant_topology, source_video, original_topology)) {
1090                         ast_stream_topology_free(participant_topology);
1091                         ast_stream_topology_free(original_topology);
1092                         goto cleanup;
1093                 }
1094
1095                 ast_channel_request_stream_topology_change(participant->chan, participant_topology, NULL);
1096                 ast_stream_topology_free(participant_topology);
1097                 ast_stream_topology_free(original_topology);
1098         }
1099
1100 cleanup:
1101         ast_stream_topology_free(source_video);
1102 }
1103
1104 /*!
1105  * \internal
1106  * \brief Determine what to do with a control frame.
1107  * \since 12.0.0
1108  *
1109  * \param bridge Which bridge is getting the frame
1110  * \param bridge_channel Which channel is writing the frame.
1111  * \param frame What is being written.
1112  *
1113  * \retval 0 Frame accepted into the bridge.
1114  * \retval -1 Frame needs to be deferred.
1115  */
1116 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1117 {
1118         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1119
1120         /*
1121          * XXX Softmix needs to use channel roles to determine what to
1122          * do with control frames.
1123          */
1124
1125         switch (frame->subclass.integer) {
1126         case AST_CONTROL_VIDUPDATE:
1127                 if (!bridge->softmix.video_mode.video_update_discard ||
1128                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1129                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1130                         softmix_data->last_video_update = ast_tvnow();
1131                 }
1132                 break;
1133         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1134                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1135                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1136                 }
1137                 break;
1138         default:
1139                 break;
1140         }
1141
1142         return 0;
1143 }
1144
1145 /*!
1146  * \internal
1147  * \brief Determine what to do with a frame written into the bridge.
1148  * \since 12.0.0
1149  *
1150  * \param bridge Which bridge is getting the frame
1151  * \param bridge_channel Which channel is writing the frame.
1152  * \param frame What is being written.
1153  *
1154  * \retval 0 Frame accepted into the bridge.
1155  * \retval -1 Frame needs to be deferred.
1156  *
1157  * \note On entry, bridge is already locked.
1158  */
1159 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1160 {
1161         int res = 0;
1162
1163         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1164                 /* "Accept" the frame and discard it. */
1165                 return 0;
1166         }
1167
1168         /*
1169          * XXX Softmix needs to use channel roles to determine who gets
1170          * what frame.  Possible roles: announcer, recorder, agent,
1171          * supervisor.
1172          */
1173         switch (frame->frametype) {
1174         case AST_FRAME_NULL:
1175                 /* "Accept" the frame and discard it. */
1176                 break;
1177         case AST_FRAME_DTMF_BEGIN:
1178         case AST_FRAME_DTMF_END:
1179                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1180                 break;
1181         case AST_FRAME_VOICE:
1182                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1183                 break;
1184         case AST_FRAME_VIDEO:
1185                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1186                 break;
1187         case AST_FRAME_CONTROL:
1188                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1189                 break;
1190         case AST_FRAME_BRIDGE_ACTION:
1191                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1192                 break;
1193         case AST_FRAME_BRIDGE_ACTION_SYNC:
1194                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1195                 ast_assert(0);
1196         default:
1197                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1198                 /* "Accept" the frame and discard it. */
1199                 break;
1200         }
1201
1202         return res;
1203 }
1204
1205 static void gather_softmix_stats(struct softmix_stats *stats,
1206         const struct softmix_bridge_data *softmix_data,
1207         struct ast_bridge_channel *bridge_channel)
1208 {
1209         int channel_native_rate;
1210
1211         /* Gather stats about channel sample rates. */
1212         ast_channel_lock(bridge_channel->chan);
1213         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1214                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1215         ast_channel_unlock(bridge_channel->chan);
1216
1217         if (stats->highest_supported_rate < channel_native_rate) {
1218                 stats->highest_supported_rate = channel_native_rate;
1219         }
1220         if (softmix_data->internal_rate < channel_native_rate) {
1221                 int i;
1222
1223                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1224                         if (stats->sample_rates[i] == channel_native_rate) {
1225                                 stats->num_channels[i]++;
1226                                 break;
1227                         } else if (!stats->sample_rates[i]) {
1228                                 stats->sample_rates[i] = channel_native_rate;
1229                                 stats->num_channels[i]++;
1230                                 break;
1231                         }
1232                 }
1233                 stats->num_above_internal_rate++;
1234         } else if (softmix_data->internal_rate == channel_native_rate) {
1235                 stats->num_at_internal_rate++;
1236         }
1237 }
1238
1239 /*!
1240  * \internal
1241  * \brief Analyse mixing statistics and change bridges internal rate
1242  * if necessary.
1243  *
1244  * \retval 0, no changes to internal rate
1245  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1246  */
1247 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1248                 struct softmix_bridge_data *softmix_data, int binaural_active)
1249 {
1250         int i;
1251
1252         if (binaural_active) {
1253                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1254         }
1255
1256         /*
1257          * Re-adjust the internal bridge sample rate if
1258          * 1. The bridge's internal sample rate is locked in at a sample
1259          *    rate other than the current sample rate being used.
1260          * 2. two or more channels support a higher sample rate
1261          * 3. no channels support the current sample rate or a higher rate
1262          */
1263         if (stats->locked_rate) {
1264                 /* if the rate is locked by the bridge, only update it if it differs
1265                  * from the current rate we are using. */
1266                 if (softmix_data->internal_rate != stats->locked_rate) {
1267                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1268                                 softmix_data->internal_rate, stats->locked_rate);
1269                         softmix_data->internal_rate = stats->locked_rate;
1270                         return 1;
1271                 }
1272         } else if (stats->num_above_internal_rate >= 2) {
1273                 /* the highest rate is just used as a starting point */
1274                 unsigned int best_rate = stats->highest_supported_rate;
1275                 int best_index = -1;
1276
1277                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1278                         if (stats->num_channels[i]) {
1279                                 break;
1280                         }
1281                         if (2 <= stats->num_channels[i]) {
1282                                 /* Two or more channels support this rate. */
1283                                 if (best_index == -1
1284                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1285                                         /*
1286                                          * best_rate starts out being the first sample rate
1287                                          * greater than the internal sample rate that two or
1288                                          * more channels support.
1289                                          *
1290                                          * or
1291                                          *
1292                                          * There are multiple rates above the internal rate
1293                                          * and this rate is higher than the previous rate two
1294                                          * or more channels support.
1295                                          */
1296                                         best_rate = stats->sample_rates[i];
1297                                         best_index = i;
1298                                 }
1299                         } else if (best_index == -1) {
1300                                 /*
1301                                  * It is possible that multiple channels exist with native sample
1302                                  * rates above the internal sample rate, but none of those channels
1303                                  * have the same rate in common.  In this case, the lowest sample
1304                                  * rate among those channels is picked. Over time as additional
1305                                  * statistic runs are made the internal sample rate number will
1306                                  * adjust to the most optimal sample rate, but it may take multiple
1307                                  * iterations.
1308                                  */
1309                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1310                         }
1311                 }
1312
1313                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1314                         softmix_data->internal_rate, best_rate);
1315                 softmix_data->internal_rate = best_rate;
1316                 return 1;
1317         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1318                 /* In this case, the highest supported rate is actually lower than the internal rate */
1319                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1320                         softmix_data->internal_rate, stats->highest_supported_rate);
1321                 softmix_data->internal_rate = stats->highest_supported_rate;
1322                 return 1;
1323         }
1324         return 0;
1325 }
1326
1327 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1328                 unsigned int starting_num_entries, unsigned int binaural_active)
1329 {
1330         memset(mixing_array, 0, sizeof(*mixing_array));
1331         mixing_array->max_num_entries = starting_num_entries;
1332         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1333                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1334                 return -1;
1335         }
1336         if (binaural_active) {
1337                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1338                                 sizeof(struct convolve_channel_pair *)))) {
1339                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1340                         return -1;
1341                 }
1342         }
1343         return 0;
1344 }
1345
1346 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1347                 unsigned int binaural_active)
1348 {
1349         ast_free(mixing_array->buffers);
1350         if (binaural_active) {
1351                 ast_free(mixing_array->chan_pairs);
1352         }
1353 }
1354
1355 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1356                 unsigned int num_entries, unsigned int binaural_active)
1357 {
1358         int16_t **tmp;
1359
1360         /* give it some room to grow since memory is cheap but allocations can be expensive */
1361         mixing_array->max_num_entries = num_entries;
1362         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1363                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1364                 return -1;
1365         }
1366         if (binaural_active) {
1367                 struct convolve_channel_pair **tmp2;
1368                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1369                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1370                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1371                         return -1;
1372                 }
1373                 mixing_array->chan_pairs = tmp2;
1374         }
1375         mixing_array->buffers = tmp;
1376         return 0;
1377 }
1378
1379 /*!
1380  * \brief Mixing loop.
1381  *
1382  * \retval 0 on success
1383  * \retval -1 on failure
1384  */
1385 static int softmix_mixing_loop(struct ast_bridge *bridge)
1386 {
1387         struct softmix_stats stats = { { 0 }, };
1388         struct softmix_mixing_array mixing_array;
1389         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1390         struct ast_timer *timer;
1391         struct softmix_translate_helper trans_helper;
1392         int16_t buf[MAX_DATALEN];
1393 #ifdef BINAURAL_RENDERING
1394         int16_t bin_buf[MAX_DATALEN];
1395         int16_t ann_buf[MAX_DATALEN];
1396 #endif
1397         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1398         int timingfd;
1399         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1400         unsigned int idx;
1401         unsigned int x;
1402         int res = -1;
1403
1404         timer = softmix_data->timer;
1405         timingfd = ast_timer_fd(timer);
1406         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1407         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1408
1409         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1410         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1411                         bridge->softmix.binaural_active)) {
1412                 goto softmix_cleanup;
1413         }
1414
1415         /*
1416          * XXX Softmix needs to use channel roles to determine who gets
1417          * what audio mixed.
1418          */
1419         while (!softmix_data->stop && bridge->num_active) {
1420                 struct ast_bridge_channel *bridge_channel;
1421                 int timeout = -1;
1422                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1423                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1424                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1425
1426                 if (softmix_datalen > MAX_DATALEN) {
1427                         /* This should NEVER happen, but if it does we need to know about it. Almost
1428                          * all the memcpys used during this process depend on this assumption.  Rather
1429                          * than checking this over and over again through out the code, this single
1430                          * verification is done on each iteration. */
1431                         ast_log(LOG_WARNING,
1432                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1433                                 bridge->uniqueid);
1434                         goto softmix_cleanup;
1435                 }
1436
1437                 /* Grow the mixing array buffer as participants are added. */
1438                 if (mixing_array.max_num_entries < bridge->num_channels
1439                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1440                                         bridge->softmix.binaural_active)) {
1441                         goto softmix_cleanup;
1442                 }
1443
1444                 /* init the number of buffers stored in the mixing array to 0.
1445                  * As buffers are added for mixing, this number is incremented. */
1446                 mixing_array.used_entries = 0;
1447
1448                 /* These variables help determine if a rate change is required */
1449                 if (!stat_iteration_counter) {
1450                         memset(&stats, 0, sizeof(stats));
1451                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1452                 }
1453
1454                 /* If the sample rate has changed, update the translator helper */
1455                 if (update_all_rates) {
1456                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1457                 }
1458
1459 #ifdef BINAURAL_RENDERING
1460                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1461 #endif
1462
1463                 /* Go through pulling audio from each factory that has it available */
1464                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1465                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1466
1467                         if (!sc) {
1468                                 /* This channel failed to join successfully. */
1469                                 continue;
1470                         }
1471
1472                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1473                         if (update_all_rates) {
1474                                 set_softmix_bridge_data(softmix_data->internal_rate,
1475                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1476                         }
1477
1478                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1479                         if (!stat_iteration_counter) {
1480                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1481                         }
1482
1483                         /* if the channel is suspended, don't check for audio, but still gather stats */
1484                         if (bridge_channel->suspended) {
1485                                 continue;
1486                         }
1487
1488                         /* Try to get audio from the factory if available */
1489                         ast_mutex_lock(&sc->lock);
1490                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1491 #ifdef BINAURAL_RENDERING
1492                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1493                                                 ast_channel_name(bridge_channel->chan));
1494 #endif
1495                                 mixing_array.used_entries++;
1496                         }
1497                         ast_mutex_unlock(&sc->lock);
1498                 }
1499
1500                 /* mix it like crazy (non binaural channels)*/
1501                 memset(buf, 0, softmix_datalen);
1502                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1503                         for (x = 0; x < softmix_samples; ++x) {
1504                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1505                         }
1506                 }
1507
1508 #ifdef BINAURAL_RENDERING
1509                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1510 #endif
1511
1512                 /* Next step go through removing the channel's own audio and creating a good frame... */
1513                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1514                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1515
1516                         if (!sc || bridge_channel->suspended) {
1517                                 /* This channel failed to join successfully or is suspended. */
1518                                 continue;
1519                         }
1520
1521                         ast_mutex_lock(&sc->lock);
1522
1523                         /* Make SLINEAR write frame from local buffer */
1524                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1525                                 "Replace softmix channel slin format");
1526 #ifdef BINAURAL_RENDERING
1527                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1528                                         && sc->binaural) {
1529                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1530                                                 softmix_samples, buf);
1531                         } else
1532 #endif
1533                         {
1534                                 sc->write_frame.datalen = softmix_datalen;
1535                                 sc->write_frame.samples = softmix_samples;
1536                                 memcpy(sc->final_buf, buf, softmix_datalen);
1537                         }
1538                         /* process the softmix channel's new write audio */
1539                         softmix_process_write_audio(&trans_helper,
1540                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1541                                         softmix_data->default_sample_size);
1542
1543                         ast_mutex_unlock(&sc->lock);
1544
1545                         /* A frame is now ready for the channel. */
1546                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1547                 }
1548
1549                 update_all_rates = 0;
1550                 if (!stat_iteration_counter) {
1551                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1552                                         bridge->softmix.binaural_active);
1553                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1554                 }
1555                 stat_iteration_counter--;
1556
1557                 ast_bridge_unlock(bridge);
1558                 /* cleanup any translation frame data from the previous mixing iteration. */
1559                 softmix_translate_helper_cleanup(&trans_helper);
1560                 /* Wait for the timing source to tell us to wake up and get things done */
1561                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1562                 if (ast_timer_ack(timer, 1) < 0) {
1563                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1564                                 bridge->uniqueid);
1565                         ast_bridge_lock(bridge);
1566                         goto softmix_cleanup;
1567                 }
1568                 ast_bridge_lock(bridge);
1569
1570                 /* make sure to detect mixing interval changes if they occur. */
1571                 if (bridge->softmix.internal_mixing_interval
1572                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1573                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1574                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1575                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1576                 }
1577         }
1578
1579         res = 0;
1580
1581 softmix_cleanup:
1582         softmix_translate_helper_destroy(&trans_helper);
1583         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1584         return res;
1585 }
1586
1587 /*!
1588  * \internal
1589  * \brief Mixing thread.
1590  * \since 12.0.0
1591  *
1592  * \note The thread does not have its own reference to the
1593  * bridge.  The lifetime of the thread is tied to the lifetime
1594  * of the mixing technology association with the bridge.
1595  */
1596 static void *softmix_mixing_thread(void *data)
1597 {
1598         struct softmix_bridge_data *softmix_data = data;
1599         struct ast_bridge *bridge = softmix_data->bridge;
1600
1601         ast_bridge_lock(bridge);
1602         if (bridge->callid) {
1603                 ast_callid_threadassoc_add(bridge->callid);
1604         }
1605
1606         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1607
1608         while (!softmix_data->stop) {
1609                 if (!bridge->num_active) {
1610                         /* Wait for something to happen to the bridge. */
1611                         ast_bridge_unlock(bridge);
1612                         ast_mutex_lock(&softmix_data->lock);
1613                         if (!softmix_data->stop) {
1614                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1615                         }
1616                         ast_mutex_unlock(&softmix_data->lock);
1617                         ast_bridge_lock(bridge);
1618                         continue;
1619                 }
1620
1621                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1622 #ifndef BINAURAL_RENDERING
1623                         ast_bridge_lock(bridge);
1624                         bridge->softmix.binaural_active = 0;
1625                         ast_bridge_unlock(bridge);
1626                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1627                                         "compiled.\n", bridge->uniqueid);
1628 #else
1629                         /* Set and init binaural data if binaural is activated in the configuration. */
1630                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1631                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1632                                         softmix_data->internal_mixing_interval);
1633                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1634                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1635                                         == -1) {
1636                                 ast_bridge_lock(bridge);
1637                                 bridge->softmix.binaural_active = 0;
1638                                 ast_bridge_unlock(bridge);
1639                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1640                                                 "binaural processing,  Will only process mono audio.\n",
1641                                                 bridge->uniqueid);
1642                         }
1643                         softmix_data->binaural_init = 1;
1644 #endif
1645                 }
1646
1647                 if (softmix_mixing_loop(bridge)) {
1648                         /*
1649                          * A mixing error occurred.  Sleep and try again later so we
1650                          * won't flood the logs.
1651                          */
1652                         ast_bridge_unlock(bridge);
1653                         sleep(1);
1654                         ast_bridge_lock(bridge);
1655                 }
1656         }
1657
1658         ast_bridge_unlock(bridge);
1659
1660         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1661
1662         return NULL;
1663 }
1664
1665 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1666 {
1667         if (softmix_data->timer) {
1668                 ast_timer_close(softmix_data->timer);
1669                 softmix_data->timer = NULL;
1670         }
1671         ast_mutex_destroy(&softmix_data->lock);
1672         ast_cond_destroy(&softmix_data->cond);
1673         ast_free(softmix_data);
1674 }
1675
1676 /*! \brief Function called when a bridge is created */
1677 static int softmix_bridge_create(struct ast_bridge *bridge)
1678 {
1679         struct softmix_bridge_data *softmix_data;
1680
1681         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1682         if (!softmix_data) {
1683                 return -1;
1684         }
1685         softmix_data->bridge = bridge;
1686         ast_mutex_init(&softmix_data->lock);
1687         ast_cond_init(&softmix_data->cond, NULL);
1688         softmix_data->timer = ast_timer_open();
1689         if (!softmix_data->timer) {
1690                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1691                 softmix_bridge_data_destroy(softmix_data);
1692                 return -1;
1693         }
1694         /* start at minimum rate, let it grow from there */
1695         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1696         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1697
1698 #ifdef BINAURAL_RENDERING
1699         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1700                         softmix_data->internal_mixing_interval);
1701 #endif
1702
1703         bridge->tech_pvt = softmix_data;
1704
1705         /* Start the mixing thread. */
1706         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1707                 softmix_data)) {
1708                 softmix_data->thread = AST_PTHREADT_NULL;
1709                 softmix_bridge_data_destroy(softmix_data);
1710                 bridge->tech_pvt = NULL;
1711                 return -1;
1712         }
1713
1714         return 0;
1715 }
1716
1717 /*!
1718  * \internal
1719  * \brief Request the softmix mixing thread stop.
1720  * \since 12.0.0
1721  *
1722  * \param bridge Which bridge is being stopped.
1723  *
1724  * \return Nothing
1725  */
1726 static void softmix_bridge_stop(struct ast_bridge *bridge)
1727 {
1728         struct softmix_bridge_data *softmix_data;
1729
1730         softmix_data = bridge->tech_pvt;
1731         if (!softmix_data) {
1732                 return;
1733         }
1734
1735         ast_mutex_lock(&softmix_data->lock);
1736         softmix_data->stop = 1;
1737         ast_mutex_unlock(&softmix_data->lock);
1738 }
1739
1740 /*! \brief Function called when a bridge is destroyed */
1741 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1742 {
1743         struct softmix_bridge_data *softmix_data;
1744         pthread_t thread;
1745
1746         softmix_data = bridge->tech_pvt;
1747         if (!softmix_data) {
1748                 return;
1749         }
1750
1751         /* Stop the mixing thread. */
1752         ast_mutex_lock(&softmix_data->lock);
1753         softmix_data->stop = 1;
1754         ast_cond_signal(&softmix_data->cond);
1755         thread = softmix_data->thread;
1756         softmix_data->thread = AST_PTHREADT_NULL;
1757         ast_mutex_unlock(&softmix_data->lock);
1758         if (thread != AST_PTHREADT_NULL) {
1759                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1760                 pthread_join(thread, NULL);
1761         }
1762 #ifdef BINAURAL_RENDERING
1763         free_convolve_data(&softmix_data->convolve);
1764 #endif
1765         softmix_bridge_data_destroy(softmix_data);
1766         bridge->tech_pvt = NULL;
1767 }
1768
1769 /*!
1770  * \brief Map a source stream to all of its destination streams.
1771  *
1772  * \param source_stream_name Name of the source stream
1773  * \param source_channel_name Name of channel where the source stream originates
1774  * \param bridge_stream_position The slot in the bridge where source video will come from
1775  * \param participants The bridge_channels in the bridge
1776  */
1777 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1778         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1779 {
1780         struct ast_bridge_channel *participant;
1781
1782         AST_LIST_TRAVERSE(participants, participant, entry) {
1783                 int i;
1784                 struct ast_stream_topology *topology;
1785
1786                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1787                         continue;
1788                 }
1789
1790                 ast_bridge_channel_lock(participant);
1791                 ast_channel_lock(participant->chan);
1792                 topology = ast_channel_get_stream_topology(participant->chan);
1793
1794                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1795                         struct ast_stream *stream;
1796
1797                         stream = ast_stream_topology_get_stream(topology, i);
1798                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1799                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1800                                 break;
1801                         }
1802                 }
1803                 ast_channel_unlock(participant->chan);
1804                 ast_bridge_channel_unlock(participant);
1805         }
1806 }
1807
1808 /*!
1809  * \brief stream_topology_changed callback
1810  *
1811  * For most video modes, nothing beyond the ordinary is required.
1812  * For the SFU case, though, we need to completely remap the streams
1813  * in order to ensure video gets directed where it is expected to go.
1814  *
1815  * \param bridge The bridge
1816  * \param bridge_channel Channel whose topology has changed
1817  */
1818 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1819 {
1820         struct ast_bridge_channel *participant;
1821         struct ast_vector_int media_types;
1822         int nths[AST_MEDIA_TYPE_END] = {0};
1823
1824         switch (bridge->softmix.video_mode.mode) {
1825         case AST_BRIDGE_VIDEO_MODE_NONE:
1826         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1827         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1828         default:
1829                 ast_bridge_channel_stream_map(bridge_channel);
1830                 return;
1831         case AST_BRIDGE_VIDEO_MODE_SFU:
1832                 break;
1833         }
1834
1835         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1836
1837         /* First traversal: re-initialize all of the participants' stream maps */
1838         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1839                 ast_bridge_channel_lock(participant);
1840                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1841                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1842                 ast_bridge_channel_unlock(participant);
1843         }
1844
1845         /* Second traversal: Map specific video channels from their source to their destinations.
1846          *
1847          * This is similar to what is done in ast_stream_topology_map(),
1848          * except that video channels are handled differently.  Each video
1849          * source has it's own unique index on the bridge.  This way, a
1850          * particular channel's source video can be distributed to the
1851          * appropriate destination streams on the other channels.
1852          */
1853         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1854                 int i;
1855                 struct ast_stream_topology *topology;
1856
1857                 ast_bridge_channel_lock(participant);
1858                 ast_channel_lock(participant->chan);
1859
1860                 topology = ast_channel_get_stream_topology(participant->chan);
1861                 if (topology) {
1862                         /*
1863                          * Sigh.  We have to clone to avoid deadlock in
1864                          * map_source_to_destinations() because topology
1865                          * is not an ao2 object.
1866                          */
1867                         topology = ast_stream_topology_clone(topology);
1868                 }
1869                 if (!topology) {
1870                         /* Oh, my, we are in trouble. */
1871                         ast_channel_unlock(participant->chan);
1872                         ast_bridge_channel_unlock(participant);
1873                         continue;
1874                 }
1875
1876                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1877                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1878
1879                         if (is_video_source(stream)) {
1880                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1881                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1882                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1883                                 /*
1884                                  * Unlock the channel and participant to prevent
1885                                  * potential deadlock in map_source_to_destinations().
1886                                  */
1887                                 ast_channel_unlock(participant->chan);
1888                                 ast_bridge_channel_unlock(participant);
1889                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1890                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1891                                 ast_bridge_channel_lock(participant);
1892                                 ast_channel_lock(participant->chan);
1893                         } else if (is_video_dest(stream, NULL, NULL)) {
1894                                 /* We expect to never read media from video destination channels, but just
1895                                  * in case, we should set their to_bridge value to -1.
1896                                  */
1897                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1898                         } else {
1899                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1900                                  * be factored out in some way
1901                                  */
1902                                 enum ast_media_type type = ast_stream_get_type(stream);
1903                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1904                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1905
1906                                 if (index == -1) {
1907                                         AST_VECTOR_APPEND(&media_types, type);
1908                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1909                                 }
1910
1911                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1912                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1913                         }
1914                 }
1915
1916                 ast_stream_topology_free(topology);
1917
1918                 ast_channel_unlock(participant->chan);
1919                 ast_bridge_channel_unlock(participant);
1920         }
1921 }
1922
1923 static struct ast_bridge_technology softmix_bridge = {
1924         .name = "softmix",
1925         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1926         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1927         .create = softmix_bridge_create,
1928         .stop = softmix_bridge_stop,
1929         .destroy = softmix_bridge_destroy,
1930         .join = softmix_bridge_join,
1931         .leave = softmix_bridge_leave,
1932         .unsuspend = softmix_bridge_unsuspend,
1933         .write = softmix_bridge_write,
1934         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1935 };
1936
1937 #ifdef TEST_FRAMEWORK
1938 struct stream_parameters {
1939         const char *name;
1940         const char *formats;
1941         enum ast_media_type type;
1942 };
1943
1944 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1945 {
1946         struct ast_stream_topology *topology;
1947         size_t i;
1948
1949         topology = ast_stream_topology_alloc();
1950         if (!topology) {
1951                 return NULL;
1952         }
1953
1954         for (i = 0; i < num_streams; ++i) {
1955                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1956                 struct ast_stream *stream;
1957
1958                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1959                 if (!caps) {
1960                         goto fail;
1961                 }
1962                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1963                         goto fail;
1964                 }
1965                 stream = ast_stream_alloc(params[i].name, params[i].type);
1966                 if (!stream) {
1967                         goto fail;
1968                 }
1969                 ast_stream_set_formats(stream, caps);
1970                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1971                         ast_stream_free(stream);
1972                         goto fail;
1973                 }
1974         }
1975
1976         return topology;
1977
1978 fail:
1979         ast_stream_topology_free(topology);
1980         return NULL;
1981 }
1982
1983 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1984         const struct stream_parameters *params)
1985 {
1986         struct ast_format_cap *stream_caps;
1987         struct ast_format_cap *params_caps;
1988
1989         if (ast_stream_get_type(stream) != params->type) {
1990                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1991                         ast_codec_media_type2str(params->type),
1992                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1993                 return -1;
1994         }
1995         if (strcmp(ast_stream_get_name(stream), params->name)) {
1996                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1997                         params->name, ast_stream_get_name(stream));
1998                 return -1;
1999         }
2000
2001         stream_caps = ast_stream_get_formats(stream);
2002         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
2003         if (!params_caps) {
2004                 ast_test_status_update(test, "Allocation error on capabilities\n");
2005                 return -1;
2006         }
2007         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
2008
2009         if (ast_format_cap_identical(stream_caps, params_caps)) {
2010                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
2011                         ast_stream_get_name(stream));
2012                 ao2_cleanup(params_caps);
2013                 return -1;
2014         }
2015
2016         ao2_cleanup(params_caps);
2017         return 0;
2018 }
2019
2020 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
2021         const struct stream_parameters *params, size_t num_streams)
2022 {
2023         int i;
2024
2025         if (ast_stream_topology_get_count(topology) < num_streams) {
2026                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
2027                         ast_stream_topology_get_count(topology), num_streams);
2028                 return -1;
2029         }
2030
2031         for (i = 0; i < ARRAY_LEN(params); ++i) {
2032                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
2033                         return -1;
2034                 }
2035         }
2036
2037         return 0;
2038 }
2039
2040 AST_TEST_DEFINE(sfu_append_source_streams)
2041 {
2042         enum ast_test_result_state res = AST_TEST_FAIL;
2043         static const struct stream_parameters bob_streams[] = {
2044                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2045                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2046         };
2047         static const struct stream_parameters alice_streams[] = {
2048                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
2049                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2050         };
2051         static const struct stream_parameters alice_dest_stream = {
2052                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
2053         };
2054         static const struct stream_parameters bob_dest_stream = {
2055                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2056         };
2057         struct ast_stream_topology *topology_alice = NULL;
2058         struct ast_stream_topology *topology_bob = NULL;
2059
2060         switch (cmd) {
2061         case TEST_INIT:
2062                 info->name = "sfu_append_source_streams";
2063                 info->category = "/bridges/bridge_softmix/";
2064                 info->summary = "Test appending of video streams";
2065                 info->description =
2066                         "This tests does stuff.";
2067                 return AST_TEST_NOT_RUN;
2068         case TEST_EXECUTE:
2069                 break;
2070         }
2071
2072         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2073         if (!topology_alice) {
2074                 goto end;
2075         }
2076
2077         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2078         if (!topology_bob) {
2079                 goto end;
2080         }
2081
2082         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2083                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2084                 goto end;
2085         }
2086
2087         if (ast_stream_topology_get_count(topology_alice) != 3) {
2088                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2089                         ast_stream_topology_get_count(topology_alice), 3);
2090                 goto end;
2091         }
2092
2093         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2094                 goto end;
2095         }
2096
2097         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2098                 goto end;
2099         }
2100
2101         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2102                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2103                 goto end;
2104         }
2105
2106         if (ast_stream_topology_get_count(topology_bob) != 3) {
2107                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2108                         ast_stream_topology_get_count(topology_bob), 3);
2109                 goto end;
2110         }
2111
2112         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2113                 goto end;
2114         }
2115
2116         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2117                 goto end;
2118         }
2119
2120         res = AST_TEST_PASS;
2121
2122 end:
2123         ast_stream_topology_free(topology_alice);
2124         ast_stream_topology_free(topology_bob);
2125         return res;
2126 }
2127
2128 AST_TEST_DEFINE(sfu_remove_destination_streams)
2129 {
2130         enum ast_test_result_state res = AST_TEST_FAIL;
2131         static const struct stream_parameters params[] = {
2132                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2133                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2134                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2135                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2136         };
2137         static const struct {
2138                 const char *channel_name;
2139                 int num_streams;
2140                 int params_index[4];
2141         } removal_results[] = {
2142                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2143                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2144                 { "", 4, { 0, 1, 2, 3 }, },
2145         };
2146         struct ast_stream_topology *orig = NULL;
2147         struct ast_stream_topology *result = NULL;
2148         int i;
2149
2150         switch (cmd) {
2151         case TEST_INIT:
2152                 info->name = "sfu_remove_destination_streams";
2153                 info->category = "/bridges/bridge_softmix/";
2154                 info->summary = "Test removal of destination video streams";
2155                 info->description =
2156                         "This tests does stuff.";
2157                 return AST_TEST_NOT_RUN;
2158         case TEST_EXECUTE:
2159                 break;
2160         }
2161
2162         orig = build_topology(params, ARRAY_LEN(params));
2163         if (!orig) {
2164                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2165                 goto end;
2166         }
2167
2168         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2169                 int j;
2170
2171                 result = ast_stream_topology_alloc();
2172                 if (!result) {
2173                         ast_test_status_update(test, "Unable to allocate result stream topology\n");
2174                         goto end;
2175                 }
2176
2177                 if (remove_destination_streams(result, removal_results[i].channel_name, orig)) {
2178                         ast_test_status_update(test, "Failure while attempting to remove video streams\n");
2179                         goto end;
2180                 }
2181
2182                 if (ast_stream_topology_get_count(result) != removal_results[i].num_streams) {
2183                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2184                                 ast_stream_topology_get_count(result), removal_results[i].num_streams);
2185                         goto end;
2186                 }
2187
2188                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2189                         struct ast_stream *actual;
2190                         struct ast_stream *expected;
2191                         int orig_index;
2192
2193                         actual = ast_stream_topology_get_stream(result, j);
2194
2195                         orig_index = removal_results[i].params_index[j];
2196                         expected = ast_stream_topology_get_stream(orig, orig_index);
2197
2198                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2199                                 ast_stream_get_formats(expected))) {
2200                                 struct ast_str *expected_str;
2201                                 struct ast_str *actual_str;
2202
2203                                 expected_str = ast_str_alloca(64);
2204                                 actual_str = ast_str_alloca(64);
2205
2206                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2207                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2208                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2209                                 goto end;
2210                         }
2211
2212                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2213                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2214                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2215                                 goto end;
2216                         }
2217                 }
2218         }
2219
2220         res = AST_TEST_PASS;
2221
2222 end:
2223         ast_stream_topology_free(orig);
2224         ast_stream_topology_free(result);
2225         return res;
2226 }
2227
2228 #endif
2229
2230 static int unload_module(void)
2231 {
2232         ast_bridge_technology_unregister(&softmix_bridge);
2233         AST_TEST_UNREGISTER(sfu_append_source_streams);
2234         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2235         return 0;
2236 }
2237
2238 static int load_module(void)
2239 {
2240         if (ast_bridge_technology_register(&softmix_bridge)) {
2241                 unload_module();
2242                 return AST_MODULE_LOAD_DECLINE;
2243         }
2244         AST_TEST_REGISTER(sfu_append_source_streams);
2245         AST_TEST_REGISTER(sfu_remove_destination_streams);
2246         return AST_MODULE_LOAD_SUCCESS;
2247 }
2248
2249 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");