bridge_softmix: Reduce topology cloning and improve renegotiation.
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 #include "asterisk/stream.h"
37 #include "asterisk/test.h"
38 #include "asterisk/vector.h"
39 #include "bridge_softmix/include/bridge_softmix_internal.h"
40
41 /*! The minimum sample rate of the bridge. */
42 #define SOFTMIX_MIN_SAMPLE_RATE 8000    /* 8 kHz sample rate */
43
44 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
45 #define DEFAULT_SOFTMIX_INTERVAL 20
46
47 /*! \brief Size of the buffer used for sample manipulation */
48 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
49
50 /*! \brief Number of samples we are dealing with */
51 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
52
53 /*! \brief Number of mixing iterations to perform between gathering statistics. */
54 #define SOFTMIX_STAT_INTERVAL 100
55
56 /* This is the threshold in ms at which a channel's own audio will stop getting
57  * mixed out its own write audio stream because it is not talking. */
58 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
59 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
60
61 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
62 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
63 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
64
65 struct softmix_stats {
66         /*! Each index represents a sample rate used above the internal rate. */
67         unsigned int sample_rates[16];
68         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
69         unsigned int num_channels[16];
70         /*! The number of channels above the internal sample rate */
71         unsigned int num_above_internal_rate;
72         /*! The number of channels at the internal sample rate */
73         unsigned int num_at_internal_rate;
74         /*! The absolute highest sample rate preferred by any channel in the bridge */
75         unsigned int highest_supported_rate;
76         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
77         unsigned int locked_rate;
78 };
79
80 struct softmix_translate_helper_entry {
81         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
82                                                                   and re-init if it was usable. */
83         struct ast_format *dst_format; /*!< The destination format for this helper */
84         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
85         struct ast_frame *out_frame; /*!< The output frame from the last translation */
86         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
87 };
88
89 struct softmix_translate_helper {
90         struct ast_format *slin_src; /*!< the source format expected for all the translators */
91         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
92 };
93
94 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
95 {
96         struct softmix_translate_helper_entry *entry;
97         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
98                 return NULL;
99         }
100         entry->dst_format = ao2_bump(dst);
101         /* initialize this to one so that the first time through the cleanup code after
102            allocation it won't be removed from the entry list */
103         entry->num_times_requested = 1;
104         return entry;
105 }
106
107 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
108 {
109         ao2_cleanup(entry->dst_format);
110
111         if (entry->trans_pvt) {
112                 ast_translator_free_path(entry->trans_pvt);
113         }
114         if (entry->out_frame) {
115                 ast_frfree(entry->out_frame);
116         }
117         ast_free(entry);
118         return NULL;
119 }
120
121 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
122 {
123         memset(trans_helper, 0, sizeof(*trans_helper));
124         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
125 }
126
127 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
128 {
129         struct softmix_translate_helper_entry *entry;
130
131         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
132                 softmix_translate_helper_free_entry(entry);
133         }
134 }
135
136 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
137 {
138         struct softmix_translate_helper_entry *entry;
139
140         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
141         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
142                 if (entry->trans_pvt) {
143                         ast_translator_free_path(entry->trans_pvt);
144                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
145                                 AST_LIST_REMOVE_CURRENT(entry);
146                                 entry = softmix_translate_helper_free_entry(entry);
147                         }
148                 }
149         }
150         AST_LIST_TRAVERSE_SAFE_END;
151 }
152
153 /*!
154  * \internal
155  * \brief Get the next available audio on the softmix channel's read stream
156  * and determine if it should be mixed out or not on the write stream.
157  *
158  * \retval pointer to buffer containing the exact number of samples requested on success.
159  * \retval NULL if no samples are present
160  */
161 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
162 {
163         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
164                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
165                 sc->have_audio = 1;
166                 return sc->our_buf;
167         }
168         sc->have_audio = 0;
169         return NULL;
170 }
171
172 /*!
173  * \internal
174  * \brief Process a softmix channel's write audio
175  *
176  * \details This function will remove the channel's talking from its own audio if present and
177  * possibly even do the channel's write translation for it depending on how many other
178  * channels use the same write format.
179  */
180 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
181         struct ast_format *raw_write_fmt,
182         struct softmix_channel *sc, unsigned int default_sample_size)
183 {
184         struct softmix_translate_helper_entry *entry = NULL;
185         int i;
186
187         /* If we provided audio that was not determined to be silence,
188          * then take it out while in slinear format. */
189         if (sc->have_audio && sc->talking && !sc->binaural) {
190                 for (i = 0; i < sc->write_frame.samples; i++) {
191                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
192                 }
193                 /* check to see if any entries exist for the format. if not we'll want
194                    to remove it during cleanup */
195                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
196                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
197                                 ++entry->num_times_requested;
198                                 break;
199                         }
200                 }
201                 /* do not do any special write translate optimization if we had to make
202                  * a special mix for them to remove their own audio. */
203                 return;
204         } else if (sc->have_audio && sc->talking && sc->binaural > 0) {
205                 /*
206                  * Binaural audio requires special saturated substract since we have two
207                  * audio signals per channel now.
208                  */
209                 softmix_process_write_binaural_audio(sc, default_sample_size);
210                 return;
211         }
212
213         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
214            of translation paths and track the number of references for each type. Each one of the same
215            type should be able to use the same out_frame. Since the optimization is only necessary for
216            multiple channels (>=2) using the same codec make sure resources are allocated only when
217            needed and released when not (see also softmix_translate_helper_cleanup */
218         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
219                 if (sc->binaural != 0) {
220                         continue;
221                 }
222                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
223                         entry->num_times_requested++;
224                 } else {
225                         continue;
226                 }
227                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
228                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
229                 }
230                 if (entry->trans_pvt && !entry->out_frame) {
231                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
232                 }
233                 if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
234                                 && entry->out_frame->datalen < MAX_DATALEN) {
235                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
236                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
237                         sc->write_frame.datalen = entry->out_frame->datalen;
238                         sc->write_frame.samples = entry->out_frame->samples;
239                 }
240                 break;
241         }
242
243         /* add new entry into list if this format destination was not matched. */
244         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
245                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
246         }
247 }
248
249 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
250 {
251         struct softmix_translate_helper_entry *entry;
252
253         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
254                 /* if it hasn't been requested then remove it */
255                 if (!entry->num_times_requested) {
256                         AST_LIST_REMOVE_CURRENT(entry);
257                         softmix_translate_helper_free_entry(entry);
258                         continue;
259                 }
260
261                 if (entry->out_frame) {
262                         ast_frfree(entry->out_frame);
263                         entry->out_frame = NULL;
264                 }
265
266                 /* nothing is optimized for a single path reference, so there is
267                    no reason to continue to hold onto the codec */
268                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
269                         ast_translator_free_path(entry->trans_pvt);
270                         entry->trans_pvt = NULL;
271                 }
272
273                 /* for each iteration (a mixing run) in the bridge softmix thread the number
274                    of references to a given entry is recalculated, so reset the number of
275                    times requested */
276                 entry->num_times_requested = 0;
277         }
278         AST_LIST_TRAVERSE_SAFE_END;
279 }
280
281 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
282 {
283         struct softmix_channel *sc = bridge_channel->tech_pvt;
284         struct ast_format *slin_format;
285         int setup_fail;
286
287 #ifdef BINAURAL_RENDERING
288         if (interval != BINAURAL_MIXING_INTERVAL) {
289                 interval = BINAURAL_MIXING_INTERVAL;
290         }
291 #endif
292
293         /* The callers have already ensured that sc is never NULL. */
294         ast_assert(sc != NULL);
295
296         slin_format = ast_format_cache_get_slin_by_rate(rate);
297
298         ast_mutex_lock(&sc->lock);
299         if (reset) {
300                 ast_slinfactory_destroy(&sc->factory);
301                 ast_dsp_free(sc->dsp);
302         }
303
304         /* Setup write frame parameters */
305         sc->write_frame.frametype = AST_FRAME_VOICE;
306         /*
307          * NOTE: The write_frame format holds a reference because translation
308          * could be needed and the format changed to the translated format
309          * for the channel.  The translated format may not be a
310          * static cached format.
311          */
312         ao2_replace(sc->write_frame.subclass.format, slin_format);
313         sc->write_frame.data.ptr = sc->final_buf;
314         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
315         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
316
317         /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
318         sc->rate = rate;
319
320         /* If the channel will contain binaural data we will set a identifier in the channel
321          * if set_binaural == -1 this is just a sample rate update, will ignore it. */
322         if (set_binaural == 1) {
323                 sc->binaural = 1;
324         } else if (set_binaural == 0) {
325                 sc->binaural = 0;
326         }
327
328         /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
329          * and doesn't have to be done if we just updating sample rates. */
330         if (binaural_pos_id != -1) {
331                 sc->binaural_pos = binaural_pos_id;
332         }
333         if (is_announcement != -1) {
334                 sc->is_announcement = is_announcement;
335         }
336
337         /*
338          * NOTE: The read_slin_format does not hold a reference because it
339          * will always be a signed linear format.
340          */
341         sc->read_slin_format = slin_format;
342
343         /* Setup smoother */
344         setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
345
346         /* set new read and write formats on channel. */
347         ast_channel_lock(bridge_channel->chan);
348         setup_fail |= ast_set_read_format_path(bridge_channel->chan,
349                 ast_channel_rawreadformat(bridge_channel->chan), slin_format);
350         ast_channel_unlock(bridge_channel->chan);
351
352         /* If channel contains binaural data we will set it here for the trans_pvt. */
353         if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
354                 setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
355         } else if (set_binaural == 0) {
356                 setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
357         }
358
359         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
360         sc->dsp = ast_dsp_new_with_rate(rate);
361         if (setup_fail || !sc->dsp) {
362                 /* Bad news.  Could not setup the channel for softmix. */
363                 ast_mutex_unlock(&sc->lock);
364                 ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
365                 return;
366         }
367
368         /* we want to aggressively detect silence to avoid feedback */
369         if (bridge_channel->tech_args.talking_threshold) {
370                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
371         } else {
372                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
373         }
374
375         ast_mutex_unlock(&sc->lock);
376 }
377
378 /*!
379  * \internal
380  * \brief Poke the mixing thread in case it is waiting for an active channel.
381  * \since 12.0.0
382  *
383  * \param softmix_data Bridge mixing data.
384  *
385  * \return Nothing
386  */
387 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
388 {
389         ast_mutex_lock(&softmix_data->lock);
390         ast_cond_signal(&softmix_data->cond);
391         ast_mutex_unlock(&softmix_data->lock);
392 }
393
394 /*! \brief Function called when a channel is unsuspended from the bridge */
395 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
396 {
397 #ifdef BINAURAL_RENDERING
398         struct softmix_channel *sc = bridge_channel->tech_pvt;
399         if (sc->binaural) {
400                 /* Restore some usefull data if it was a binaural channel */
401                 struct ast_format *slin_format;
402
403                 slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
404                 ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
405         }
406 #endif
407         if (bridge->tech_pvt) {
408                 softmix_poke_thread(bridge->tech_pvt);
409         }
410 }
411
412 /*!
413  * \brief Determine if a stream is a video source stream.
414  *
415  * \param stream The stream to test
416  * \retval 1 The stream is a video source
417  * \retval 0 The stream is not a video source
418  */
419 static int is_video_source(const struct ast_stream *stream)
420 {
421         if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO &&
422                 strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
423                         SOFTBRIDGE_VIDEO_DEST_LEN)) {
424                 return 1;
425         }
426
427         return 0;
428 }
429
430 /*!
431  * \brief Determine if a stream is a video destination stream.
432  *
433  * A source channel name can be provided to narrow this to a destination stream
434  * for a particular source channel. Further, a source stream name can be provided
435  * to narrow this to a particular source stream's destination. However, empty strings
436  * can be provided to match any destination video stream, regardless of source channel
437  * or source stream.
438  *
439  * \param stream The stream to test
440  * \param source_channel_name The name of a source video channel to match
441  * \param source_stream_name The name of the source video stream to match
442  * \retval 1 The stream is a video destination stream
443  * \retval 0 The stream is not a video destination stream
444  */
445 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
446         const char *source_stream_name)
447 {
448         char *dest_video_name;
449         size_t dest_video_name_len;
450
451         if (ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
452                 return 0;
453         }
454
455         dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
456
457         if (!ast_strlen_zero(source_channel_name)) {
458                 dest_video_name_len += strlen(source_channel_name) + 1;
459                 if (!ast_strlen_zero(source_stream_name)) {
460                         dest_video_name_len += strlen(source_stream_name) + 1;
461                 }
462         }
463         dest_video_name = ast_alloca(dest_video_name_len);
464
465         if (!ast_strlen_zero(source_channel_name)) {
466                 if (!ast_strlen_zero(source_stream_name)) {
467                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%s",
468                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
469                                 source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
470                                 source_stream_name);
471                         return !strcmp(ast_stream_get_name(stream), dest_video_name);
472                 } else {
473                         snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
474                                 SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
475                                 source_channel_name);
476                         return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
477                 }
478         } else {
479                 snprintf(dest_video_name, dest_video_name_len, "%s",
480                         SOFTBRIDGE_VIDEO_DEST_PREFIX);
481                 return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
482         }
483
484         return 0;
485 }
486
487 static int append_source_streams(struct ast_stream_topology *dest,
488         const char *channel_name,
489         const struct ast_stream_topology *source)
490 {
491         int i;
492
493         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
494                 struct ast_stream *stream;
495                 struct ast_stream *stream_clone;
496                 char *stream_clone_name = NULL;
497
498                 stream = ast_stream_topology_get_stream(source, i);
499                 if (!is_video_source(stream)) {
500                         continue;
501                 }
502
503                 if (ast_asprintf(&stream_clone_name, "%s_%s_%s", SOFTBRIDGE_VIDEO_DEST_PREFIX,
504                         channel_name, ast_stream_get_name(stream)) < 0) {
505                         ast_free(stream_clone_name);
506                         return -1;
507                 }
508
509                 stream_clone = ast_stream_clone(stream, stream_clone_name);
510                 ast_free(stream_clone_name);
511                 if (!stream_clone) {
512                         return -1;
513                 }
514                 if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
515                         ast_stream_free(stream_clone);
516                         return -1;
517                 }
518         }
519
520         return 0;
521 }
522
523 static int append_all_streams(struct ast_stream_topology *dest,
524         const struct ast_stream_topology *source)
525 {
526         int i;
527         int dest_index = 0;
528
529         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
530                 struct ast_stream *clone;
531                 int added = 0;
532
533                 clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
534                 if (!clone) {
535                         return -1;
536                 }
537
538                 /* If we can reuse an existing removed stream then do so */
539                 while (dest_index < ast_stream_topology_get_count(dest)) {
540                         struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
541
542                         dest_index++;
543
544                         if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
545                                 ast_stream_topology_set_stream(dest, dest_index - 1, clone);
546                                 added = 1;
547                                 break;
548                         }
549                 }
550
551                 /* If no removed stream exists that we took the place of append the stream */
552                 if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
553                         ast_stream_free(clone);
554                         return -1;
555                 }
556         }
557
558         return 0;
559 }
560
561 /*!
562  * \brief Issue channel stream topology change requests.
563  *
564  * When in SFU mode, each participant needs to be able to
565  * send video directly to other participants in the bridge.
566  * This means that all participants need to have their topologies
567  * updated. The joiner needs to have destination streams for
568  * all current participants, and the current participants need
569  * to have destinations streams added for the joiner's sources.
570  *
571  * \param joiner The channel that is joining the softmix bridge
572  * \param participants The current participants in the softmix bridge
573  */
574 static void sfu_topologies_on_join(struct ast_bridge_channel *joiner, struct ast_bridge_channels_list *participants)
575 {
576         struct ast_stream_topology *joiner_video = NULL;
577         struct ast_bridge_channel *participant;
578         int res;
579         struct softmix_channel *sc;
580
581         joiner_video = ast_stream_topology_alloc();
582         if (!joiner_video) {
583                 return;
584         }
585
586         sc = joiner->tech_pvt;
587
588         ast_channel_lock(joiner->chan);
589         res = append_source_streams(joiner_video, ast_channel_name(joiner->chan), ast_channel_get_stream_topology(joiner->chan));
590         sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
591         ast_channel_unlock(joiner->chan);
592
593         if (res || !sc->topology) {
594                 goto cleanup;
595         }
596
597         AST_LIST_TRAVERSE(participants, participant, entry) {
598                 if (participant == joiner) {
599                         continue;
600                 }
601                 ast_channel_lock(participant->chan);
602                 res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
603                                 ast_channel_get_stream_topology(participant->chan));
604                 ast_channel_unlock(participant->chan);
605                 if (res) {
606                         goto cleanup;
607                 }
608         }
609
610         ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
611
612         AST_LIST_TRAVERSE(participants, participant, entry) {
613                 if (participant == joiner) {
614                         continue;
615                 }
616
617                 sc = participant->tech_pvt;
618                 if (append_all_streams(sc->topology, joiner_video)) {
619                         goto cleanup;
620                 }
621                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
622         }
623
624 cleanup:
625         ast_stream_topology_free(joiner_video);
626 }
627
628 /*! \brief Function called when a channel is joined into the bridge */
629 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
630 {
631         struct softmix_channel *sc;
632         struct softmix_bridge_data *softmix_data;
633         int set_binaural = 0;
634         /*
635          * If false, the channel will be convolved, but since it is a non stereo channel, output
636          * will be mono.
637          */
638         int skip_binaural_output = 1;
639         int pos_id;
640         int is_announcement = 0;
641         int samplerate_change;
642
643         softmix_data = bridge->tech_pvt;
644         if (!softmix_data) {
645                 return -1;
646         }
647
648         /* Create a new softmix_channel structure and allocate various things on it */
649         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
650                 return -1;
651         }
652
653         samplerate_change = softmix_data->internal_rate;
654         pos_id = -1;
655         if (bridge->softmix.binaural_active) {
656                 if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
657                         set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
658                         if (set_binaural) {
659                                 softmix_data->internal_rate = samplerate_change;
660                         }
661                         skip_binaural_output = 0;
662                 } else {
663                         is_announcement = 1;
664                 }
665                 if (set_binaural) {
666                         softmix_data->convolve.binaural_active = 1;
667                 }
668                 if (!skip_binaural_output)      {
669                         pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
670                         if (pos_id == -1) {
671                                 ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
672                                                 "Could not allocate enough memory.\n", bridge->uniqueid,
673                                                 ast_channel_name(bridge_channel->chan));
674                                 return -1;
675                         }
676                 }
677         }
678
679         /* Can't forget the lock */
680         ast_mutex_init(&sc->lock);
681
682         /* Can't forget to record our pvt structure within the bridged channel structure */
683         bridge_channel->tech_pvt = sc;
684
685         set_softmix_bridge_data(softmix_data->internal_rate,
686                 softmix_data->internal_mixing_interval
687                         ? softmix_data->internal_mixing_interval
688                         : DEFAULT_SOFTMIX_INTERVAL,
689                 bridge_channel, 0, set_binaural, pos_id, is_announcement);
690
691         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
692                 sfu_topologies_on_join(bridge_channel, &bridge->channels);
693         }
694
695         softmix_poke_thread(softmix_data);
696         return 0;
697 }
698
699 static void remove_destination_streams(struct ast_stream_topology *topology,
700         const char *channel_name)
701 {
702         int i;
703
704         for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
705                 struct ast_stream *stream;
706
707                 stream = ast_stream_topology_get_stream(topology, i);
708
709                 if (is_video_dest(stream, channel_name, NULL)) {
710                         ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
711                 }
712         }
713 }
714
715 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
716 {
717         struct ast_bridge_channel *participant;
718         struct softmix_channel *sc;
719
720         AST_LIST_TRAVERSE(participants, participant, entry) {
721                 sc = participant->tech_pvt;
722                 remove_destination_streams(sc->topology, ast_channel_name(leaver->chan));
723                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
724         }
725
726         sc = leaver->tech_pvt;
727         remove_destination_streams(sc->topology, "");
728         ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
729
730         return 0;
731 }
732
733 /*! \brief Function called when a channel leaves the bridge */
734 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
735 {
736         struct softmix_channel *sc;
737         struct softmix_bridge_data *softmix_data;
738
739         softmix_data = bridge->tech_pvt;
740         sc = bridge_channel->tech_pvt;
741         if (!sc) {
742                 return;
743         }
744
745         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
746                 sfu_topologies_on_leave(bridge_channel, &bridge->channels);
747         }
748
749         if (bridge->softmix.binaural_active) {
750                 if (sc->binaural) {
751                         set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
752                                         softmix_data->default_sample_size);
753                 }
754         }
755
756         bridge_channel->tech_pvt = NULL;
757
758         ast_stream_topology_free(sc->topology);
759
760         /* Drop mutex lock */
761         ast_mutex_destroy(&sc->lock);
762
763         /* Drop the factory */
764         ast_slinfactory_destroy(&sc->factory);
765
766         /* Drop any formats on the frames */
767         ao2_cleanup(sc->write_frame.subclass.format);
768
769         /* Drop the DSP */
770         ast_dsp_free(sc->dsp);
771
772         /* Eep! drop ourselves */
773         ast_free(sc);
774 }
775
776 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
777 {
778         struct ast_bridge_channel *cur;
779
780         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
781                 if (cur->suspended) {
782                         continue;
783                 }
784                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
785                         ast_bridge_channel_queue_frame(cur, frame);
786                         break;
787                 }
788         }
789 }
790
791 /*!
792  * \internal
793  * \brief Determine what to do with a video frame.
794  * \since 12.0.0
795  *
796  * \param bridge Which bridge is getting the frame
797  * \param bridge_channel Which channel is writing the frame.
798  * \param frame What is being written.
799  *
800  * \return Nothing
801  */
802 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
803 {
804         struct softmix_channel *sc;
805         int video_src_priority;
806
807         /* Determine if the video frame should be distributed or not */
808         switch (bridge->softmix.video_mode.mode) {
809         case AST_BRIDGE_VIDEO_MODE_NONE:
810                 break;
811         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
812                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
813                 if (video_src_priority == 1) {
814                         /* Pass to me and everyone else. */
815                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
816                 }
817                 break;
818         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
819                 sc = bridge_channel->tech_pvt;
820                 ast_mutex_lock(&sc->lock);
821                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
822                         sc->video_talker.energy_average,
823                         frame->subclass.frame_ending);
824                 ast_mutex_unlock(&sc->lock);
825                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
826                 if (video_src_priority == 1) {
827                         int num_src = ast_bridge_number_video_src(bridge);
828                         int echo = num_src > 1 ? 0 : 1;
829
830                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
831                 } else if (video_src_priority == 2) {
832                         softmix_pass_video_top_priority(bridge, frame);
833                 }
834                 break;
835         case AST_BRIDGE_VIDEO_MODE_SFU:
836                 /* Nothing special to do here, the bridge channel stream map will ensure the
837                  * video goes everywhere it needs to
838                  */
839                 ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
840                 break;
841         }
842 }
843
844 /*!
845  * \internal
846  * \brief Determine what to do with a voice frame.
847  * \since 12.0.0
848  *
849  * \param bridge Which bridge is getting the frame
850  * \param bridge_channel Which channel is writing the frame.
851  * \param frame What is being written.
852  *
853  * \return Nothing
854  */
855 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
856 {
857         struct softmix_channel *sc = bridge_channel->tech_pvt;
858         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
859         int totalsilence = 0;
860         int cur_energy = 0;
861         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
862                 bridge_channel->tech_args.silence_threshold :
863                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
864         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
865
866         /* Write the frame into the conference */
867         ast_mutex_lock(&sc->lock);
868
869         if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
870                 /*
871                  * The incoming frame is not the expected format.  Update
872                  * the channel's translation path to get us slinear from
873                  * the new format for the next frame.
874                  *
875                  * There is the possibility that this frame is an old slinear
876                  * rate frame that was in flight when the softmix bridge
877                  * changed rates.  If so it will self correct on subsequent
878                  * frames.
879                  */
880                 ast_channel_lock(bridge_channel->chan);
881                 ast_debug(1, "Channel %s wrote unexpected format into bridge.  Got %s, expected %s.\n",
882                         ast_channel_name(bridge_channel->chan),
883                         ast_format_get_name(frame->subclass.format),
884                         ast_format_get_name(sc->read_slin_format));
885                 ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
886                         sc->read_slin_format);
887                 ast_channel_unlock(bridge_channel->chan);
888         }
889
890         /* The channel will be leaving soon if there is no dsp. */
891         if (sc->dsp) {
892                 ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
893         }
894
895         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
896                 int cur_slot = sc->video_talker.energy_history_cur_slot;
897
898                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
899                 sc->video_talker.energy_accum += cur_energy;
900                 sc->video_talker.energy_history[cur_slot] = cur_energy;
901                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
902                 sc->video_talker.energy_history_cur_slot++;
903                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
904                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
905                 }
906         }
907
908         if (totalsilence < silence_threshold) {
909                 if (!sc->talking) {
910                         update_talking = 1;
911                 }
912                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
913         } else {
914                 if (sc->talking) {
915                         update_talking = 0;
916                 }
917                 sc->talking = 0;
918         }
919
920         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
921          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
922          * the audio by flushing the buffer before adding new audio in. */
923         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
924                 ast_slinfactory_flush(&sc->factory);
925         }
926
927         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
928          * is not determined to be talking. */
929         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
930                 ast_slinfactory_feed(&sc->factory, frame);
931         }
932
933         /* Alllll done */
934         ast_mutex_unlock(&sc->lock);
935
936         if (update_talking != -1) {
937                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
938         }
939 }
940
941 static int remove_all_original_streams(struct ast_stream_topology *dest,
942         const struct ast_stream_topology *source,
943         const struct ast_stream_topology *original)
944 {
945         int i;
946
947         for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
948                 struct ast_stream *stream;
949                 int original_index;
950
951                 stream = ast_stream_topology_get_stream(source, i);
952
953                 /* Mark the existing stream as removed so we get a new one, this will get
954                  * reused on a subsequent renegotiation.
955                  */
956                 for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
957                         struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
958
959                         if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
960                                 struct ast_stream *removed;
961
962                                 /* Since the participant is still going to be in the bridge we
963                                  * change the name so that routing does not attempt to route video
964                                  * to this stream.
965                                  */
966                                 removed = ast_stream_clone(stream, "removed");
967                                 if (!removed) {
968                                         return -1;
969                                 }
970
971                                 ast_stream_set_state(removed, AST_STREAM_STATE_REMOVED);
972
973                                 /* The destination topology can only ever contain the same, or more,
974                                  * streams than the original so this is safe.
975                                  */
976                                 if (ast_stream_topology_set_stream(dest, original_index, removed)) {
977                                         ast_stream_free(removed);
978                                         return -1;
979                                 }
980
981                                 break;
982                         }
983                 }
984         }
985
986         return 0;
987 }
988
989 static void sfu_topologies_on_source_change(struct ast_bridge_channel *source, struct ast_bridge_channels_list *participants)
990 {
991         struct ast_stream_topology *source_video = NULL;
992         struct ast_bridge_channel *participant;
993         int res;
994
995         source_video = ast_stream_topology_alloc();
996         if (!source_video) {
997                 return;
998         }
999
1000         ast_channel_lock(source->chan);
1001         res = append_source_streams(source_video, ast_channel_name(source->chan), ast_channel_get_stream_topology(source->chan));
1002         ast_channel_unlock(source->chan);
1003         if (res) {
1004                 goto cleanup;
1005         }
1006
1007         AST_LIST_TRAVERSE(participants, participant, entry) {
1008                 struct ast_stream_topology *original_topology;
1009                 struct softmix_channel *sc;
1010
1011                 if (participant == source) {
1012                         continue;
1013                 }
1014
1015                 sc = participant->tech_pvt;
1016
1017                 original_topology = ast_stream_topology_clone(sc->topology);
1018                 if (!original_topology) {
1019                         goto cleanup;
1020                 }
1021
1022                 /* We add all the source streams back in, if any removed streams are already present they will
1023                  * get used first followed by appending new ones.
1024                  */
1025                 if (append_all_streams(sc->topology, source_video)) {
1026                         ast_stream_topology_free(original_topology);
1027                         goto cleanup;
1028                 }
1029
1030                 /* And the original existing streams get marked as removed. This causes the remote side to see
1031                  * a new stream for the source streams.
1032                  */
1033                 if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1034                         ast_stream_topology_free(original_topology);
1035                         goto cleanup;
1036                 }
1037
1038                 ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
1039                 ast_stream_topology_free(original_topology);
1040         }
1041
1042 cleanup:
1043         ast_stream_topology_free(source_video);
1044 }
1045
1046 /*!
1047  * \internal
1048  * \brief Determine what to do with a control frame.
1049  * \since 12.0.0
1050  *
1051  * \param bridge Which bridge is getting the frame
1052  * \param bridge_channel Which channel is writing the frame.
1053  * \param frame What is being written.
1054  *
1055  * \retval 0 Frame accepted into the bridge.
1056  * \retval -1 Frame needs to be deferred.
1057  */
1058 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1059 {
1060         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1061
1062         /*
1063          * XXX Softmix needs to use channel roles to determine what to
1064          * do with control frames.
1065          */
1066
1067         switch (frame->subclass.integer) {
1068         case AST_CONTROL_VIDUPDATE:
1069                 if (!bridge->softmix.video_mode.video_update_discard ||
1070                         ast_tvdiff_ms(ast_tvnow(), softmix_data->last_video_update) > bridge->softmix.video_mode.video_update_discard) {
1071                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
1072                         softmix_data->last_video_update = ast_tvnow();
1073                 }
1074                 break;
1075         case AST_CONTROL_STREAM_TOPOLOGY_SOURCE_CHANGED:
1076                 if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
1077                         sfu_topologies_on_source_change(bridge_channel, &bridge->channels);
1078                 }
1079                 break;
1080         default:
1081                 break;
1082         }
1083
1084         return 0;
1085 }
1086
1087 /*!
1088  * \internal
1089  * \brief Determine what to do with a frame written into the bridge.
1090  * \since 12.0.0
1091  *
1092  * \param bridge Which bridge is getting the frame
1093  * \param bridge_channel Which channel is writing the frame.
1094  * \param frame What is being written.
1095  *
1096  * \retval 0 Frame accepted into the bridge.
1097  * \retval -1 Frame needs to be deferred.
1098  *
1099  * \note On entry, bridge is already locked.
1100  */
1101 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1102 {
1103         int res = 0;
1104
1105         if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1106                 /* "Accept" the frame and discard it. */
1107                 return 0;
1108         }
1109
1110         /*
1111          * XXX Softmix needs to use channel roles to determine who gets
1112          * what frame.  Possible roles: announcer, recorder, agent,
1113          * supervisor.
1114          */
1115         switch (frame->frametype) {
1116         case AST_FRAME_NULL:
1117                 /* "Accept" the frame and discard it. */
1118                 break;
1119         case AST_FRAME_DTMF_BEGIN:
1120         case AST_FRAME_DTMF_END:
1121                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1122                 break;
1123         case AST_FRAME_VOICE:
1124                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
1125                 break;
1126         case AST_FRAME_VIDEO:
1127                 softmix_bridge_write_video(bridge, bridge_channel, frame);
1128                 break;
1129         case AST_FRAME_CONTROL:
1130                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1131                 break;
1132         case AST_FRAME_BRIDGE_ACTION:
1133                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1134                 break;
1135         case AST_FRAME_BRIDGE_ACTION_SYNC:
1136                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1137                 ast_assert(0);
1138         default:
1139                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1140                 /* "Accept" the frame and discard it. */
1141                 break;
1142         }
1143
1144         return res;
1145 }
1146
1147 static void gather_softmix_stats(struct softmix_stats *stats,
1148         const struct softmix_bridge_data *softmix_data,
1149         struct ast_bridge_channel *bridge_channel)
1150 {
1151         int channel_native_rate;
1152
1153         /* Gather stats about channel sample rates. */
1154         ast_channel_lock(bridge_channel->chan);
1155         channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1156                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
1157         ast_channel_unlock(bridge_channel->chan);
1158
1159         if (stats->highest_supported_rate < channel_native_rate) {
1160                 stats->highest_supported_rate = channel_native_rate;
1161         }
1162         if (softmix_data->internal_rate < channel_native_rate) {
1163                 int i;
1164
1165                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1166                         if (stats->sample_rates[i] == channel_native_rate) {
1167                                 stats->num_channels[i]++;
1168                                 break;
1169                         } else if (!stats->sample_rates[i]) {
1170                                 stats->sample_rates[i] = channel_native_rate;
1171                                 stats->num_channels[i]++;
1172                                 break;
1173                         }
1174                 }
1175                 stats->num_above_internal_rate++;
1176         } else if (softmix_data->internal_rate == channel_native_rate) {
1177                 stats->num_at_internal_rate++;
1178         }
1179 }
1180
1181 /*!
1182  * \internal
1183  * \brief Analyse mixing statistics and change bridges internal rate
1184  * if necessary.
1185  *
1186  * \retval 0, no changes to internal rate
1187  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1188  */
1189 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1190                 struct softmix_bridge_data *softmix_data, int binaural_active)
1191 {
1192         int i;
1193
1194         if (binaural_active) {
1195                 stats->locked_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1196         }
1197
1198         /*
1199          * Re-adjust the internal bridge sample rate if
1200          * 1. The bridge's internal sample rate is locked in at a sample
1201          *    rate other than the current sample rate being used.
1202          * 2. two or more channels support a higher sample rate
1203          * 3. no channels support the current sample rate or a higher rate
1204          */
1205         if (stats->locked_rate) {
1206                 /* if the rate is locked by the bridge, only update it if it differs
1207                  * from the current rate we are using. */
1208                 if (softmix_data->internal_rate != stats->locked_rate) {
1209                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
1210                                 softmix_data->internal_rate, stats->locked_rate);
1211                         softmix_data->internal_rate = stats->locked_rate;
1212                         return 1;
1213                 }
1214         } else if (stats->num_above_internal_rate >= 2) {
1215                 /* the highest rate is just used as a starting point */
1216                 unsigned int best_rate = stats->highest_supported_rate;
1217                 int best_index = -1;
1218
1219                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1220                         if (stats->num_channels[i]) {
1221                                 break;
1222                         }
1223                         if (2 <= stats->num_channels[i]) {
1224                                 /* Two or more channels support this rate. */
1225                                 if (best_index == -1
1226                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1227                                         /*
1228                                          * best_rate starts out being the first sample rate
1229                                          * greater than the internal sample rate that two or
1230                                          * more channels support.
1231                                          *
1232                                          * or
1233                                          *
1234                                          * There are multiple rates above the internal rate
1235                                          * and this rate is higher than the previous rate two
1236                                          * or more channels support.
1237                                          */
1238                                         best_rate = stats->sample_rates[i];
1239                                         best_index = i;
1240                                 }
1241                         } else if (best_index == -1) {
1242                                 /*
1243                                  * It is possible that multiple channels exist with native sample
1244                                  * rates above the internal sample rate, but none of those channels
1245                                  * have the same rate in common.  In this case, the lowest sample
1246                                  * rate among those channels is picked. Over time as additional
1247                                  * statistic runs are made the internal sample rate number will
1248                                  * adjust to the most optimal sample rate, but it may take multiple
1249                                  * iterations.
1250                                  */
1251                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
1252                         }
1253                 }
1254
1255                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
1256                         softmix_data->internal_rate, best_rate);
1257                 softmix_data->internal_rate = best_rate;
1258                 return 1;
1259         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1260                 /* In this case, the highest supported rate is actually lower than the internal rate */
1261                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
1262                         softmix_data->internal_rate, stats->highest_supported_rate);
1263                 softmix_data->internal_rate = stats->highest_supported_rate;
1264                 return 1;
1265         }
1266         return 0;
1267 }
1268
1269 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1270                 unsigned int starting_num_entries, unsigned int binaural_active)
1271 {
1272         memset(mixing_array, 0, sizeof(*mixing_array));
1273         mixing_array->max_num_entries = starting_num_entries;
1274         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1275                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1276                 return -1;
1277         }
1278         if (binaural_active) {
1279                 if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1280                                 sizeof(struct convolve_channel_pair *)))) {
1281                         ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1282                         return -1;
1283                 }
1284         }
1285         return 0;
1286 }
1287
1288 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1289                 unsigned int binaural_active)
1290 {
1291         ast_free(mixing_array->buffers);
1292         if (binaural_active) {
1293                 ast_free(mixing_array->chan_pairs);
1294         }
1295 }
1296
1297 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1298                 unsigned int num_entries, unsigned int binaural_active)
1299 {
1300         int16_t **tmp;
1301
1302         /* give it some room to grow since memory is cheap but allocations can be expensive */
1303         mixing_array->max_num_entries = num_entries;
1304         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1305                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1306                 return -1;
1307         }
1308         if (binaural_active) {
1309                 struct convolve_channel_pair **tmp2;
1310                 if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1311                                 (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1312                         ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1313                         return -1;
1314                 }
1315                 mixing_array->chan_pairs = tmp2;
1316         }
1317         mixing_array->buffers = tmp;
1318         return 0;
1319 }
1320
1321 /*!
1322  * \brief Mixing loop.
1323  *
1324  * \retval 0 on success
1325  * \retval -1 on failure
1326  */
1327 static int softmix_mixing_loop(struct ast_bridge *bridge)
1328 {
1329         struct softmix_stats stats = { { 0 }, };
1330         struct softmix_mixing_array mixing_array;
1331         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1332         struct ast_timer *timer;
1333         struct softmix_translate_helper trans_helper;
1334         int16_t buf[MAX_DATALEN];
1335 #ifdef BINAURAL_RENDERING
1336         int16_t bin_buf[MAX_DATALEN];
1337         int16_t ann_buf[MAX_DATALEN];
1338 #endif
1339         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1340         int timingfd;
1341         int update_all_rates = 0; /* set this when the internal sample rate has changed */
1342         unsigned int idx;
1343         unsigned int x;
1344         int res = -1;
1345
1346         timer = softmix_data->timer;
1347         timingfd = ast_timer_fd(timer);
1348         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1349         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1350
1351         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1352         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1353                         bridge->softmix.binaural_active)) {
1354                 goto softmix_cleanup;
1355         }
1356
1357         /*
1358          * XXX Softmix needs to use channel roles to determine who gets
1359          * what audio mixed.
1360          */
1361         while (!softmix_data->stop && bridge->num_active) {
1362                 struct ast_bridge_channel *bridge_channel;
1363                 int timeout = -1;
1364                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1365                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1366                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1367
1368                 if (softmix_datalen > MAX_DATALEN) {
1369                         /* This should NEVER happen, but if it does we need to know about it. Almost
1370                          * all the memcpys used during this process depend on this assumption.  Rather
1371                          * than checking this over and over again through out the code, this single
1372                          * verification is done on each iteration. */
1373                         ast_log(LOG_WARNING,
1374                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1375                                 bridge->uniqueid);
1376                         goto softmix_cleanup;
1377                 }
1378
1379                 /* Grow the mixing array buffer as participants are added. */
1380                 if (mixing_array.max_num_entries < bridge->num_channels
1381                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1382                                         bridge->softmix.binaural_active)) {
1383                         goto softmix_cleanup;
1384                 }
1385
1386                 /* init the number of buffers stored in the mixing array to 0.
1387                  * As buffers are added for mixing, this number is incremented. */
1388                 mixing_array.used_entries = 0;
1389
1390                 /* These variables help determine if a rate change is required */
1391                 if (!stat_iteration_counter) {
1392                         memset(&stats, 0, sizeof(stats));
1393                         stats.locked_rate = bridge->softmix.internal_sample_rate;
1394                 }
1395
1396                 /* If the sample rate has changed, update the translator helper */
1397                 if (update_all_rates) {
1398                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1399                 }
1400
1401 #ifdef BINAURAL_RENDERING
1402                 check_binaural_position_change(bridge, softmix_data, bridge_channel);
1403 #endif
1404
1405                 /* Go through pulling audio from each factory that has it available */
1406                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1407                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1408
1409                         if (!sc) {
1410                                 /* This channel failed to join successfully. */
1411                                 continue;
1412                         }
1413
1414                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
1415                         if (update_all_rates) {
1416                                 set_softmix_bridge_data(softmix_data->internal_rate,
1417                                                 softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1418                         }
1419
1420                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1421                         if (!stat_iteration_counter) {
1422                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
1423                         }
1424
1425                         /* if the channel is suspended, don't check for audio, but still gather stats */
1426                         if (bridge_channel->suspended) {
1427                                 continue;
1428                         }
1429
1430                         /* Try to get audio from the factory if available */
1431                         ast_mutex_lock(&sc->lock);
1432                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1433 #ifdef BINAURAL_RENDERING
1434                                 add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1435                                                 ast_channel_name(bridge_channel->chan));
1436 #endif
1437                                 mixing_array.used_entries++;
1438                         }
1439                         ast_mutex_unlock(&sc->lock);
1440                 }
1441
1442                 /* mix it like crazy (non binaural channels)*/
1443                 memset(buf, 0, softmix_datalen);
1444                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1445                         for (x = 0; x < softmix_samples; ++x) {
1446                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1447                         }
1448                 }
1449
1450 #ifdef BINAURAL_RENDERING
1451                 binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1452 #endif
1453
1454                 /* Next step go through removing the channel's own audio and creating a good frame... */
1455                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1456                         struct softmix_channel *sc = bridge_channel->tech_pvt;
1457
1458                         if (!sc || bridge_channel->suspended) {
1459                                 /* This channel failed to join successfully or is suspended. */
1460                                 continue;
1461                         }
1462
1463                         ast_mutex_lock(&sc->lock);
1464
1465                         /* Make SLINEAR write frame from local buffer */
1466                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1467                                 "Replace softmix channel slin format");
1468 #ifdef BINAURAL_RENDERING
1469                         if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1470                                         && sc->binaural) {
1471                                 create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1472                                                 softmix_samples, buf);
1473                         } else
1474 #endif
1475                         {
1476                                 sc->write_frame.datalen = softmix_datalen;
1477                                 sc->write_frame.samples = softmix_samples;
1478                                 memcpy(sc->final_buf, buf, softmix_datalen);
1479                         }
1480                         /* process the softmix channel's new write audio */
1481                         softmix_process_write_audio(&trans_helper,
1482                                         ast_channel_rawwriteformat(bridge_channel->chan), sc,
1483                                         softmix_data->default_sample_size);
1484
1485                         ast_mutex_unlock(&sc->lock);
1486
1487                         /* A frame is now ready for the channel. */
1488                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1489                 }
1490
1491                 update_all_rates = 0;
1492                 if (!stat_iteration_counter) {
1493                         update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1494                                         bridge->softmix.binaural_active);
1495                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1496                 }
1497                 stat_iteration_counter--;
1498
1499                 ast_bridge_unlock(bridge);
1500                 /* cleanup any translation frame data from the previous mixing iteration. */
1501                 softmix_translate_helper_cleanup(&trans_helper);
1502                 /* Wait for the timing source to tell us to wake up and get things done */
1503                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1504                 if (ast_timer_ack(timer, 1) < 0) {
1505                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1506                                 bridge->uniqueid);
1507                         ast_bridge_lock(bridge);
1508                         goto softmix_cleanup;
1509                 }
1510                 ast_bridge_lock(bridge);
1511
1512                 /* make sure to detect mixing interval changes if they occur. */
1513                 if (bridge->softmix.internal_mixing_interval
1514                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1515                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1516                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1517                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1518                 }
1519         }
1520
1521         res = 0;
1522
1523 softmix_cleanup:
1524         softmix_translate_helper_destroy(&trans_helper);
1525         softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1526         return res;
1527 }
1528
1529 /*!
1530  * \internal
1531  * \brief Mixing thread.
1532  * \since 12.0.0
1533  *
1534  * \note The thread does not have its own reference to the
1535  * bridge.  The lifetime of the thread is tied to the lifetime
1536  * of the mixing technology association with the bridge.
1537  */
1538 static void *softmix_mixing_thread(void *data)
1539 {
1540         struct softmix_bridge_data *softmix_data = data;
1541         struct ast_bridge *bridge = softmix_data->bridge;
1542
1543         ast_bridge_lock(bridge);
1544         if (bridge->callid) {
1545                 ast_callid_threadassoc_add(bridge->callid);
1546         }
1547
1548         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1549
1550         while (!softmix_data->stop) {
1551                 if (!bridge->num_active) {
1552                         /* Wait for something to happen to the bridge. */
1553                         ast_bridge_unlock(bridge);
1554                         ast_mutex_lock(&softmix_data->lock);
1555                         if (!softmix_data->stop) {
1556                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1557                         }
1558                         ast_mutex_unlock(&softmix_data->lock);
1559                         ast_bridge_lock(bridge);
1560                         continue;
1561                 }
1562
1563                 if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
1564 #ifndef BINAURAL_RENDERING
1565                         ast_bridge_lock(bridge);
1566                         bridge->softmix.binaural_active = 0;
1567                         ast_bridge_unlock(bridge);
1568                         ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
1569                                         "compiled.\n", bridge->uniqueid);
1570 #else
1571                         /* Set and init binaural data if binaural is activated in the configuration. */
1572                         softmix_data->internal_rate = SOFTMIX_BINAURAL_SAMPLE_RATE;
1573                         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1574                                         softmix_data->internal_mixing_interval);
1575                         /* If init for binaural processing fails we will fall back to mono audio processing. */
1576                         if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
1577                                         == -1) {
1578                                 ast_bridge_lock(bridge);
1579                                 bridge->softmix.binaural_active = 0;
1580                                 ast_bridge_unlock(bridge);
1581                                 ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
1582                                                 "binaural processing,  Will only process mono audio.\n",
1583                                                 bridge->uniqueid);
1584                         }
1585                         softmix_data->binaural_init = 1;
1586 #endif
1587                 }
1588
1589                 if (softmix_mixing_loop(bridge)) {
1590                         /*
1591                          * A mixing error occurred.  Sleep and try again later so we
1592                          * won't flood the logs.
1593                          */
1594                         ast_bridge_unlock(bridge);
1595                         sleep(1);
1596                         ast_bridge_lock(bridge);
1597                 }
1598         }
1599
1600         ast_bridge_unlock(bridge);
1601
1602         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1603
1604         return NULL;
1605 }
1606
1607 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1608 {
1609         if (softmix_data->timer) {
1610                 ast_timer_close(softmix_data->timer);
1611                 softmix_data->timer = NULL;
1612         }
1613         ast_mutex_destroy(&softmix_data->lock);
1614         ast_cond_destroy(&softmix_data->cond);
1615         ast_free(softmix_data);
1616 }
1617
1618 /*! \brief Function called when a bridge is created */
1619 static int softmix_bridge_create(struct ast_bridge *bridge)
1620 {
1621         struct softmix_bridge_data *softmix_data;
1622
1623         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1624         if (!softmix_data) {
1625                 return -1;
1626         }
1627         softmix_data->bridge = bridge;
1628         ast_mutex_init(&softmix_data->lock);
1629         ast_cond_init(&softmix_data->cond, NULL);
1630         softmix_data->timer = ast_timer_open();
1631         if (!softmix_data->timer) {
1632                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1633                 softmix_bridge_data_destroy(softmix_data);
1634                 return -1;
1635         }
1636         /* start at minimum rate, let it grow from there */
1637         softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
1638         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1639
1640 #ifdef BINAURAL_RENDERING
1641         softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
1642                         softmix_data->internal_mixing_interval);
1643 #endif
1644
1645         bridge->tech_pvt = softmix_data;
1646
1647         /* Start the mixing thread. */
1648         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1649                 softmix_data)) {
1650                 softmix_data->thread = AST_PTHREADT_NULL;
1651                 softmix_bridge_data_destroy(softmix_data);
1652                 bridge->tech_pvt = NULL;
1653                 return -1;
1654         }
1655
1656         return 0;
1657 }
1658
1659 /*!
1660  * \internal
1661  * \brief Request the softmix mixing thread stop.
1662  * \since 12.0.0
1663  *
1664  * \param bridge Which bridge is being stopped.
1665  *
1666  * \return Nothing
1667  */
1668 static void softmix_bridge_stop(struct ast_bridge *bridge)
1669 {
1670         struct softmix_bridge_data *softmix_data;
1671
1672         softmix_data = bridge->tech_pvt;
1673         if (!softmix_data) {
1674                 return;
1675         }
1676
1677         ast_mutex_lock(&softmix_data->lock);
1678         softmix_data->stop = 1;
1679         ast_mutex_unlock(&softmix_data->lock);
1680 }
1681
1682 /*! \brief Function called when a bridge is destroyed */
1683 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1684 {
1685         struct softmix_bridge_data *softmix_data;
1686         pthread_t thread;
1687
1688         softmix_data = bridge->tech_pvt;
1689         if (!softmix_data) {
1690                 return;
1691         }
1692
1693         /* Stop the mixing thread. */
1694         ast_mutex_lock(&softmix_data->lock);
1695         softmix_data->stop = 1;
1696         ast_cond_signal(&softmix_data->cond);
1697         thread = softmix_data->thread;
1698         softmix_data->thread = AST_PTHREADT_NULL;
1699         ast_mutex_unlock(&softmix_data->lock);
1700         if (thread != AST_PTHREADT_NULL) {
1701                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1702                 pthread_join(thread, NULL);
1703         }
1704 #ifdef BINAURAL_RENDERING
1705         free_convolve_data(&softmix_data->convolve);
1706 #endif
1707         softmix_bridge_data_destroy(softmix_data);
1708         bridge->tech_pvt = NULL;
1709 }
1710
1711 /*!
1712  * \brief Map a source stream to all of its destination streams.
1713  *
1714  * \param source_stream_name Name of the source stream
1715  * \param source_channel_name Name of channel where the source stream originates
1716  * \param bridge_stream_position The slot in the bridge where source video will come from
1717  * \param participants The bridge_channels in the bridge
1718  */
1719 static void map_source_to_destinations(const char *source_stream_name, const char *source_channel_name,
1720         size_t bridge_stream_position, struct ast_bridge_channels_list *participants)
1721 {
1722         struct ast_bridge_channel *participant;
1723
1724         AST_LIST_TRAVERSE(participants, participant, entry) {
1725                 int i;
1726                 struct ast_stream_topology *topology;
1727
1728                 if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
1729                         continue;
1730                 }
1731
1732                 ast_bridge_channel_lock(participant);
1733                 ast_channel_lock(participant->chan);
1734                 topology = ast_channel_get_stream_topology(participant->chan);
1735
1736                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1737                         struct ast_stream *stream;
1738
1739                         stream = ast_stream_topology_get_stream(topology, i);
1740                         if (is_video_dest(stream, source_channel_name, source_stream_name)) {
1741                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
1742                                 break;
1743                         }
1744                 }
1745                 ast_channel_unlock(participant->chan);
1746                 ast_bridge_channel_unlock(participant);
1747         }
1748 }
1749
1750 /*!
1751  * \brief stream_topology_changed callback
1752  *
1753  * For most video modes, nothing beyond the ordinary is required.
1754  * For the SFU case, though, we need to completely remap the streams
1755  * in order to ensure video gets directed where it is expected to go.
1756  *
1757  * \param bridge The bridge
1758  * \param bridge_channel Channel whose topology has changed
1759  */
1760 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1761 {
1762         struct ast_bridge_channel *participant;
1763         struct ast_vector_int media_types;
1764         int nths[AST_MEDIA_TYPE_END] = {0};
1765
1766         switch (bridge->softmix.video_mode.mode) {
1767         case AST_BRIDGE_VIDEO_MODE_NONE:
1768         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
1769         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
1770         default:
1771                 ast_bridge_channel_stream_map(bridge_channel);
1772                 return;
1773         case AST_BRIDGE_VIDEO_MODE_SFU:
1774                 break;
1775         }
1776
1777         AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
1778
1779         /* First traversal: re-initialize all of the participants' stream maps */
1780         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1781                 ast_bridge_channel_lock(participant);
1782                 AST_VECTOR_RESET(&participant->stream_map.to_channel, AST_VECTOR_ELEM_CLEANUP_NOOP);
1783                 AST_VECTOR_RESET(&participant->stream_map.to_bridge, AST_VECTOR_ELEM_CLEANUP_NOOP);
1784                 ast_bridge_channel_unlock(participant);
1785         }
1786
1787         /* Second traversal: Map specific video channels from their source to their destinations.
1788          *
1789          * This is similar to what is done in ast_stream_topology_map(),
1790          * except that video channels are handled differently.  Each video
1791          * source has it's own unique index on the bridge.  This way, a
1792          * particular channel's source video can be distributed to the
1793          * appropriate destination streams on the other channels.
1794          */
1795         AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
1796                 int i;
1797                 struct ast_stream_topology *topology;
1798
1799                 ast_bridge_channel_lock(participant);
1800                 ast_channel_lock(participant->chan);
1801
1802                 topology = ast_channel_get_stream_topology(participant->chan);
1803                 if (topology) {
1804                         /*
1805                          * Sigh.  We have to clone to avoid deadlock in
1806                          * map_source_to_destinations() because topology
1807                          * is not an ao2 object.
1808                          */
1809                         topology = ast_stream_topology_clone(topology);
1810                 }
1811                 if (!topology) {
1812                         /* Oh, my, we are in trouble. */
1813                         ast_channel_unlock(participant->chan);
1814                         ast_bridge_channel_unlock(participant);
1815                         continue;
1816                 }
1817
1818                 for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
1819                         struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
1820
1821                         if (is_video_source(stream)) {
1822                                 AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
1823                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
1824                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, -1);
1825                                 /*
1826                                  * Unlock the channel and participant to prevent
1827                                  * potential deadlock in map_source_to_destinations().
1828                                  */
1829                                 ast_channel_unlock(participant->chan);
1830                                 ast_bridge_channel_unlock(participant);
1831                                 map_source_to_destinations(ast_stream_get_name(stream), ast_channel_name(participant->chan),
1832                                         AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels);
1833                                 ast_bridge_channel_lock(participant);
1834                                 ast_channel_lock(participant->chan);
1835                         } else if (is_video_dest(stream, NULL, NULL)) {
1836                                 /* We expect to never read media from video destination channels, but just
1837                                  * in case, we should set their to_bridge value to -1.
1838                                  */
1839                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
1840                         } else {
1841                                 /* XXX This is copied from ast_stream_topology_map(). This likely could
1842                                  * be factored out in some way
1843                                  */
1844                                 enum ast_media_type type = ast_stream_get_type(stream);
1845                                 int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
1846                                         type, AST_VECTOR_ELEM_DEFAULT_CMP);
1847
1848                                 if (index == -1) {
1849                                         AST_VECTOR_APPEND(&media_types, type);
1850                                         index = AST_VECTOR_SIZE(&media_types) - 1;
1851                                 }
1852
1853                                 AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
1854                                 AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
1855                         }
1856                 }
1857
1858                 ast_stream_topology_free(topology);
1859
1860                 ast_channel_unlock(participant->chan);
1861                 ast_bridge_channel_unlock(participant);
1862         }
1863 }
1864
1865 static struct ast_bridge_technology softmix_bridge = {
1866         .name = "softmix",
1867         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1868         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1869         .create = softmix_bridge_create,
1870         .stop = softmix_bridge_stop,
1871         .destroy = softmix_bridge_destroy,
1872         .join = softmix_bridge_join,
1873         .leave = softmix_bridge_leave,
1874         .unsuspend = softmix_bridge_unsuspend,
1875         .write = softmix_bridge_write,
1876         .stream_topology_changed = softmix_bridge_stream_topology_changed,
1877 };
1878
1879 #ifdef TEST_FRAMEWORK
1880 struct stream_parameters {
1881         const char *name;
1882         const char *formats;
1883         enum ast_media_type type;
1884 };
1885
1886 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
1887 {
1888         struct ast_stream_topology *topology;
1889         size_t i;
1890
1891         topology = ast_stream_topology_alloc();
1892         if (!topology) {
1893                 return NULL;
1894         }
1895
1896         for (i = 0; i < num_streams; ++i) {
1897                 RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
1898                 struct ast_stream *stream;
1899
1900                 caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1901                 if (!caps) {
1902                         goto fail;
1903                 }
1904                 if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
1905                         goto fail;
1906                 }
1907                 stream = ast_stream_alloc(params[i].name, params[i].type);
1908                 if (!stream) {
1909                         goto fail;
1910                 }
1911                 ast_stream_set_formats(stream, caps);
1912                 if (ast_stream_topology_append_stream(topology, stream) < 0) {
1913                         ast_stream_free(stream);
1914                         goto fail;
1915                 }
1916         }
1917
1918         return topology;
1919
1920 fail:
1921         ast_stream_topology_free(topology);
1922         return NULL;
1923 }
1924
1925 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
1926         const struct stream_parameters *params)
1927 {
1928         struct ast_format_cap *stream_caps;
1929         struct ast_format_cap *params_caps;
1930
1931         if (ast_stream_get_type(stream) != params->type) {
1932                 ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
1933                         ast_codec_media_type2str(params->type),
1934                         ast_codec_media_type2str(ast_stream_get_type(stream)));
1935                 return -1;
1936         }
1937         if (strcmp(ast_stream_get_name(stream), params->name)) {
1938                 ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
1939                         params->name, ast_stream_get_name(stream));
1940                 return -1;
1941         }
1942
1943         stream_caps = ast_stream_get_formats(stream);
1944         params_caps = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
1945         if (!params_caps) {
1946                 ast_test_status_update(test, "Allocation error on capabilities\n");
1947                 return -1;
1948         }
1949         ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
1950
1951         if (ast_format_cap_identical(stream_caps, params_caps)) {
1952                 ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
1953                         ast_stream_get_name(stream));
1954                 ao2_cleanup(params_caps);
1955                 return -1;
1956         }
1957
1958         ao2_cleanup(params_caps);
1959         return 0;
1960 }
1961
1962 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
1963         const struct stream_parameters *params, size_t num_streams)
1964 {
1965         int i;
1966
1967         if (ast_stream_topology_get_count(topology) < num_streams) {
1968                 ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
1969                         ast_stream_topology_get_count(topology), num_streams);
1970                 return -1;
1971         }
1972
1973         for (i = 0; i < ARRAY_LEN(params); ++i) {
1974                 if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
1975                         return -1;
1976                 }
1977         }
1978
1979         return 0;
1980 }
1981
1982 AST_TEST_DEFINE(sfu_append_source_streams)
1983 {
1984         enum ast_test_result_state res = AST_TEST_FAIL;
1985         static const struct stream_parameters bob_streams[] = {
1986                 { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
1987                 { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
1988         };
1989         static const struct stream_parameters alice_streams[] = {
1990                 { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
1991                 { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
1992         };
1993         static const struct stream_parameters alice_dest_stream = {
1994                 "softbridge_dest_PJSIP/Bob-00000001_bob_video", "vp8", AST_MEDIA_TYPE_VIDEO,
1995         };
1996         static const struct stream_parameters bob_dest_stream = {
1997                 "softbridge_dest_PJSIP/Alice-00000000_alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
1998         };
1999         struct ast_stream_topology *topology_alice = NULL;
2000         struct ast_stream_topology *topology_bob = NULL;
2001
2002         switch (cmd) {
2003         case TEST_INIT:
2004                 info->name = "sfu_append_source_streams";
2005                 info->category = "/bridges/bridge_softmix/";
2006                 info->summary = "Test appending of video streams";
2007                 info->description =
2008                         "This tests does stuff.";
2009                 return AST_TEST_NOT_RUN;
2010         case TEST_EXECUTE:
2011                 break;
2012         }
2013
2014         topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2015         if (!topology_alice) {
2016                 goto end;
2017         }
2018
2019         topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2020         if (!topology_bob) {
2021                 goto end;
2022         }
2023
2024         if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", topology_bob)) {
2025                 ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2026                 goto end;
2027         }
2028
2029         if (ast_stream_topology_get_count(topology_alice) != 3) {
2030                 ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2031                         ast_stream_topology_get_count(topology_alice), 3);
2032                 goto end;
2033         }
2034
2035         if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2036                 goto end;
2037         }
2038
2039         if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2040                 goto end;
2041         }
2042
2043         if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", topology_alice)) {
2044                 ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2045                 goto end;
2046         }
2047
2048         if (ast_stream_topology_get_count(topology_bob) != 3) {
2049                 ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2050                         ast_stream_topology_get_count(topology_bob), 3);
2051                 goto end;
2052         }
2053
2054         if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2055                 goto end;
2056         }
2057
2058         if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2059                 goto end;
2060         }
2061
2062         res = AST_TEST_PASS;
2063
2064 end:
2065         ast_stream_topology_free(topology_alice);
2066         ast_stream_topology_free(topology_bob);
2067         return res;
2068 }
2069
2070 AST_TEST_DEFINE(sfu_remove_destination_streams)
2071 {
2072         enum ast_test_result_state res = AST_TEST_FAIL;
2073         static const struct stream_parameters params[] = {
2074                 { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2075                 { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2076                 { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2077                 { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2078         };
2079         static const struct {
2080                 const char *channel_name;
2081                 int num_streams;
2082                 int params_index[4];
2083         } removal_results[] = {
2084                 { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2085                 { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2086                 { "", 4, { 0, 1, 2, 3 }, },
2087         };
2088         struct ast_stream_topology *orig = NULL;
2089         int i;
2090
2091         switch (cmd) {
2092         case TEST_INIT:
2093                 info->name = "sfu_remove_destination_streams";
2094                 info->category = "/bridges/bridge_softmix/";
2095                 info->summary = "Test removal of destination video streams";
2096                 info->description =
2097                         "This tests does stuff.";
2098                 return AST_TEST_NOT_RUN;
2099         case TEST_EXECUTE:
2100                 break;
2101         }
2102
2103         orig = build_topology(params, ARRAY_LEN(params));
2104         if (!orig) {
2105                 ast_test_status_update(test, "Unable to build initial stream topology\n");
2106                 goto end;
2107         }
2108
2109         for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2110                 int j;
2111
2112                 remove_destination_streams(orig, removal_results[i].channel_name);
2113
2114                 if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2115                         ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2116                                 ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2117                         goto end;
2118                 }
2119
2120                 for (j = 0; j < removal_results[i].num_streams; ++j) {
2121                         struct ast_stream *actual;
2122                         struct ast_stream *expected;
2123                         int orig_index;
2124
2125                         actual = ast_stream_topology_get_stream(orig, j);
2126
2127                         orig_index = removal_results[i].params_index[j];
2128                         expected = ast_stream_topology_get_stream(orig, orig_index);
2129
2130                         if (!ast_format_cap_identical(ast_stream_get_formats(actual),
2131                                 ast_stream_get_formats(expected))) {
2132                                 struct ast_str *expected_str;
2133                                 struct ast_str *actual_str;
2134
2135                                 expected_str = ast_str_alloca(64);
2136                                 actual_str = ast_str_alloca(64);
2137
2138                                 ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2139                                         ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2140                                         ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2141                                 goto end;
2142                         }
2143
2144                         if (is_video_dest(actual, removal_results[i].channel_name, NULL) &&
2145                                 ast_stream_get_state(actual) != AST_STREAM_STATE_REMOVED) {
2146                                 ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2147                                 goto end;
2148                         }
2149                 }
2150         }
2151
2152         res = AST_TEST_PASS;
2153
2154 end:
2155         ast_stream_topology_free(orig);
2156         return res;
2157 }
2158
2159 #endif
2160
2161 static int unload_module(void)
2162 {
2163         ast_bridge_technology_unregister(&softmix_bridge);
2164         AST_TEST_UNREGISTER(sfu_append_source_streams);
2165         AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2166         return 0;
2167 }
2168
2169 static int load_module(void)
2170 {
2171         if (ast_bridge_technology_register(&softmix_bridge)) {
2172                 unload_module();
2173                 return AST_MODULE_LOAD_DECLINE;
2174         }
2175         AST_TEST_REGISTER(sfu_append_source_streams);
2176         AST_TEST_REGISTER(sfu_remove_destination_streams);
2177         return AST_MODULE_LOAD_SUCCESS;
2178 }
2179
2180 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");