72299209c1993e732b4d2968ec581964b07f9bd3
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
37
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <sys/time.h>
42 #include <signal.h>
43 #include <errno.h>
44 #include <unistd.h>
45
46 #include "asterisk/module.h"
47 #include "asterisk/channel.h"
48 #include "asterisk/bridge.h"
49 #include "asterisk/bridge_technology.h"
50 #include "asterisk/frame.h"
51 #include "asterisk/options.h"
52 #include "asterisk/logger.h"
53 #include "asterisk/slinfactory.h"
54 #include "asterisk/astobj2.h"
55 #include "asterisk/timing.h"
56 #include "asterisk/translate.h"
57
58 #define MAX_DATALEN 8096
59
60 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
61 #define DEFAULT_SOFTMIX_INTERVAL 20
62
63 /*! \brief Size of the buffer used for sample manipulation */
64 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
65
66 /*! \brief Number of samples we are dealing with */
67 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
68
69 /*! \brief Number of mixing iterations to perform between gathering statistics. */
70 #define SOFTMIX_STAT_INTERVAL 100
71
72 /* This is the threshold in ms at which a channel's own audio will stop getting
73  * mixed out its own write audio stream because it is not talking. */
74 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
75 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
76
77 #define DEFAULT_ENERGY_HISTORY_LEN 150
78
79 struct video_follow_talker_data {
80         /*! audio energy history */
81         int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
82         /*! The current slot being used in the history buffer, this
83          *  increments and wraps around */
84         int energy_history_cur_slot;
85         /*! The current energy sum used for averages. */
86         int energy_accum;
87         /*! The current energy average */
88         int energy_average;
89 };
90
91 /*! \brief Structure which contains per-channel mixing information */
92 struct softmix_channel {
93         /*! Lock to protect this structure */
94         ast_mutex_t lock;
95         /*! Factory which contains audio read in from the channel */
96         struct ast_slinfactory factory;
97         /*! Frame that contains mixed audio to be written out to the channel */
98         struct ast_frame write_frame;
99         /*! Frame that contains mixed audio read from the channel */
100         struct ast_frame read_frame;
101         /*! DSP for detecting silence */
102         struct ast_dsp *dsp;
103         /*!
104          * \brief TRUE if a channel is talking.
105          *
106          * \note This affects how the channel's audio is mixed back to
107          * it.
108          */
109         unsigned int talking:1;
110         /*! TRUE if the channel provided audio for this mixing interval */
111         unsigned int have_audio:1;
112         /*! Buffer containing final mixed audio from all sources */
113         short final_buf[MAX_DATALEN];
114         /*! Buffer containing only the audio from the channel */
115         short our_buf[MAX_DATALEN];
116         /*! Data pertaining to talker mode for video conferencing */
117         struct video_follow_talker_data video_talker;
118 };
119
120 struct softmix_bridge_data {
121         struct ast_timer *timer;
122         /*!
123          * \brief Bridge pointer passed to the softmix mixing thread.
124          *
125          * \note Does not need a reference because the bridge will
126          * always exist while the mixing thread exists even if the
127          * bridge is no longer actively using the softmix technology.
128          */
129         struct ast_bridge *bridge;
130         /*! Lock for signaling the mixing thread. */
131         ast_mutex_t lock;
132         /*! Condition, used if we need to wake up the mixing thread. */
133         ast_cond_t cond;
134         /*! Thread handling the mixing */
135         pthread_t thread;
136         unsigned int internal_rate;
137         unsigned int internal_mixing_interval;
138         /*! TRUE if the mixing thread should stop */
139         unsigned int stop:1;
140 };
141
142 struct softmix_stats {
143         /*! Each index represents a sample rate used above the internal rate. */
144         unsigned int sample_rates[16];
145         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
146         unsigned int num_channels[16];
147         /*! the number of channels above the internal sample rate */
148         unsigned int num_above_internal_rate;
149         /*! the number of channels at the internal sample rate */
150         unsigned int num_at_internal_rate;
151         /*! the absolute highest sample rate supported by any channel in the bridge */
152         unsigned int highest_supported_rate;
153         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
154         unsigned int locked_rate;
155 };
156
157 struct softmix_mixing_array {
158         unsigned int max_num_entries;
159         unsigned int used_entries;
160         int16_t **buffers;
161 };
162
163 struct softmix_translate_helper_entry {
164         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
165                                       and re-init if it was usable. */
166         struct ast_format *dst_format; /*!< The destination format for this helper */
167         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
168         struct ast_frame *out_frame; /*!< The output frame from the last translation */
169         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
170 };
171
172 struct softmix_translate_helper {
173         struct ast_format *slin_src; /*!< the source format expected for all the translators */
174         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
175 };
176
177 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
178 {
179         struct softmix_translate_helper_entry *entry;
180         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
181                 return NULL;
182         }
183         entry->dst_format = ao2_bump(dst);
184         /* initialize this to one so that the first time through the cleanup code after
185            allocation it won't be removed from the entry list */
186         entry->num_times_requested = 1;
187         return entry;
188 }
189
190 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
191 {
192         ao2_cleanup(entry->dst_format);
193
194         if (entry->trans_pvt) {
195                 ast_translator_free_path(entry->trans_pvt);
196         }
197         if (entry->out_frame) {
198                 ast_frfree(entry->out_frame);
199         }
200         ast_free(entry);
201         return NULL;
202 }
203
204 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
205 {
206         memset(trans_helper, 0, sizeof(*trans_helper));
207         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
208 }
209
210 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
211 {
212         struct softmix_translate_helper_entry *entry;
213
214         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
215                 softmix_translate_helper_free_entry(entry);
216         }
217 }
218
219 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
220 {
221         struct softmix_translate_helper_entry *entry;
222
223         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
224         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
225                 if (entry->trans_pvt) {
226                         ast_translator_free_path(entry->trans_pvt);
227                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
228                                 AST_LIST_REMOVE_CURRENT(entry);
229                                 entry = softmix_translate_helper_free_entry(entry);
230                         }
231                 }
232         }
233         AST_LIST_TRAVERSE_SAFE_END;
234 }
235
236 /*!
237  * \internal
238  * \brief Get the next available audio on the softmix channel's read stream
239  * and determine if it should be mixed out or not on the write stream.
240  *
241  * \retval pointer to buffer containing the exact number of samples requested on success.
242  * \retval NULL if no samples are present
243  */
244 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
245 {
246         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
247                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
248                 sc->have_audio = 1;
249                 return sc->our_buf;
250         }
251         sc->have_audio = 0;
252         return NULL;
253 }
254
255 /*!
256  * \internal
257  * \brief Process a softmix channel's write audio
258  *
259  * \details This function will remove the channel's talking from its own audio if present and
260  * possibly even do the channel's write translation for it depending on how many other
261  * channels use the same write format.
262  */
263 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
264         struct ast_format *raw_write_fmt,
265         struct softmix_channel *sc)
266 {
267         struct softmix_translate_helper_entry *entry = NULL;
268         int i;
269
270         /* If we provided audio that was not determined to be silence,
271          * then take it out while in slinear format. */
272         if (sc->have_audio && sc->talking) {
273                 for (i = 0; i < sc->write_frame.samples; i++) {
274                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
275                 }
276                 /* check to see if any entries exist for the format. if not we'll want
277                    to remove it during cleanup */
278                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
279                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
280                                 ++entry->num_times_requested;
281                                 break;
282                         }
283                 }
284                 /* do not do any special write translate optimization if we had to make
285                  * a special mix for them to remove their own audio. */
286                 return;
287         }
288
289         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
290            of translation paths and track the number of references for each type. Each one of the same
291            type should be able to use the same out_frame. Since the optimization is only necessary for
292            multiple channels (>=2) using the same codec make sure resources are allocated only when
293            needed and released when not (see also softmix_translate_helper_cleanup */
294         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
295                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
296                         entry->num_times_requested++;
297                 } else {
298                         continue;
299                 }
300                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
301                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
302                 }
303                 if (entry->trans_pvt && !entry->out_frame) {
304                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
305                 }
306                 if (entry->out_frame && (entry->out_frame->datalen < MAX_DATALEN)) {
307                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
308                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
309                         sc->write_frame.datalen = entry->out_frame->datalen;
310                         sc->write_frame.samples = entry->out_frame->samples;
311                 }
312                 break;
313         }
314
315         /* add new entry into list if this format destination was not matched. */
316         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
317                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
318         }
319 }
320
321 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
322 {
323         struct softmix_translate_helper_entry *entry;
324
325         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
326                 /* if it hasn't been requested then remove it */
327                 if (!entry->num_times_requested) {
328                         AST_LIST_REMOVE_CURRENT(entry);
329                         softmix_translate_helper_free_entry(entry);
330                         continue;
331                 }
332
333                 if (entry->out_frame) {
334                         ast_frfree(entry->out_frame);
335                         entry->out_frame = NULL;
336                 }
337
338                 /* nothing is optimized for a single path reference, so there is
339                    no reason to continue to hold onto the codec */
340                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
341                         ast_translator_free_path(entry->trans_pvt);
342                         entry->trans_pvt = NULL;
343                 }
344
345                 /* for each iteration (a mixing run) in the bridge softmix thread the number
346                    of references to a given entry is recalculated, so reset the number of
347                    times requested */
348                 entry->num_times_requested = 0;
349         }
350         AST_LIST_TRAVERSE_SAFE_END;
351 }
352
353 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset)
354 {
355         struct softmix_channel *sc = bridge_channel->tech_pvt;
356         unsigned int channel_read_rate = ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan));
357
358         ast_mutex_lock(&sc->lock);
359         if (reset) {
360                 ast_slinfactory_destroy(&sc->factory);
361                 ast_dsp_free(sc->dsp);
362         }
363
364         /* Setup write frame parameters */
365         sc->write_frame.frametype = AST_FRAME_VOICE;
366         /*
367          * NOTE: The write_frame format holds a reference because translation
368          * could be needed and the format changed to the translated format
369          * for the channel.  The translated format may not be a
370          * static cached format.
371          */
372         ao2_replace(sc->write_frame.subclass.format, ast_format_cache_get_slin_by_rate(rate));
373         sc->write_frame.data.ptr = sc->final_buf;
374         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
375         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
376
377         /* Setup read frame parameters */
378         sc->read_frame.frametype = AST_FRAME_VOICE;
379         /*
380          * NOTE: The read_frame format does not hold a reference because it
381          * will always be a signed linear format.
382          */
383         sc->read_frame.subclass.format = ast_format_cache_get_slin_by_rate(channel_read_rate);
384         sc->read_frame.data.ptr = sc->our_buf;
385         sc->read_frame.datalen = SOFTMIX_DATALEN(channel_read_rate, interval);
386         sc->read_frame.samples = SOFTMIX_SAMPLES(channel_read_rate, interval);
387
388         /* Setup smoother */
389         ast_slinfactory_init_with_format(&sc->factory, sc->write_frame.subclass.format);
390
391         /* set new read and write formats on channel. */
392         ast_set_read_format(bridge_channel->chan, sc->read_frame.subclass.format);
393         ast_set_write_format(bridge_channel->chan, sc->write_frame.subclass.format);
394
395         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
396         sc->dsp = ast_dsp_new_with_rate(channel_read_rate);
397         /* we want to aggressively detect silence to avoid feedback */
398         if (bridge_channel->tech_args.talking_threshold) {
399                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
400         } else {
401                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
402         }
403
404         ast_mutex_unlock(&sc->lock);
405 }
406
407 /*!
408  * \internal
409  * \brief Poke the mixing thread in case it is waiting for an active channel.
410  * \since 12.0.0
411  *
412  * \param softmix_data Bridge mixing data.
413  *
414  * \return Nothing
415  */
416 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
417 {
418         ast_mutex_lock(&softmix_data->lock);
419         ast_cond_signal(&softmix_data->cond);
420         ast_mutex_unlock(&softmix_data->lock);
421 }
422
423 /*! \brief Function called when a channel is unsuspended from the bridge */
424 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
425 {
426         if (bridge->tech_pvt) {
427                 softmix_poke_thread(bridge->tech_pvt);
428         }
429 }
430
431 /*!
432  * \internal
433  * \brief Indicate a source change to the channel.
434  * \since 12.0.0
435  *
436  * \param bridge_channel Which channel source is changing.
437  *
438  * \retval 0 on success.
439  * \retval -1 on error.
440  */
441 static int softmix_src_change(struct ast_bridge_channel *bridge_channel)
442 {
443         return ast_bridge_channel_queue_control_data(bridge_channel, AST_CONTROL_SRCCHANGE, NULL, 0);
444 }
445
446 /*! \brief Function called when a channel is joined into the bridge */
447 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
448 {
449         struct softmix_channel *sc;
450         struct softmix_bridge_data *softmix_data;
451
452         softmix_data = bridge->tech_pvt;
453         if (!softmix_data) {
454                 return -1;
455         }
456
457         /* Create a new softmix_channel structure and allocate various things on it */
458         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
459                 return -1;
460         }
461
462         softmix_src_change(bridge_channel);
463
464         /* Can't forget the lock */
465         ast_mutex_init(&sc->lock);
466
467         /* Can't forget to record our pvt structure within the bridged channel structure */
468         bridge_channel->tech_pvt = sc;
469
470         set_softmix_bridge_data(softmix_data->internal_rate,
471                 softmix_data->internal_mixing_interval
472                         ? softmix_data->internal_mixing_interval
473                         : DEFAULT_SOFTMIX_INTERVAL,
474                 bridge_channel, 0);
475
476         softmix_poke_thread(softmix_data);
477         return 0;
478 }
479
480 /*! \brief Function called when a channel leaves the bridge */
481 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
482 {
483         struct softmix_channel *sc = bridge_channel->tech_pvt;
484
485         if (!sc) {
486                 return;
487         }
488         bridge_channel->tech_pvt = NULL;
489
490         softmix_src_change(bridge_channel);
491
492         /* Drop mutex lock */
493         ast_mutex_destroy(&sc->lock);
494
495         /* Drop the factory */
496         ast_slinfactory_destroy(&sc->factory);
497
498         /* Drop any formats on the frames */
499         ao2_cleanup(sc->write_frame.subclass.format);
500
501         /* Drop the DSP */
502         ast_dsp_free(sc->dsp);
503
504         /* Eep! drop ourselves */
505         ast_free(sc);
506 }
507
508 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
509 {
510         struct ast_bridge_channel *cur;
511
512         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
513                 if (cur->suspended) {
514                         continue;
515                 }
516                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
517                         ast_bridge_channel_queue_frame(cur, frame);
518                         break;
519                 }
520         }
521 }
522
523 /*!
524  * \internal
525  * \brief Determine what to do with a video frame.
526  * \since 12.0.0
527  *
528  * \param bridge Which bridge is getting the frame
529  * \param bridge_channel Which channel is writing the frame.
530  * \param frame What is being written.
531  *
532  * \return Nothing
533  */
534 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
535 {
536         struct softmix_channel *sc;
537         int video_src_priority;
538
539         /* Determine if the video frame should be distributed or not */
540         switch (bridge->softmix.video_mode.mode) {
541         case AST_BRIDGE_VIDEO_MODE_NONE:
542                 break;
543         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
544                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
545                 if (video_src_priority == 1) {
546                         /* Pass to me and everyone else. */
547                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
548                 }
549                 break;
550         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
551                 sc = bridge_channel->tech_pvt;
552                 ast_mutex_lock(&sc->lock);
553                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
554                         sc->video_talker.energy_average,
555                         frame->subclass.frame_ending);
556                 ast_mutex_unlock(&sc->lock);
557                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
558                 if (video_src_priority == 1) {
559                         int num_src = ast_bridge_number_video_src(bridge);
560                         int echo = num_src > 1 ? 0 : 1;
561
562                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
563                 } else if (video_src_priority == 2) {
564                         softmix_pass_video_top_priority(bridge, frame);
565                 }
566                 break;
567         }
568 }
569
570 /*!
571  * \internal
572  * \brief Determine what to do with a voice frame.
573  * \since 12.0.0
574  *
575  * \param bridge Which bridge is getting the frame
576  * \param bridge_channel Which channel is writing the frame.
577  * \param frame What is being written.
578  *
579  * \return Nothing
580  */
581 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
582 {
583         struct softmix_channel *sc = bridge_channel->tech_pvt;
584         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
585         int totalsilence = 0;
586         int cur_energy = 0;
587         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
588                 bridge_channel->tech_args.silence_threshold :
589                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
590         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
591
592         /* Write the frame into the conference */
593         ast_mutex_lock(&sc->lock);
594         ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
595
596         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
597                 int cur_slot = sc->video_talker.energy_history_cur_slot;
598
599                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
600                 sc->video_talker.energy_accum += cur_energy;
601                 sc->video_talker.energy_history[cur_slot] = cur_energy;
602                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
603                 sc->video_talker.energy_history_cur_slot++;
604                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
605                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
606                 }
607         }
608
609         if (totalsilence < silence_threshold) {
610                 if (!sc->talking) {
611                         update_talking = 1;
612                 }
613                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
614         } else {
615                 if (sc->talking) {
616                         update_talking = 0;
617                 }
618                 sc->talking = 0;
619         }
620
621         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
622          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
623          * the audio by flushing the buffer before adding new audio in. */
624         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
625                 ast_slinfactory_flush(&sc->factory);
626         }
627
628         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
629          * is not determined to be talking. */
630         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
631                 ast_slinfactory_feed(&sc->factory, frame);
632         }
633
634         /* Alllll done */
635         ast_mutex_unlock(&sc->lock);
636
637         if (update_talking != -1) {
638                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
639         }
640 }
641
642 /*!
643  * \internal
644  * \brief Determine what to do with a control frame.
645  * \since 12.0.0
646  *
647  * \param bridge Which bridge is getting the frame
648  * \param bridge_channel Which channel is writing the frame.
649  * \param frame What is being written.
650  *
651  * \retval 0 Frame accepted into the bridge.
652  * \retval -1 Frame needs to be deferred.
653  */
654 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
655 {
656         /*
657          * XXX Softmix needs to use channel roles to determine what to
658          * do with control frames.
659          */
660         return 0;
661 }
662
663 /*!
664  * \internal
665  * \brief Determine what to do with a frame written into the bridge.
666  * \since 12.0.0
667  *
668  * \param bridge Which bridge is getting the frame
669  * \param bridge_channel Which channel is writing the frame.
670  * \param frame What is being written.
671  *
672  * \retval 0 Frame accepted into the bridge.
673  * \retval -1 Frame needs to be deferred.
674  *
675  * \note On entry, bridge is already locked.
676  */
677 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
678 {
679         int res = 0;
680
681         if (!bridge->tech_pvt || (bridge_channel && !bridge_channel->tech_pvt)) {
682                 /* "Accept" the frame and discard it. */
683                 return 0;
684         }
685
686         /*
687          * XXX Softmix needs to use channel roles to determine who gets
688          * what frame.  Possible roles: announcer, recorder, agent,
689          * supervisor.
690          */
691         switch (frame->frametype) {
692         case AST_FRAME_NULL:
693                 /* "Accept" the frame and discard it. */
694                 break;
695         case AST_FRAME_DTMF_BEGIN:
696         case AST_FRAME_DTMF_END:
697                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
698                 break;
699         case AST_FRAME_VOICE:
700                 if (bridge_channel) {
701                         softmix_bridge_write_voice(bridge, bridge_channel, frame);
702                 }
703                 break;
704         case AST_FRAME_VIDEO:
705                 if (bridge_channel) {
706                         softmix_bridge_write_video(bridge, bridge_channel, frame);
707                 }
708                 break;
709         case AST_FRAME_CONTROL:
710                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
711                 break;
712         case AST_FRAME_BRIDGE_ACTION:
713                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
714                 break;
715         case AST_FRAME_BRIDGE_ACTION_SYNC:
716                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
717                 ast_assert(0);
718         default:
719                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
720                 /* "Accept" the frame and discard it. */
721                 break;
722         }
723
724         return res;
725 }
726
727 static void gather_softmix_stats(struct softmix_stats *stats,
728         const struct softmix_bridge_data *softmix_data,
729         struct ast_bridge_channel *bridge_channel)
730 {
731         int channel_native_rate;
732         int i;
733         /* Gather stats about channel sample rates. */
734         channel_native_rate = MAX(ast_format_get_sample_rate(ast_channel_rawwriteformat(bridge_channel->chan)),
735                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
736
737         if (channel_native_rate > stats->highest_supported_rate) {
738                 stats->highest_supported_rate = channel_native_rate;
739         }
740         if (channel_native_rate > softmix_data->internal_rate) {
741                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
742                         if (stats->sample_rates[i] == channel_native_rate) {
743                                 stats->num_channels[i]++;
744                                 break;
745                         } else if (!stats->sample_rates[i]) {
746                                 stats->sample_rates[i] = channel_native_rate;
747                                 stats->num_channels[i]++;
748                                 break;
749                         }
750                 }
751                 stats->num_above_internal_rate++;
752         } else if (channel_native_rate == softmix_data->internal_rate) {
753                 stats->num_at_internal_rate++;
754         }
755 }
756 /*!
757  * \internal
758  * \brief Analyse mixing statistics and change bridges internal rate
759  * if necessary.
760  *
761  * \retval 0, no changes to internal rate
762  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
763  */
764 static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data)
765 {
766         int i;
767         /* Re-adjust the internal bridge sample rate if
768          * 1. The bridge's internal sample rate is locked in at a sample
769          *    rate other than the current sample rate being used.
770          * 2. two or more channels support a higher sample rate
771          * 3. no channels support the current sample rate or a higher rate
772          */
773         if (stats->locked_rate) {
774                 /* if the rate is locked by the bridge, only update it if it differs
775                  * from the current rate we are using. */
776                 if (softmix_data->internal_rate != stats->locked_rate) {
777                         softmix_data->internal_rate = stats->locked_rate;
778                         ast_debug(1, "Bridge is locked in at sample rate %u\n",
779                                 softmix_data->internal_rate);
780                         return 1;
781                 }
782         } else if (stats->num_above_internal_rate >= 2) {
783                 /* the highest rate is just used as a starting point */
784                 unsigned int best_rate = stats->highest_supported_rate;
785                 int best_index = -1;
786
787                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
788                         if (stats->num_channels[i]) {
789                                 break;
790                         }
791                         /* best_rate starts out being the first sample rate
792                          * greater than the internal sample rate that 2 or
793                          * more channels support. */
794                         if (stats->num_channels[i] >= 2 && (best_index == -1)) {
795                                 best_rate = stats->sample_rates[i];
796                                 best_index = i;
797                         /* If it has been detected that multiple rates above
798                          * the internal rate are present, compare those rates
799                          * to each other and pick the highest one two or more
800                          * channels support. */
801                         } else if (((best_index != -1) &&
802                                 (stats->num_channels[i] >= 2) &&
803                                 (stats->sample_rates[best_index] < stats->sample_rates[i]))) {
804                                 best_rate = stats->sample_rates[i];
805                                 best_index = i;
806                         /* It is possible that multiple channels exist with native sample
807                          * rates above the internal sample rate, but none of those channels
808                          * have the same rate in common.  In this case, the lowest sample
809                          * rate among those channels is picked. Over time as additional
810                          * statistic runs are made the internal sample rate number will
811                          * adjust to the most optimal sample rate, but it may take multiple
812                          * iterations. */
813                         } else if (best_index == -1) {
814                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
815                         }
816                 }
817
818                 ast_debug(1, "Bridge changed from %u To %u\n",
819                         softmix_data->internal_rate, best_rate);
820                 softmix_data->internal_rate = best_rate;
821                 return 1;
822         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
823                 /* In this case, the highest supported rate is actually lower than the internal rate */
824                 softmix_data->internal_rate = stats->highest_supported_rate;
825                 ast_debug(1, "Bridge changed from %u to %u\n",
826                         softmix_data->internal_rate, stats->highest_supported_rate);
827                 return 1;
828         }
829         return 0;
830 }
831
832 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries)
833 {
834         memset(mixing_array, 0, sizeof(*mixing_array));
835         mixing_array->max_num_entries = starting_num_entries;
836         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
837                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
838                 return -1;
839         }
840         return 0;
841 }
842
843 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array)
844 {
845         ast_free(mixing_array->buffers);
846 }
847
848 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries)
849 {
850         int16_t **tmp;
851         /* give it some room to grow since memory is cheap but allocations can be expensive */
852         mixing_array->max_num_entries = num_entries;
853         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
854                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
855                 return -1;
856         }
857         mixing_array->buffers = tmp;
858         return 0;
859 }
860
861 /*!
862  * \brief Mixing loop.
863  *
864  * \retval 0 on success
865  * \retval -1 on failure
866  */
867 static int softmix_mixing_loop(struct ast_bridge *bridge)
868 {
869         struct softmix_stats stats = { { 0 }, };
870         struct softmix_mixing_array mixing_array;
871         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
872         struct ast_timer *timer;
873         struct softmix_translate_helper trans_helper;
874         int16_t buf[MAX_DATALEN];
875         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
876         int timingfd;
877         int update_all_rates = 0; /* set this when the internal sample rate has changed */
878         unsigned int idx;
879         unsigned int x;
880         int res = -1;
881
882         timer = softmix_data->timer;
883         timingfd = ast_timer_fd(timer);
884         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
885         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
886
887         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
888         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10)) {
889                 goto softmix_cleanup;
890         }
891
892         /*
893          * XXX Softmix needs to use channel roles to determine who gets
894          * what audio mixed.
895          */
896         while (!softmix_data->stop && bridge->num_active) {
897                 struct ast_bridge_channel *bridge_channel;
898                 int timeout = -1;
899                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
900                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
901                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
902
903                 if (softmix_datalen > MAX_DATALEN) {
904                         /* This should NEVER happen, but if it does we need to know about it. Almost
905                          * all the memcpys used during this process depend on this assumption.  Rather
906                          * than checking this over and over again through out the code, this single
907                          * verification is done on each iteration. */
908                         ast_log(LOG_WARNING,
909                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
910                                 bridge->uniqueid);
911                         goto softmix_cleanup;
912                 }
913
914                 /* Grow the mixing array buffer as participants are added. */
915                 if (mixing_array.max_num_entries < bridge->num_channels
916                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5)) {
917                         goto softmix_cleanup;
918                 }
919
920                 /* init the number of buffers stored in the mixing array to 0.
921                  * As buffers are added for mixing, this number is incremented. */
922                 mixing_array.used_entries = 0;
923
924                 /* These variables help determine if a rate change is required */
925                 if (!stat_iteration_counter) {
926                         memset(&stats, 0, sizeof(stats));
927                         stats.locked_rate = bridge->softmix.internal_sample_rate;
928                 }
929
930                 /* If the sample rate has changed, update the translator helper */
931                 if (update_all_rates) {
932                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
933                 }
934
935                 /* Go through pulling audio from each factory that has it available */
936                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
937                         struct softmix_channel *sc = bridge_channel->tech_pvt;
938
939                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
940                         if (update_all_rates) {
941                                 set_softmix_bridge_data(softmix_data->internal_rate, softmix_data->internal_mixing_interval, bridge_channel, 1);
942                         }
943
944                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
945                         if (!stat_iteration_counter) {
946                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
947                         }
948
949                         /* if the channel is suspended, don't check for audio, but still gather stats */
950                         if (bridge_channel->suspended) {
951                                 continue;
952                         }
953
954                         /* Try to get audio from the factory if available */
955                         ast_mutex_lock(&sc->lock);
956                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
957                                 mixing_array.used_entries++;
958                         }
959                         ast_mutex_unlock(&sc->lock);
960                 }
961
962                 /* mix it like crazy */
963                 memset(buf, 0, softmix_datalen);
964                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
965                         for (x = 0; x < softmix_samples; ++x) {
966                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
967                         }
968                 }
969
970                 /* Next step go through removing the channel's own audio and creating a good frame... */
971                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
972                         struct softmix_channel *sc = bridge_channel->tech_pvt;
973
974                         if (bridge_channel->suspended) {
975                                 continue;
976                         }
977
978                         ast_mutex_lock(&sc->lock);
979
980                         /* Make SLINEAR write frame from local buffer */
981                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
982                                 "Replace softmix channel slin format");
983                         sc->write_frame.datalen = softmix_datalen;
984                         sc->write_frame.samples = softmix_samples;
985                         memcpy(sc->final_buf, buf, softmix_datalen);
986
987                         /* process the softmix channel's new write audio */
988                         softmix_process_write_audio(&trans_helper, ast_channel_rawwriteformat(bridge_channel->chan), sc);
989
990                         ast_mutex_unlock(&sc->lock);
991
992                         /* A frame is now ready for the channel. */
993                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
994                 }
995
996                 update_all_rates = 0;
997                 if (!stat_iteration_counter) {
998                         update_all_rates = analyse_softmix_stats(&stats, softmix_data);
999                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1000                 }
1001                 stat_iteration_counter--;
1002
1003                 ast_bridge_unlock(bridge);
1004                 /* cleanup any translation frame data from the previous mixing iteration. */
1005                 softmix_translate_helper_cleanup(&trans_helper);
1006                 /* Wait for the timing source to tell us to wake up and get things done */
1007                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1008                 if (ast_timer_ack(timer, 1) < 0) {
1009                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1010                                 bridge->uniqueid);
1011                         ast_bridge_lock(bridge);
1012                         goto softmix_cleanup;
1013                 }
1014                 ast_bridge_lock(bridge);
1015
1016                 /* make sure to detect mixing interval changes if they occur. */
1017                 if (bridge->softmix.internal_mixing_interval
1018                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1019                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1020                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1021                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1022                 }
1023         }
1024
1025         res = 0;
1026
1027 softmix_cleanup:
1028         softmix_translate_helper_destroy(&trans_helper);
1029         softmix_mixing_array_destroy(&mixing_array);
1030         return res;
1031 }
1032
1033 /*!
1034  * \internal
1035  * \brief Mixing thread.
1036  * \since 12.0.0
1037  *
1038  * \note The thread does not have its own reference to the
1039  * bridge.  The lifetime of the thread is tied to the lifetime
1040  * of the mixing technology association with the bridge.
1041  */
1042 static void *softmix_mixing_thread(void *data)
1043 {
1044         struct softmix_bridge_data *softmix_data = data;
1045         struct ast_bridge *bridge = softmix_data->bridge;
1046
1047         ast_bridge_lock(bridge);
1048         if (bridge->callid) {
1049                 ast_callid_threadassoc_add(bridge->callid);
1050         }
1051
1052         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1053
1054         while (!softmix_data->stop) {
1055                 if (!bridge->num_active) {
1056                         /* Wait for something to happen to the bridge. */
1057                         ast_bridge_unlock(bridge);
1058                         ast_mutex_lock(&softmix_data->lock);
1059                         if (!softmix_data->stop) {
1060                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1061                         }
1062                         ast_mutex_unlock(&softmix_data->lock);
1063                         ast_bridge_lock(bridge);
1064                         continue;
1065                 }
1066
1067                 if (softmix_mixing_loop(bridge)) {
1068                         /*
1069                          * A mixing error occurred.  Sleep and try again later so we
1070                          * won't flood the logs.
1071                          */
1072                         ast_bridge_unlock(bridge);
1073                         sleep(1);
1074                         ast_bridge_lock(bridge);
1075                 }
1076         }
1077
1078         ast_bridge_unlock(bridge);
1079
1080         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1081
1082         return NULL;
1083 }
1084
1085 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1086 {
1087         if (softmix_data->timer) {
1088                 ast_timer_close(softmix_data->timer);
1089                 softmix_data->timer = NULL;
1090         }
1091         ast_mutex_destroy(&softmix_data->lock);
1092         ast_cond_destroy(&softmix_data->cond);
1093         ast_free(softmix_data);
1094 }
1095
1096 /*! \brief Function called when a bridge is created */
1097 static int softmix_bridge_create(struct ast_bridge *bridge)
1098 {
1099         struct softmix_bridge_data *softmix_data;
1100
1101         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1102         if (!softmix_data) {
1103                 return -1;
1104         }
1105         softmix_data->bridge = bridge;
1106         ast_mutex_init(&softmix_data->lock);
1107         ast_cond_init(&softmix_data->cond, NULL);
1108         softmix_data->timer = ast_timer_open();
1109         if (!softmix_data->timer) {
1110                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1111                 softmix_bridge_data_destroy(softmix_data);
1112                 return -1;
1113         }
1114         /* start at 8khz, let it grow from there */
1115         softmix_data->internal_rate = 8000;
1116         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1117
1118         bridge->tech_pvt = softmix_data;
1119
1120         /* Start the mixing thread. */
1121         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1122                 softmix_data)) {
1123                 softmix_data->thread = AST_PTHREADT_NULL;
1124                 softmix_bridge_data_destroy(softmix_data);
1125                 bridge->tech_pvt = NULL;
1126                 return -1;
1127         }
1128
1129         return 0;
1130 }
1131
1132 /*!
1133  * \internal
1134  * \brief Request the softmix mixing thread stop.
1135  * \since 12.0.0
1136  *
1137  * \param bridge Which bridge is being stopped.
1138  *
1139  * \return Nothing
1140  */
1141 static void softmix_bridge_stop(struct ast_bridge *bridge)
1142 {
1143         struct softmix_bridge_data *softmix_data;
1144
1145         softmix_data = bridge->tech_pvt;
1146         if (!softmix_data) {
1147                 return;
1148         }
1149
1150         ast_mutex_lock(&softmix_data->lock);
1151         softmix_data->stop = 1;
1152         ast_mutex_unlock(&softmix_data->lock);
1153 }
1154
1155 /*! \brief Function called when a bridge is destroyed */
1156 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1157 {
1158         struct softmix_bridge_data *softmix_data;
1159         pthread_t thread;
1160
1161         softmix_data = bridge->tech_pvt;
1162         if (!softmix_data) {
1163                 return;
1164         }
1165
1166         /* Stop the mixing thread. */
1167         ast_mutex_lock(&softmix_data->lock);
1168         softmix_data->stop = 1;
1169         ast_cond_signal(&softmix_data->cond);
1170         thread = softmix_data->thread;
1171         softmix_data->thread = AST_PTHREADT_NULL;
1172         ast_mutex_unlock(&softmix_data->lock);
1173         if (thread != AST_PTHREADT_NULL) {
1174                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1175                 pthread_join(thread, NULL);
1176         }
1177
1178         softmix_bridge_data_destroy(softmix_data);
1179         bridge->tech_pvt = NULL;
1180 }
1181
1182 static struct ast_bridge_technology softmix_bridge = {
1183         .name = "softmix",
1184         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1185         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1186         .create = softmix_bridge_create,
1187         .stop = softmix_bridge_stop,
1188         .destroy = softmix_bridge_destroy,
1189         .join = softmix_bridge_join,
1190         .leave = softmix_bridge_leave,
1191         .unsuspend = softmix_bridge_unsuspend,
1192         .write = softmix_bridge_write,
1193 };
1194
1195 static int unload_module(void)
1196 {
1197         ast_bridge_technology_unregister(&softmix_bridge);
1198         return 0;
1199 }
1200
1201 static int load_module(void)
1202 {
1203         if (ast_bridge_technology_register(&softmix_bridge)) {
1204                 unload_module();
1205                 return AST_MODULE_LOAD_DECLINE;
1206         }
1207         return AST_MODULE_LOAD_SUCCESS;
1208 }
1209
1210 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");