bridge_softmix.c,channel.c: Minor code simplification and cleanup.
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
37
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <sys/time.h>
42 #include <signal.h>
43 #include <errno.h>
44 #include <unistd.h>
45
46 #include "asterisk/module.h"
47 #include "asterisk/channel.h"
48 #include "asterisk/bridge.h"
49 #include "asterisk/bridge_technology.h"
50 #include "asterisk/frame.h"
51 #include "asterisk/options.h"
52 #include "asterisk/logger.h"
53 #include "asterisk/slinfactory.h"
54 #include "asterisk/astobj2.h"
55 #include "asterisk/timing.h"
56 #include "asterisk/translate.h"
57
58 #define MAX_DATALEN 8096
59
60 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
61 #define DEFAULT_SOFTMIX_INTERVAL 20
62
63 /*! \brief Size of the buffer used for sample manipulation */
64 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
65
66 /*! \brief Number of samples we are dealing with */
67 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
68
69 /*! \brief Number of mixing iterations to perform between gathering statistics. */
70 #define SOFTMIX_STAT_INTERVAL 100
71
72 /* This is the threshold in ms at which a channel's own audio will stop getting
73  * mixed out its own write audio stream because it is not talking. */
74 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
75 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
76
77 #define DEFAULT_ENERGY_HISTORY_LEN 150
78
79 struct video_follow_talker_data {
80         /*! audio energy history */
81         int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
82         /*! The current slot being used in the history buffer, this
83          *  increments and wraps around */
84         int energy_history_cur_slot;
85         /*! The current energy sum used for averages. */
86         int energy_accum;
87         /*! The current energy average */
88         int energy_average;
89 };
90
91 /*! \brief Structure which contains per-channel mixing information */
92 struct softmix_channel {
93         /*! Lock to protect this structure */
94         ast_mutex_t lock;
95         /*! Factory which contains audio read in from the channel */
96         struct ast_slinfactory factory;
97         /*! Frame that contains mixed audio to be written out to the channel */
98         struct ast_frame write_frame;
99         /*! Frame that contains mixed audio read from the channel */
100         struct ast_frame read_frame;
101         /*! DSP for detecting silence */
102         struct ast_dsp *dsp;
103         /*!
104          * \brief TRUE if a channel is talking.
105          *
106          * \note This affects how the channel's audio is mixed back to
107          * it.
108          */
109         unsigned int talking:1;
110         /*! TRUE if the channel provided audio for this mixing interval */
111         unsigned int have_audio:1;
112         /*! Buffer containing final mixed audio from all sources */
113         short final_buf[MAX_DATALEN];
114         /*! Buffer containing only the audio from the channel */
115         short our_buf[MAX_DATALEN];
116         /*! Data pertaining to talker mode for video conferencing */
117         struct video_follow_talker_data video_talker;
118 };
119
120 struct softmix_bridge_data {
121         struct ast_timer *timer;
122         /*!
123          * \brief Bridge pointer passed to the softmix mixing thread.
124          *
125          * \note Does not need a reference because the bridge will
126          * always exist while the mixing thread exists even if the
127          * bridge is no longer actively using the softmix technology.
128          */
129         struct ast_bridge *bridge;
130         /*! Lock for signaling the mixing thread. */
131         ast_mutex_t lock;
132         /*! Condition, used if we need to wake up the mixing thread. */
133         ast_cond_t cond;
134         /*! Thread handling the mixing */
135         pthread_t thread;
136         unsigned int internal_rate;
137         unsigned int internal_mixing_interval;
138         /*! TRUE if the mixing thread should stop */
139         unsigned int stop:1;
140 };
141
142 struct softmix_stats {
143         /*! Each index represents a sample rate used above the internal rate. */
144         unsigned int sample_rates[16];
145         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
146         unsigned int num_channels[16];
147         /*! The number of channels above the internal sample rate */
148         unsigned int num_above_internal_rate;
149         /*! The number of channels at the internal sample rate */
150         unsigned int num_at_internal_rate;
151         /*! The absolute highest sample rate preferred by any channel in the bridge */
152         unsigned int highest_supported_rate;
153         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
154         unsigned int locked_rate;
155 };
156
157 struct softmix_mixing_array {
158         unsigned int max_num_entries;
159         unsigned int used_entries;
160         int16_t **buffers;
161 };
162
163 struct softmix_translate_helper_entry {
164         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
165                                       and re-init if it was usable. */
166         struct ast_format *dst_format; /*!< The destination format for this helper */
167         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
168         struct ast_frame *out_frame; /*!< The output frame from the last translation */
169         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
170 };
171
172 struct softmix_translate_helper {
173         struct ast_format *slin_src; /*!< the source format expected for all the translators */
174         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
175 };
176
177 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
178 {
179         struct softmix_translate_helper_entry *entry;
180         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
181                 return NULL;
182         }
183         entry->dst_format = ao2_bump(dst);
184         /* initialize this to one so that the first time through the cleanup code after
185            allocation it won't be removed from the entry list */
186         entry->num_times_requested = 1;
187         return entry;
188 }
189
190 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
191 {
192         ao2_cleanup(entry->dst_format);
193
194         if (entry->trans_pvt) {
195                 ast_translator_free_path(entry->trans_pvt);
196         }
197         if (entry->out_frame) {
198                 ast_frfree(entry->out_frame);
199         }
200         ast_free(entry);
201         return NULL;
202 }
203
204 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
205 {
206         memset(trans_helper, 0, sizeof(*trans_helper));
207         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
208 }
209
210 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
211 {
212         struct softmix_translate_helper_entry *entry;
213
214         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
215                 softmix_translate_helper_free_entry(entry);
216         }
217 }
218
219 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
220 {
221         struct softmix_translate_helper_entry *entry;
222
223         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
224         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
225                 if (entry->trans_pvt) {
226                         ast_translator_free_path(entry->trans_pvt);
227                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
228                                 AST_LIST_REMOVE_CURRENT(entry);
229                                 entry = softmix_translate_helper_free_entry(entry);
230                         }
231                 }
232         }
233         AST_LIST_TRAVERSE_SAFE_END;
234 }
235
236 /*!
237  * \internal
238  * \brief Get the next available audio on the softmix channel's read stream
239  * and determine if it should be mixed out or not on the write stream.
240  *
241  * \retval pointer to buffer containing the exact number of samples requested on success.
242  * \retval NULL if no samples are present
243  */
244 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
245 {
246         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
247                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
248                 sc->have_audio = 1;
249                 return sc->our_buf;
250         }
251         sc->have_audio = 0;
252         return NULL;
253 }
254
255 /*!
256  * \internal
257  * \brief Process a softmix channel's write audio
258  *
259  * \details This function will remove the channel's talking from its own audio if present and
260  * possibly even do the channel's write translation for it depending on how many other
261  * channels use the same write format.
262  */
263 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
264         struct ast_format *raw_write_fmt,
265         struct softmix_channel *sc)
266 {
267         struct softmix_translate_helper_entry *entry = NULL;
268         int i;
269
270         /* If we provided audio that was not determined to be silence,
271          * then take it out while in slinear format. */
272         if (sc->have_audio && sc->talking) {
273                 for (i = 0; i < sc->write_frame.samples; i++) {
274                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
275                 }
276                 /* check to see if any entries exist for the format. if not we'll want
277                    to remove it during cleanup */
278                 AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
279                         if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
280                                 ++entry->num_times_requested;
281                                 break;
282                         }
283                 }
284                 /* do not do any special write translate optimization if we had to make
285                  * a special mix for them to remove their own audio. */
286                 return;
287         }
288
289         /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
290            of translation paths and track the number of references for each type. Each one of the same
291            type should be able to use the same out_frame. Since the optimization is only necessary for
292            multiple channels (>=2) using the same codec make sure resources are allocated only when
293            needed and released when not (see also softmix_translate_helper_cleanup */
294         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
295                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
296                         entry->num_times_requested++;
297                 } else {
298                         continue;
299                 }
300                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
301                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
302                 }
303                 if (entry->trans_pvt && !entry->out_frame) {
304                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
305                 }
306                 if (entry->out_frame && (entry->out_frame->datalen < MAX_DATALEN)) {
307                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
308                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
309                         sc->write_frame.datalen = entry->out_frame->datalen;
310                         sc->write_frame.samples = entry->out_frame->samples;
311                 }
312                 break;
313         }
314
315         /* add new entry into list if this format destination was not matched. */
316         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
317                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
318         }
319 }
320
321 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
322 {
323         struct softmix_translate_helper_entry *entry;
324
325         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
326                 /* if it hasn't been requested then remove it */
327                 if (!entry->num_times_requested) {
328                         AST_LIST_REMOVE_CURRENT(entry);
329                         softmix_translate_helper_free_entry(entry);
330                         continue;
331                 }
332
333                 if (entry->out_frame) {
334                         ast_frfree(entry->out_frame);
335                         entry->out_frame = NULL;
336                 }
337
338                 /* nothing is optimized for a single path reference, so there is
339                    no reason to continue to hold onto the codec */
340                 if (entry->num_times_requested == 1 && entry->trans_pvt) {
341                         ast_translator_free_path(entry->trans_pvt);
342                         entry->trans_pvt = NULL;
343                 }
344
345                 /* for each iteration (a mixing run) in the bridge softmix thread the number
346                    of references to a given entry is recalculated, so reset the number of
347                    times requested */
348                 entry->num_times_requested = 0;
349         }
350         AST_LIST_TRAVERSE_SAFE_END;
351 }
352
353 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset)
354 {
355         struct softmix_channel *sc = bridge_channel->tech_pvt;
356         unsigned int channel_read_rate = ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan));
357
358         ast_mutex_lock(&sc->lock);
359         if (reset) {
360                 ast_slinfactory_destroy(&sc->factory);
361                 ast_dsp_free(sc->dsp);
362         }
363
364         /* Setup write frame parameters */
365         sc->write_frame.frametype = AST_FRAME_VOICE;
366         /*
367          * NOTE: The write_frame format holds a reference because translation
368          * could be needed and the format changed to the translated format
369          * for the channel.  The translated format may not be a
370          * static cached format.
371          */
372         ao2_replace(sc->write_frame.subclass.format, ast_format_cache_get_slin_by_rate(rate));
373         sc->write_frame.data.ptr = sc->final_buf;
374         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
375         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
376
377         /* Setup read frame parameters */
378         sc->read_frame.frametype = AST_FRAME_VOICE;
379         /*
380          * NOTE: The read_frame format does not hold a reference because it
381          * will always be a signed linear format.
382          */
383         sc->read_frame.subclass.format = ast_format_cache_get_slin_by_rate(channel_read_rate);
384         sc->read_frame.data.ptr = sc->our_buf;
385         sc->read_frame.datalen = SOFTMIX_DATALEN(channel_read_rate, interval);
386         sc->read_frame.samples = SOFTMIX_SAMPLES(channel_read_rate, interval);
387
388         /* Setup smoother */
389         ast_slinfactory_init_with_format(&sc->factory, sc->write_frame.subclass.format);
390
391         /* set new read and write formats on channel. */
392         ast_set_read_format(bridge_channel->chan, sc->read_frame.subclass.format);
393         ast_set_write_format(bridge_channel->chan, sc->write_frame.subclass.format);
394
395         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
396         sc->dsp = ast_dsp_new_with_rate(channel_read_rate);
397         /* we want to aggressively detect silence to avoid feedback */
398         if (bridge_channel->tech_args.talking_threshold) {
399                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
400         } else {
401                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
402         }
403
404         ast_mutex_unlock(&sc->lock);
405 }
406
407 /*!
408  * \internal
409  * \brief Poke the mixing thread in case it is waiting for an active channel.
410  * \since 12.0.0
411  *
412  * \param softmix_data Bridge mixing data.
413  *
414  * \return Nothing
415  */
416 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
417 {
418         ast_mutex_lock(&softmix_data->lock);
419         ast_cond_signal(&softmix_data->cond);
420         ast_mutex_unlock(&softmix_data->lock);
421 }
422
423 /*! \brief Function called when a channel is unsuspended from the bridge */
424 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
425 {
426         if (bridge->tech_pvt) {
427                 softmix_poke_thread(bridge->tech_pvt);
428         }
429 }
430
431 /*!
432  * \internal
433  * \brief Indicate a source change to the channel.
434  * \since 12.0.0
435  *
436  * \param bridge_channel Which channel source is changing.
437  *
438  * \retval 0 on success.
439  * \retval -1 on error.
440  */
441 static int softmix_src_change(struct ast_bridge_channel *bridge_channel)
442 {
443         return ast_bridge_channel_queue_control_data(bridge_channel, AST_CONTROL_SRCCHANGE, NULL, 0);
444 }
445
446 /*! \brief Function called when a channel is joined into the bridge */
447 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
448 {
449         struct softmix_channel *sc;
450         struct softmix_bridge_data *softmix_data;
451
452         softmix_data = bridge->tech_pvt;
453         if (!softmix_data) {
454                 return -1;
455         }
456
457         /* Create a new softmix_channel structure and allocate various things on it */
458         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
459                 return -1;
460         }
461
462         softmix_src_change(bridge_channel);
463
464         /* Can't forget the lock */
465         ast_mutex_init(&sc->lock);
466
467         /* Can't forget to record our pvt structure within the bridged channel structure */
468         bridge_channel->tech_pvt = sc;
469
470         set_softmix_bridge_data(softmix_data->internal_rate,
471                 softmix_data->internal_mixing_interval
472                         ? softmix_data->internal_mixing_interval
473                         : DEFAULT_SOFTMIX_INTERVAL,
474                 bridge_channel, 0);
475
476         softmix_poke_thread(softmix_data);
477         return 0;
478 }
479
480 /*! \brief Function called when a channel leaves the bridge */
481 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
482 {
483         struct softmix_channel *sc = bridge_channel->tech_pvt;
484
485         if (!sc) {
486                 return;
487         }
488         bridge_channel->tech_pvt = NULL;
489
490         softmix_src_change(bridge_channel);
491
492         /* Drop mutex lock */
493         ast_mutex_destroy(&sc->lock);
494
495         /* Drop the factory */
496         ast_slinfactory_destroy(&sc->factory);
497
498         /* Drop any formats on the frames */
499         ao2_cleanup(sc->write_frame.subclass.format);
500
501         /* Drop the DSP */
502         ast_dsp_free(sc->dsp);
503
504         /* Eep! drop ourselves */
505         ast_free(sc);
506 }
507
508 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
509 {
510         struct ast_bridge_channel *cur;
511
512         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
513                 if (cur->suspended) {
514                         continue;
515                 }
516                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
517                         ast_bridge_channel_queue_frame(cur, frame);
518                         break;
519                 }
520         }
521 }
522
523 /*!
524  * \internal
525  * \brief Determine what to do with a video frame.
526  * \since 12.0.0
527  *
528  * \param bridge Which bridge is getting the frame
529  * \param bridge_channel Which channel is writing the frame.
530  * \param frame What is being written.
531  *
532  * \return Nothing
533  */
534 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
535 {
536         struct softmix_channel *sc;
537         int video_src_priority;
538
539         /* Determine if the video frame should be distributed or not */
540         switch (bridge->softmix.video_mode.mode) {
541         case AST_BRIDGE_VIDEO_MODE_NONE:
542                 break;
543         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
544                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
545                 if (video_src_priority == 1) {
546                         /* Pass to me and everyone else. */
547                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
548                 }
549                 break;
550         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
551                 sc = bridge_channel->tech_pvt;
552                 ast_mutex_lock(&sc->lock);
553                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
554                         sc->video_talker.energy_average,
555                         frame->subclass.frame_ending);
556                 ast_mutex_unlock(&sc->lock);
557                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
558                 if (video_src_priority == 1) {
559                         int num_src = ast_bridge_number_video_src(bridge);
560                         int echo = num_src > 1 ? 0 : 1;
561
562                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
563                 } else if (video_src_priority == 2) {
564                         softmix_pass_video_top_priority(bridge, frame);
565                 }
566                 break;
567         }
568 }
569
570 /*!
571  * \internal
572  * \brief Determine what to do with a voice frame.
573  * \since 12.0.0
574  *
575  * \param bridge Which bridge is getting the frame
576  * \param bridge_channel Which channel is writing the frame.
577  * \param frame What is being written.
578  *
579  * \return Nothing
580  */
581 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
582 {
583         struct softmix_channel *sc = bridge_channel->tech_pvt;
584         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
585         int totalsilence = 0;
586         int cur_energy = 0;
587         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
588                 bridge_channel->tech_args.silence_threshold :
589                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
590         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
591
592         /* Write the frame into the conference */
593         ast_mutex_lock(&sc->lock);
594         ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
595
596         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
597                 int cur_slot = sc->video_talker.energy_history_cur_slot;
598
599                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
600                 sc->video_talker.energy_accum += cur_energy;
601                 sc->video_talker.energy_history[cur_slot] = cur_energy;
602                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
603                 sc->video_talker.energy_history_cur_slot++;
604                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
605                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
606                 }
607         }
608
609         if (totalsilence < silence_threshold) {
610                 if (!sc->talking) {
611                         update_talking = 1;
612                 }
613                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
614         } else {
615                 if (sc->talking) {
616                         update_talking = 0;
617                 }
618                 sc->talking = 0;
619         }
620
621         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
622          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
623          * the audio by flushing the buffer before adding new audio in. */
624         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
625                 ast_slinfactory_flush(&sc->factory);
626         }
627
628         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
629          * is not determined to be talking. */
630         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
631                 ast_slinfactory_feed(&sc->factory, frame);
632         }
633
634         /* Alllll done */
635         ast_mutex_unlock(&sc->lock);
636
637         if (update_talking != -1) {
638                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
639         }
640 }
641
642 /*!
643  * \internal
644  * \brief Determine what to do with a control frame.
645  * \since 12.0.0
646  *
647  * \param bridge Which bridge is getting the frame
648  * \param bridge_channel Which channel is writing the frame.
649  * \param frame What is being written.
650  *
651  * \retval 0 Frame accepted into the bridge.
652  * \retval -1 Frame needs to be deferred.
653  */
654 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
655 {
656         /*
657          * XXX Softmix needs to use channel roles to determine what to
658          * do with control frames.
659          */
660         return 0;
661 }
662
663 /*!
664  * \internal
665  * \brief Determine what to do with a frame written into the bridge.
666  * \since 12.0.0
667  *
668  * \param bridge Which bridge is getting the frame
669  * \param bridge_channel Which channel is writing the frame.
670  * \param frame What is being written.
671  *
672  * \retval 0 Frame accepted into the bridge.
673  * \retval -1 Frame needs to be deferred.
674  *
675  * \note On entry, bridge is already locked.
676  */
677 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
678 {
679         int res = 0;
680
681         if (!bridge->tech_pvt || (bridge_channel && !bridge_channel->tech_pvt)) {
682                 /* "Accept" the frame and discard it. */
683                 return 0;
684         }
685
686         /*
687          * XXX Softmix needs to use channel roles to determine who gets
688          * what frame.  Possible roles: announcer, recorder, agent,
689          * supervisor.
690          */
691         switch (frame->frametype) {
692         case AST_FRAME_NULL:
693                 /* "Accept" the frame and discard it. */
694                 break;
695         case AST_FRAME_DTMF_BEGIN:
696         case AST_FRAME_DTMF_END:
697                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
698                 break;
699         case AST_FRAME_VOICE:
700                 if (bridge_channel) {
701                         softmix_bridge_write_voice(bridge, bridge_channel, frame);
702                 }
703                 break;
704         case AST_FRAME_VIDEO:
705                 if (bridge_channel) {
706                         softmix_bridge_write_video(bridge, bridge_channel, frame);
707                 }
708                 break;
709         case AST_FRAME_CONTROL:
710                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
711                 break;
712         case AST_FRAME_BRIDGE_ACTION:
713                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
714                 break;
715         case AST_FRAME_BRIDGE_ACTION_SYNC:
716                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
717                 ast_assert(0);
718         default:
719                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
720                 /* "Accept" the frame and discard it. */
721                 break;
722         }
723
724         return res;
725 }
726
727 static void gather_softmix_stats(struct softmix_stats *stats,
728         const struct softmix_bridge_data *softmix_data,
729         struct ast_bridge_channel *bridge_channel)
730 {
731         int channel_native_rate;
732         int i;
733         /* Gather stats about channel sample rates. */
734         channel_native_rate = MAX(ast_format_get_sample_rate(ast_channel_rawwriteformat(bridge_channel->chan)),
735                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
736
737         if (channel_native_rate > stats->highest_supported_rate) {
738                 stats->highest_supported_rate = channel_native_rate;
739         }
740         if (channel_native_rate > softmix_data->internal_rate) {
741                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
742                         if (stats->sample_rates[i] == channel_native_rate) {
743                                 stats->num_channels[i]++;
744                                 break;
745                         } else if (!stats->sample_rates[i]) {
746                                 stats->sample_rates[i] = channel_native_rate;
747                                 stats->num_channels[i]++;
748                                 break;
749                         }
750                 }
751                 stats->num_above_internal_rate++;
752         } else if (channel_native_rate == softmix_data->internal_rate) {
753                 stats->num_at_internal_rate++;
754         }
755 }
756
757 /*!
758  * \internal
759  * \brief Analyse mixing statistics and change bridges internal rate
760  * if necessary.
761  *
762  * \retval 0, no changes to internal rate
763  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
764  */
765 static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data)
766 {
767         int i;
768
769         /*
770          * Re-adjust the internal bridge sample rate if
771          * 1. The bridge's internal sample rate is locked in at a sample
772          *    rate other than the current sample rate being used.
773          * 2. two or more channels support a higher sample rate
774          * 3. no channels support the current sample rate or a higher rate
775          */
776         if (stats->locked_rate) {
777                 /* if the rate is locked by the bridge, only update it if it differs
778                  * from the current rate we are using. */
779                 if (softmix_data->internal_rate != stats->locked_rate) {
780                         ast_debug(1, "Locking at new rate.  Bridge changed from %u to %u.\n",
781                                 softmix_data->internal_rate, stats->locked_rate);
782                         softmix_data->internal_rate = stats->locked_rate;
783                         return 1;
784                 }
785         } else if (stats->num_above_internal_rate >= 2) {
786                 /* the highest rate is just used as a starting point */
787                 unsigned int best_rate = stats->highest_supported_rate;
788                 int best_index = -1;
789
790                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
791                         if (stats->num_channels[i]) {
792                                 break;
793                         }
794                         if (2 <= stats->num_channels[i]) {
795                                 /* Two or more channels support this rate. */
796                                 if (best_index == -1
797                                         || stats->sample_rates[best_index] < stats->sample_rates[i]) {
798                                         /*
799                                          * best_rate starts out being the first sample rate
800                                          * greater than the internal sample rate that two or
801                                          * more channels support.
802                                          *
803                                          * or
804                                          *
805                                          * There are multiple rates above the internal rate
806                                          * and this rate is higher than the previous rate two
807                                          * or more channels support.
808                                          */
809                                         best_rate = stats->sample_rates[i];
810                                         best_index = i;
811                                 }
812                         } else if (best_index == -1) {
813                                 /*
814                                  * It is possible that multiple channels exist with native sample
815                                  * rates above the internal sample rate, but none of those channels
816                                  * have the same rate in common.  In this case, the lowest sample
817                                  * rate among those channels is picked. Over time as additional
818                                  * statistic runs are made the internal sample rate number will
819                                  * adjust to the most optimal sample rate, but it may take multiple
820                                  * iterations.
821                                  */
822                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
823                         }
824                 }
825
826                 ast_debug(1, "Multiple above internal rate.  Bridge changed from %u to %u.\n",
827                         softmix_data->internal_rate, best_rate);
828                 softmix_data->internal_rate = best_rate;
829                 return 1;
830         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
831                 /* In this case, the highest supported rate is actually lower than the internal rate */
832                 ast_debug(1, "All below internal rate.  Bridge changed from %u to %u.\n",
833                         softmix_data->internal_rate, stats->highest_supported_rate);
834                 softmix_data->internal_rate = stats->highest_supported_rate;
835                 return 1;
836         }
837         return 0;
838 }
839
840 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries)
841 {
842         memset(mixing_array, 0, sizeof(*mixing_array));
843         mixing_array->max_num_entries = starting_num_entries;
844         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
845                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
846                 return -1;
847         }
848         return 0;
849 }
850
851 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array)
852 {
853         ast_free(mixing_array->buffers);
854 }
855
856 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries)
857 {
858         int16_t **tmp;
859         /* give it some room to grow since memory is cheap but allocations can be expensive */
860         mixing_array->max_num_entries = num_entries;
861         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
862                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
863                 return -1;
864         }
865         mixing_array->buffers = tmp;
866         return 0;
867 }
868
869 /*!
870  * \brief Mixing loop.
871  *
872  * \retval 0 on success
873  * \retval -1 on failure
874  */
875 static int softmix_mixing_loop(struct ast_bridge *bridge)
876 {
877         struct softmix_stats stats = { { 0 }, };
878         struct softmix_mixing_array mixing_array;
879         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
880         struct ast_timer *timer;
881         struct softmix_translate_helper trans_helper;
882         int16_t buf[MAX_DATALEN];
883         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
884         int timingfd;
885         int update_all_rates = 0; /* set this when the internal sample rate has changed */
886         unsigned int idx;
887         unsigned int x;
888         int res = -1;
889
890         timer = softmix_data->timer;
891         timingfd = ast_timer_fd(timer);
892         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
893         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
894
895         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
896         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10)) {
897                 goto softmix_cleanup;
898         }
899
900         /*
901          * XXX Softmix needs to use channel roles to determine who gets
902          * what audio mixed.
903          */
904         while (!softmix_data->stop && bridge->num_active) {
905                 struct ast_bridge_channel *bridge_channel;
906                 int timeout = -1;
907                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
908                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
909                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
910
911                 if (softmix_datalen > MAX_DATALEN) {
912                         /* This should NEVER happen, but if it does we need to know about it. Almost
913                          * all the memcpys used during this process depend on this assumption.  Rather
914                          * than checking this over and over again through out the code, this single
915                          * verification is done on each iteration. */
916                         ast_log(LOG_WARNING,
917                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
918                                 bridge->uniqueid);
919                         goto softmix_cleanup;
920                 }
921
922                 /* Grow the mixing array buffer as participants are added. */
923                 if (mixing_array.max_num_entries < bridge->num_channels
924                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5)) {
925                         goto softmix_cleanup;
926                 }
927
928                 /* init the number of buffers stored in the mixing array to 0.
929                  * As buffers are added for mixing, this number is incremented. */
930                 mixing_array.used_entries = 0;
931
932                 /* These variables help determine if a rate change is required */
933                 if (!stat_iteration_counter) {
934                         memset(&stats, 0, sizeof(stats));
935                         stats.locked_rate = bridge->softmix.internal_sample_rate;
936                 }
937
938                 /* If the sample rate has changed, update the translator helper */
939                 if (update_all_rates) {
940                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
941                 }
942
943                 /* Go through pulling audio from each factory that has it available */
944                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
945                         struct softmix_channel *sc = bridge_channel->tech_pvt;
946
947                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
948                         if (update_all_rates) {
949                                 set_softmix_bridge_data(softmix_data->internal_rate, softmix_data->internal_mixing_interval, bridge_channel, 1);
950                         }
951
952                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
953                         if (!stat_iteration_counter) {
954                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
955                         }
956
957                         /* if the channel is suspended, don't check for audio, but still gather stats */
958                         if (bridge_channel->suspended) {
959                                 continue;
960                         }
961
962                         /* Try to get audio from the factory if available */
963                         ast_mutex_lock(&sc->lock);
964                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
965                                 mixing_array.used_entries++;
966                         }
967                         ast_mutex_unlock(&sc->lock);
968                 }
969
970                 /* mix it like crazy */
971                 memset(buf, 0, softmix_datalen);
972                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
973                         for (x = 0; x < softmix_samples; ++x) {
974                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
975                         }
976                 }
977
978                 /* Next step go through removing the channel's own audio and creating a good frame... */
979                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
980                         struct softmix_channel *sc = bridge_channel->tech_pvt;
981
982                         if (bridge_channel->suspended) {
983                                 continue;
984                         }
985
986                         ast_mutex_lock(&sc->lock);
987
988                         /* Make SLINEAR write frame from local buffer */
989                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
990                                 "Replace softmix channel slin format");
991                         sc->write_frame.datalen = softmix_datalen;
992                         sc->write_frame.samples = softmix_samples;
993                         memcpy(sc->final_buf, buf, softmix_datalen);
994
995                         /* process the softmix channel's new write audio */
996                         softmix_process_write_audio(&trans_helper, ast_channel_rawwriteformat(bridge_channel->chan), sc);
997
998                         ast_mutex_unlock(&sc->lock);
999
1000                         /* A frame is now ready for the channel. */
1001                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1002                 }
1003
1004                 update_all_rates = 0;
1005                 if (!stat_iteration_counter) {
1006                         update_all_rates = analyse_softmix_stats(&stats, softmix_data);
1007                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1008                 }
1009                 stat_iteration_counter--;
1010
1011                 ast_bridge_unlock(bridge);
1012                 /* cleanup any translation frame data from the previous mixing iteration. */
1013                 softmix_translate_helper_cleanup(&trans_helper);
1014                 /* Wait for the timing source to tell us to wake up and get things done */
1015                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1016                 if (ast_timer_ack(timer, 1) < 0) {
1017                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1018                                 bridge->uniqueid);
1019                         ast_bridge_lock(bridge);
1020                         goto softmix_cleanup;
1021                 }
1022                 ast_bridge_lock(bridge);
1023
1024                 /* make sure to detect mixing interval changes if they occur. */
1025                 if (bridge->softmix.internal_mixing_interval
1026                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1027                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
1028                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1029                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1030                 }
1031         }
1032
1033         res = 0;
1034
1035 softmix_cleanup:
1036         softmix_translate_helper_destroy(&trans_helper);
1037         softmix_mixing_array_destroy(&mixing_array);
1038         return res;
1039 }
1040
1041 /*!
1042  * \internal
1043  * \brief Mixing thread.
1044  * \since 12.0.0
1045  *
1046  * \note The thread does not have its own reference to the
1047  * bridge.  The lifetime of the thread is tied to the lifetime
1048  * of the mixing technology association with the bridge.
1049  */
1050 static void *softmix_mixing_thread(void *data)
1051 {
1052         struct softmix_bridge_data *softmix_data = data;
1053         struct ast_bridge *bridge = softmix_data->bridge;
1054
1055         ast_bridge_lock(bridge);
1056         if (bridge->callid) {
1057                 ast_callid_threadassoc_add(bridge->callid);
1058         }
1059
1060         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1061
1062         while (!softmix_data->stop) {
1063                 if (!bridge->num_active) {
1064                         /* Wait for something to happen to the bridge. */
1065                         ast_bridge_unlock(bridge);
1066                         ast_mutex_lock(&softmix_data->lock);
1067                         if (!softmix_data->stop) {
1068                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1069                         }
1070                         ast_mutex_unlock(&softmix_data->lock);
1071                         ast_bridge_lock(bridge);
1072                         continue;
1073                 }
1074
1075                 if (softmix_mixing_loop(bridge)) {
1076                         /*
1077                          * A mixing error occurred.  Sleep and try again later so we
1078                          * won't flood the logs.
1079                          */
1080                         ast_bridge_unlock(bridge);
1081                         sleep(1);
1082                         ast_bridge_lock(bridge);
1083                 }
1084         }
1085
1086         ast_bridge_unlock(bridge);
1087
1088         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1089
1090         return NULL;
1091 }
1092
1093 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1094 {
1095         if (softmix_data->timer) {
1096                 ast_timer_close(softmix_data->timer);
1097                 softmix_data->timer = NULL;
1098         }
1099         ast_mutex_destroy(&softmix_data->lock);
1100         ast_cond_destroy(&softmix_data->cond);
1101         ast_free(softmix_data);
1102 }
1103
1104 /*! \brief Function called when a bridge is created */
1105 static int softmix_bridge_create(struct ast_bridge *bridge)
1106 {
1107         struct softmix_bridge_data *softmix_data;
1108
1109         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1110         if (!softmix_data) {
1111                 return -1;
1112         }
1113         softmix_data->bridge = bridge;
1114         ast_mutex_init(&softmix_data->lock);
1115         ast_cond_init(&softmix_data->cond, NULL);
1116         softmix_data->timer = ast_timer_open();
1117         if (!softmix_data->timer) {
1118                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1119                 softmix_bridge_data_destroy(softmix_data);
1120                 return -1;
1121         }
1122         /* start at 8khz, let it grow from there */
1123         softmix_data->internal_rate = 8000;
1124         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1125
1126         bridge->tech_pvt = softmix_data;
1127
1128         /* Start the mixing thread. */
1129         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1130                 softmix_data)) {
1131                 softmix_data->thread = AST_PTHREADT_NULL;
1132                 softmix_bridge_data_destroy(softmix_data);
1133                 bridge->tech_pvt = NULL;
1134                 return -1;
1135         }
1136
1137         return 0;
1138 }
1139
1140 /*!
1141  * \internal
1142  * \brief Request the softmix mixing thread stop.
1143  * \since 12.0.0
1144  *
1145  * \param bridge Which bridge is being stopped.
1146  *
1147  * \return Nothing
1148  */
1149 static void softmix_bridge_stop(struct ast_bridge *bridge)
1150 {
1151         struct softmix_bridge_data *softmix_data;
1152
1153         softmix_data = bridge->tech_pvt;
1154         if (!softmix_data) {
1155                 return;
1156         }
1157
1158         ast_mutex_lock(&softmix_data->lock);
1159         softmix_data->stop = 1;
1160         ast_mutex_unlock(&softmix_data->lock);
1161 }
1162
1163 /*! \brief Function called when a bridge is destroyed */
1164 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1165 {
1166         struct softmix_bridge_data *softmix_data;
1167         pthread_t thread;
1168
1169         softmix_data = bridge->tech_pvt;
1170         if (!softmix_data) {
1171                 return;
1172         }
1173
1174         /* Stop the mixing thread. */
1175         ast_mutex_lock(&softmix_data->lock);
1176         softmix_data->stop = 1;
1177         ast_cond_signal(&softmix_data->cond);
1178         thread = softmix_data->thread;
1179         softmix_data->thread = AST_PTHREADT_NULL;
1180         ast_mutex_unlock(&softmix_data->lock);
1181         if (thread != AST_PTHREADT_NULL) {
1182                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1183                 pthread_join(thread, NULL);
1184         }
1185
1186         softmix_bridge_data_destroy(softmix_data);
1187         bridge->tech_pvt = NULL;
1188 }
1189
1190 static struct ast_bridge_technology softmix_bridge = {
1191         .name = "softmix",
1192         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1193         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1194         .create = softmix_bridge_create,
1195         .stop = softmix_bridge_stop,
1196         .destroy = softmix_bridge_destroy,
1197         .join = softmix_bridge_join,
1198         .leave = softmix_bridge_leave,
1199         .unsuspend = softmix_bridge_unsuspend,
1200         .write = softmix_bridge_write,
1201 };
1202
1203 static int unload_module(void)
1204 {
1205         ast_bridge_technology_unregister(&softmix_bridge);
1206         return 0;
1207 }
1208
1209 static int load_module(void)
1210 {
1211         if (ast_bridge_technology_register(&softmix_bridge)) {
1212                 unload_module();
1213                 return AST_MODULE_LOAD_DECLINE;
1214         }
1215         return AST_MODULE_LOAD_SUCCESS;
1216 }
1217
1218 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");