media formats: re-architect handling of media for performance improvements
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 /*** MODULEINFO
31         <support_level>core</support_level>
32  ***/
33
34 #include "asterisk.h"
35
36 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
37
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <sys/time.h>
42 #include <signal.h>
43 #include <errno.h>
44 #include <unistd.h>
45
46 #include "asterisk/module.h"
47 #include "asterisk/channel.h"
48 #include "asterisk/bridge.h"
49 #include "asterisk/bridge_technology.h"
50 #include "asterisk/frame.h"
51 #include "asterisk/options.h"
52 #include "asterisk/logger.h"
53 #include "asterisk/slinfactory.h"
54 #include "asterisk/astobj2.h"
55 #include "asterisk/timing.h"
56 #include "asterisk/translate.h"
57
58 #define MAX_DATALEN 8096
59
60 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
61 #define DEFAULT_SOFTMIX_INTERVAL 20
62
63 /*! \brief Size of the buffer used for sample manipulation */
64 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
65
66 /*! \brief Number of samples we are dealing with */
67 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
68
69 /*! \brief Number of mixing iterations to perform between gathering statistics. */
70 #define SOFTMIX_STAT_INTERVAL 100
71
72 /* This is the threshold in ms at which a channel's own audio will stop getting
73  * mixed out its own write audio stream because it is not talking. */
74 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
75 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
76
77 #define DEFAULT_ENERGY_HISTORY_LEN 150
78
79 struct video_follow_talker_data {
80         /*! audio energy history */
81         int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
82         /*! The current slot being used in the history buffer, this
83          *  increments and wraps around */
84         int energy_history_cur_slot;
85         /*! The current energy sum used for averages. */
86         int energy_accum;
87         /*! The current energy average */
88         int energy_average;
89 };
90
91 /*! \brief Structure which contains per-channel mixing information */
92 struct softmix_channel {
93         /*! Lock to protect this structure */
94         ast_mutex_t lock;
95         /*! Factory which contains audio read in from the channel */
96         struct ast_slinfactory factory;
97         /*! Frame that contains mixed audio to be written out to the channel */
98         struct ast_frame write_frame;
99         /*! Frame that contains mixed audio read from the channel */
100         struct ast_frame read_frame;
101         /*! DSP for detecting silence */
102         struct ast_dsp *dsp;
103         /*!
104          * \brief TRUE if a channel is talking.
105          *
106          * \note This affects how the channel's audio is mixed back to
107          * it.
108          */
109         unsigned int talking:1;
110         /*! TRUE if the channel provided audio for this mixing interval */
111         unsigned int have_audio:1;
112         /*! Buffer containing final mixed audio from all sources */
113         short final_buf[MAX_DATALEN];
114         /*! Buffer containing only the audio from the channel */
115         short our_buf[MAX_DATALEN];
116         /*! Data pertaining to talker mode for video conferencing */
117         struct video_follow_talker_data video_talker;
118 };
119
120 struct softmix_bridge_data {
121         struct ast_timer *timer;
122         /*!
123          * \brief Bridge pointer passed to the softmix mixing thread.
124          *
125          * \note Does not need a reference because the bridge will
126          * always exist while the mixing thread exists even if the
127          * bridge is no longer actively using the softmix technology.
128          */
129         struct ast_bridge *bridge;
130         /*! Lock for signaling the mixing thread. */
131         ast_mutex_t lock;
132         /*! Condition, used if we need to wake up the mixing thread. */
133         ast_cond_t cond;
134         /*! Thread handling the mixing */
135         pthread_t thread;
136         unsigned int internal_rate;
137         unsigned int internal_mixing_interval;
138         /*! TRUE if the mixing thread should stop */
139         unsigned int stop:1;
140 };
141
142 struct softmix_stats {
143         /*! Each index represents a sample rate used above the internal rate. */
144         unsigned int sample_rates[16];
145         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
146         unsigned int num_channels[16];
147         /*! the number of channels above the internal sample rate */
148         unsigned int num_above_internal_rate;
149         /*! the number of channels at the internal sample rate */
150         unsigned int num_at_internal_rate;
151         /*! the absolute highest sample rate supported by any channel in the bridge */
152         unsigned int highest_supported_rate;
153         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
154         unsigned int locked_rate;
155 };
156
157 struct softmix_mixing_array {
158         unsigned int max_num_entries;
159         unsigned int used_entries;
160         int16_t **buffers;
161 };
162
163 struct softmix_translate_helper_entry {
164         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
165                                       and re-init if it was usable. */
166         struct ast_format *dst_format; /*!< The destination format for this helper */
167         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
168         struct ast_frame *out_frame; /*!< The output frame from the last translation */
169         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
170 };
171
172 struct softmix_translate_helper {
173         struct ast_format *slin_src; /*!< the source format expected for all the translators */
174         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
175 };
176
177 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
178 {
179         struct softmix_translate_helper_entry *entry;
180         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
181                 return NULL;
182         }
183         entry->dst_format = ao2_bump(dst);
184         return entry;
185 }
186
187 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
188 {
189         ao2_cleanup(entry->dst_format);
190
191         if (entry->trans_pvt) {
192                 ast_translator_free_path(entry->trans_pvt);
193         }
194         if (entry->out_frame) {
195                 ast_frfree(entry->out_frame);
196         }
197         ast_free(entry);
198         return NULL;
199 }
200
201 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
202 {
203         memset(trans_helper, 0, sizeof(*trans_helper));
204         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
205 }
206
207 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
208 {
209         struct softmix_translate_helper_entry *entry;
210
211         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
212                 softmix_translate_helper_free_entry(entry);
213         }
214 }
215
216 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
217 {
218         struct softmix_translate_helper_entry *entry;
219
220         trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
221         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
222                 if (entry->trans_pvt) {
223                         ast_translator_free_path(entry->trans_pvt);
224                         if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
225                                 AST_LIST_REMOVE_CURRENT(entry);
226                                 entry = softmix_translate_helper_free_entry(entry);
227                         }
228                 }
229         }
230         AST_LIST_TRAVERSE_SAFE_END;
231 }
232
233 /*!
234  * \internal
235  * \brief Get the next available audio on the softmix channel's read stream
236  * and determine if it should be mixed out or not on the write stream.
237  *
238  * \retval pointer to buffer containing the exact number of samples requested on success.
239  * \retval NULL if no samples are present
240  */
241 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
242 {
243         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
244                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
245                 sc->have_audio = 1;
246                 return sc->our_buf;
247         }
248         sc->have_audio = 0;
249         return NULL;
250 }
251
252 /*!
253  * \internal
254  * \brief Process a softmix channel's write audio
255  *
256  * \details This function will remove the channel's talking from its own audio if present and
257  * possibly even do the channel's write translation for it depending on how many other
258  * channels use the same write format.
259  */
260 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
261         struct ast_format *raw_write_fmt,
262         struct softmix_channel *sc)
263 {
264         struct softmix_translate_helper_entry *entry = NULL;
265         int i;
266
267         /* If we provided audio that was not determined to be silence,
268          * then take it out while in slinear format. */
269         if (sc->have_audio && sc->talking) {
270                 for (i = 0; i < sc->write_frame.samples; i++) {
271                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
272                 }
273                 /* do not do any special write translate optimization if we had to make
274                  * a special mix for them to remove their own audio. */
275                 return;
276         }
277
278         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
279                 if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
280                         entry->num_times_requested++;
281                 } else {
282                         continue;
283                 }
284                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
285                         entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
286                 }
287                 if (entry->trans_pvt && !entry->out_frame) {
288                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
289                 }
290                 if (entry->out_frame && (entry->out_frame->datalen < MAX_DATALEN)) {
291                         ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
292                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
293                         sc->write_frame.datalen = entry->out_frame->datalen;
294                         sc->write_frame.samples = entry->out_frame->samples;
295                 }
296                 break;
297         }
298
299         /* add new entry into list if this format destination was not matched. */
300         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
301                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
302         }
303 }
304
305 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
306 {
307         struct softmix_translate_helper_entry *entry;
308
309         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
310                 if (entry->out_frame) {
311                         ast_frfree(entry->out_frame);
312                         entry->out_frame = NULL;
313                 }
314                 entry->num_times_requested = 0;
315         }
316 }
317
318 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset)
319 {
320         struct softmix_channel *sc = bridge_channel->tech_pvt;
321         unsigned int channel_read_rate = ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan));
322
323         ast_mutex_lock(&sc->lock);
324         if (reset) {
325                 ast_slinfactory_destroy(&sc->factory);
326                 ast_dsp_free(sc->dsp);
327         }
328
329         /* Setup write frame parameters */
330         sc->write_frame.frametype = AST_FRAME_VOICE;
331         ao2_cleanup(sc->write_frame.subclass.format);
332         /*
333          * NOTE: The format is bumped here because translation could
334          * be needed and the format changed to the translated format
335          * for the channel.  The translated format may not be a
336          * static cached format.
337          */
338         sc->write_frame.subclass.format = ao2_bump(ast_format_cache_get_slin_by_rate(rate));
339         sc->write_frame.data.ptr = sc->final_buf;
340         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
341         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
342
343         /* Setup read frame parameters */
344         sc->read_frame.frametype = AST_FRAME_VOICE;
345         /*
346          * NOTE: The format is not bumbed here because it will always
347          * be a signed linear format.
348          */
349         sc->read_frame.subclass.format = ast_format_cache_get_slin_by_rate(channel_read_rate);
350         sc->read_frame.data.ptr = sc->our_buf;
351         sc->read_frame.datalen = SOFTMIX_DATALEN(channel_read_rate, interval);
352         sc->read_frame.samples = SOFTMIX_SAMPLES(channel_read_rate, interval);
353
354         /* Setup smoother */
355         ast_slinfactory_init_with_format(&sc->factory, sc->write_frame.subclass.format);
356
357         /* set new read and write formats on channel. */
358         ast_set_read_format(bridge_channel->chan, sc->read_frame.subclass.format);
359         ast_set_write_format(bridge_channel->chan, sc->write_frame.subclass.format);
360
361         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
362         sc->dsp = ast_dsp_new_with_rate(channel_read_rate);
363         /* we want to aggressively detect silence to avoid feedback */
364         if (bridge_channel->tech_args.talking_threshold) {
365                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
366         } else {
367                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
368         }
369
370         ast_mutex_unlock(&sc->lock);
371 }
372
373 /*!
374  * \internal
375  * \brief Poke the mixing thread in case it is waiting for an active channel.
376  * \since 12.0.0
377  *
378  * \param softmix_data Bridge mixing data.
379  *
380  * \return Nothing
381  */
382 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
383 {
384         ast_mutex_lock(&softmix_data->lock);
385         ast_cond_signal(&softmix_data->cond);
386         ast_mutex_unlock(&softmix_data->lock);
387 }
388
389 /*! \brief Function called when a channel is unsuspended from the bridge */
390 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
391 {
392         if (bridge->tech_pvt) {
393                 softmix_poke_thread(bridge->tech_pvt);
394         }
395 }
396
397 /*!
398  * \internal
399  * \brief Indicate a source change to the channel.
400  * \since 12.0.0
401  *
402  * \param bridge_channel Which channel source is changing.
403  *
404  * \retval 0 on success.
405  * \retval -1 on error.
406  */
407 static int softmix_src_change(struct ast_bridge_channel *bridge_channel)
408 {
409         return ast_bridge_channel_queue_control_data(bridge_channel, AST_CONTROL_SRCCHANGE, NULL, 0);
410 }
411
412 /*! \brief Function called when a channel is joined into the bridge */
413 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
414 {
415         struct softmix_channel *sc;
416         struct softmix_bridge_data *softmix_data;
417
418         softmix_data = bridge->tech_pvt;
419         if (!softmix_data) {
420                 return -1;
421         }
422
423         /* Create a new softmix_channel structure and allocate various things on it */
424         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
425                 return -1;
426         }
427
428         softmix_src_change(bridge_channel);
429
430         /* Can't forget the lock */
431         ast_mutex_init(&sc->lock);
432
433         /* Can't forget to record our pvt structure within the bridged channel structure */
434         bridge_channel->tech_pvt = sc;
435
436         set_softmix_bridge_data(softmix_data->internal_rate,
437                 softmix_data->internal_mixing_interval
438                         ? softmix_data->internal_mixing_interval
439                         : DEFAULT_SOFTMIX_INTERVAL,
440                 bridge_channel, 0);
441
442         softmix_poke_thread(softmix_data);
443         return 0;
444 }
445
446 /*! \brief Function called when a channel leaves the bridge */
447 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
448 {
449         struct softmix_channel *sc = bridge_channel->tech_pvt;
450
451         if (!sc) {
452                 return;
453         }
454         bridge_channel->tech_pvt = NULL;
455
456         softmix_src_change(bridge_channel);
457
458         /* Drop mutex lock */
459         ast_mutex_destroy(&sc->lock);
460
461         /* Drop the factory */
462         ast_slinfactory_destroy(&sc->factory);
463
464         /* Drop any formats on the frames */
465         ao2_cleanup(sc->write_frame.subclass.format);
466
467         /* Drop the DSP */
468         ast_dsp_free(sc->dsp);
469
470         /* Eep! drop ourselves */
471         ast_free(sc);
472 }
473
474 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
475 {
476         struct ast_bridge_channel *cur;
477
478         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
479                 if (cur->suspended) {
480                         continue;
481                 }
482                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
483                         ast_bridge_channel_queue_frame(cur, frame);
484                         break;
485                 }
486         }
487 }
488
489 /*!
490  * \internal
491  * \brief Determine what to do with a video frame.
492  * \since 12.0.0
493  *
494  * \param bridge Which bridge is getting the frame
495  * \param bridge_channel Which channel is writing the frame.
496  * \param frame What is being written.
497  *
498  * \return Nothing
499  */
500 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
501 {
502         struct softmix_channel *sc;
503         int video_src_priority;
504
505         /* Determine if the video frame should be distributed or not */
506         switch (bridge->softmix.video_mode.mode) {
507         case AST_BRIDGE_VIDEO_MODE_NONE:
508                 break;
509         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
510                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
511                 if (video_src_priority == 1) {
512                         /* Pass to me and everyone else. */
513                         ast_bridge_queue_everyone_else(bridge, NULL, frame);
514                 }
515                 break;
516         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
517                 sc = bridge_channel->tech_pvt;
518                 ast_mutex_lock(&sc->lock);
519                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
520                         sc->video_talker.energy_average,
521                         frame->subclass.frame_ending);
522                 ast_mutex_unlock(&sc->lock);
523                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
524                 if (video_src_priority == 1) {
525                         int num_src = ast_bridge_number_video_src(bridge);
526                         int echo = num_src > 1 ? 0 : 1;
527
528                         ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
529                 } else if (video_src_priority == 2) {
530                         softmix_pass_video_top_priority(bridge, frame);
531                 }
532                 break;
533         }
534 }
535
536 /*!
537  * \internal
538  * \brief Determine what to do with a voice frame.
539  * \since 12.0.0
540  *
541  * \param bridge Which bridge is getting the frame
542  * \param bridge_channel Which channel is writing the frame.
543  * \param frame What is being written.
544  *
545  * \return Nothing
546  */
547 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
548 {
549         struct softmix_channel *sc = bridge_channel->tech_pvt;
550         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
551         int totalsilence = 0;
552         int cur_energy = 0;
553         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
554                 bridge_channel->tech_args.silence_threshold :
555                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
556         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
557
558         /* Write the frame into the conference */
559         ast_mutex_lock(&sc->lock);
560         ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
561
562         if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
563                 int cur_slot = sc->video_talker.energy_history_cur_slot;
564
565                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
566                 sc->video_talker.energy_accum += cur_energy;
567                 sc->video_talker.energy_history[cur_slot] = cur_energy;
568                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
569                 sc->video_talker.energy_history_cur_slot++;
570                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
571                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
572                 }
573         }
574
575         if (totalsilence < silence_threshold) {
576                 if (!sc->talking) {
577                         update_talking = 1;
578                 }
579                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
580         } else {
581                 if (sc->talking) {
582                         update_talking = 0;
583                 }
584                 sc->talking = 0;
585         }
586
587         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
588          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
589          * the audio by flushing the buffer before adding new audio in. */
590         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
591                 ast_slinfactory_flush(&sc->factory);
592         }
593
594         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
595          * is not determined to be talking. */
596         if (!(bridge_channel->tech_args.drop_silence && !sc->talking)) {
597                 ast_slinfactory_feed(&sc->factory, frame);
598         }
599
600         /* Alllll done */
601         ast_mutex_unlock(&sc->lock);
602
603         if (update_talking != -1) {
604                 ast_bridge_channel_notify_talking(bridge_channel, update_talking);
605         }
606 }
607
608 /*!
609  * \internal
610  * \brief Determine what to do with a control frame.
611  * \since 12.0.0
612  *
613  * \param bridge Which bridge is getting the frame
614  * \param bridge_channel Which channel is writing the frame.
615  * \param frame What is being written.
616  *
617  * \retval 0 Frame accepted into the bridge.
618  * \retval -1 Frame needs to be deferred.
619  */
620 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
621 {
622         /*
623          * XXX Softmix needs to use channel roles to determine what to
624          * do with control frames.
625          */
626         return 0;
627 }
628
629 /*!
630  * \internal
631  * \brief Determine what to do with a frame written into the bridge.
632  * \since 12.0.0
633  *
634  * \param bridge Which bridge is getting the frame
635  * \param bridge_channel Which channel is writing the frame.
636  * \param frame What is being written.
637  *
638  * \retval 0 Frame accepted into the bridge.
639  * \retval -1 Frame needs to be deferred.
640  *
641  * \note On entry, bridge is already locked.
642  */
643 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
644 {
645         int res = 0;
646
647         if (!bridge->tech_pvt || (bridge_channel && !bridge_channel->tech_pvt)) {
648                 /* "Accept" the frame and discard it. */
649                 return 0;
650         }
651
652         /*
653          * XXX Softmix needs to use channel roles to determine who gets
654          * what frame.  Possible roles: announcer, recorder, agent,
655          * supervisor.
656          */
657         switch (frame->frametype) {
658         case AST_FRAME_NULL:
659                 /* "Accept" the frame and discard it. */
660                 break;
661         case AST_FRAME_DTMF_BEGIN:
662         case AST_FRAME_DTMF_END:
663                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
664                 break;
665         case AST_FRAME_VOICE:
666                 if (bridge_channel) {
667                         softmix_bridge_write_voice(bridge, bridge_channel, frame);
668                 }
669                 break;
670         case AST_FRAME_VIDEO:
671                 if (bridge_channel) {
672                         softmix_bridge_write_video(bridge, bridge_channel, frame);
673                 }
674                 break;
675         case AST_FRAME_CONTROL:
676                 res = softmix_bridge_write_control(bridge, bridge_channel, frame);
677                 break;
678         case AST_FRAME_BRIDGE_ACTION:
679                 res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
680                 break;
681         case AST_FRAME_BRIDGE_ACTION_SYNC:
682                 ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
683                 ast_assert(0);
684         default:
685                 ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
686                 /* "Accept" the frame and discard it. */
687                 break;
688         }
689
690         return res;
691 }
692
693 static void gather_softmix_stats(struct softmix_stats *stats,
694         const struct softmix_bridge_data *softmix_data,
695         struct ast_bridge_channel *bridge_channel)
696 {
697         int channel_native_rate;
698         int i;
699         /* Gather stats about channel sample rates. */
700         channel_native_rate = MAX(ast_format_get_sample_rate(ast_channel_rawwriteformat(bridge_channel->chan)),
701                 ast_format_get_sample_rate(ast_channel_rawreadformat(bridge_channel->chan)));
702
703         if (channel_native_rate > stats->highest_supported_rate) {
704                 stats->highest_supported_rate = channel_native_rate;
705         }
706         if (channel_native_rate > softmix_data->internal_rate) {
707                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
708                         if (stats->sample_rates[i] == channel_native_rate) {
709                                 stats->num_channels[i]++;
710                                 break;
711                         } else if (!stats->sample_rates[i]) {
712                                 stats->sample_rates[i] = channel_native_rate;
713                                 stats->num_channels[i]++;
714                                 break;
715                         }
716                 }
717                 stats->num_above_internal_rate++;
718         } else if (channel_native_rate == softmix_data->internal_rate) {
719                 stats->num_at_internal_rate++;
720         }
721 }
722 /*!
723  * \internal
724  * \brief Analyse mixing statistics and change bridges internal rate
725  * if necessary.
726  *
727  * \retval 0, no changes to internal rate
728  * \ratval 1, internal rate was changed, update all the channels on the next mixing iteration.
729  */
730 static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data)
731 {
732         int i;
733         /* Re-adjust the internal bridge sample rate if
734          * 1. The bridge's internal sample rate is locked in at a sample
735          *    rate other than the current sample rate being used.
736          * 2. two or more channels support a higher sample rate
737          * 3. no channels support the current sample rate or a higher rate
738          */
739         if (stats->locked_rate) {
740                 /* if the rate is locked by the bridge, only update it if it differs
741                  * from the current rate we are using. */
742                 if (softmix_data->internal_rate != stats->locked_rate) {
743                         softmix_data->internal_rate = stats->locked_rate;
744                         ast_debug(1, "Bridge is locked in at sample rate %u\n",
745                                 softmix_data->internal_rate);
746                         return 1;
747                 }
748         } else if (stats->num_above_internal_rate >= 2) {
749                 /* the highest rate is just used as a starting point */
750                 unsigned int best_rate = stats->highest_supported_rate;
751                 int best_index = -1;
752
753                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
754                         if (stats->num_channels[i]) {
755                                 break;
756                         }
757                         /* best_rate starts out being the first sample rate
758                          * greater than the internal sample rate that 2 or
759                          * more channels support. */
760                         if (stats->num_channels[i] >= 2 && (best_index == -1)) {
761                                 best_rate = stats->sample_rates[i];
762                                 best_index = i;
763                         /* If it has been detected that multiple rates above
764                          * the internal rate are present, compare those rates
765                          * to each other and pick the highest one two or more
766                          * channels support. */
767                         } else if (((best_index != -1) &&
768                                 (stats->num_channels[i] >= 2) &&
769                                 (stats->sample_rates[best_index] < stats->sample_rates[i]))) {
770                                 best_rate = stats->sample_rates[i];
771                                 best_index = i;
772                         /* It is possible that multiple channels exist with native sample
773                          * rates above the internal sample rate, but none of those channels
774                          * have the same rate in common.  In this case, the lowest sample
775                          * rate among those channels is picked. Over time as additional
776                          * statistic runs are made the internal sample rate number will
777                          * adjust to the most optimal sample rate, but it may take multiple
778                          * iterations. */
779                         } else if (best_index == -1) {
780                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
781                         }
782                 }
783
784                 ast_debug(1, "Bridge changed from %u To %u\n",
785                         softmix_data->internal_rate, best_rate);
786                 softmix_data->internal_rate = best_rate;
787                 return 1;
788         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
789                 /* In this case, the highest supported rate is actually lower than the internal rate */
790                 softmix_data->internal_rate = stats->highest_supported_rate;
791                 ast_debug(1, "Bridge changed from %u to %u\n",
792                         softmix_data->internal_rate, stats->highest_supported_rate);
793                 return 1;
794         }
795         return 0;
796 }
797
798 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries)
799 {
800         memset(mixing_array, 0, sizeof(*mixing_array));
801         mixing_array->max_num_entries = starting_num_entries;
802         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
803                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
804                 return -1;
805         }
806         return 0;
807 }
808
809 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array)
810 {
811         ast_free(mixing_array->buffers);
812 }
813
814 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries)
815 {
816         int16_t **tmp;
817         /* give it some room to grow since memory is cheap but allocations can be expensive */
818         mixing_array->max_num_entries = num_entries;
819         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
820                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
821                 return -1;
822         }
823         mixing_array->buffers = tmp;
824         return 0;
825 }
826
827 /*!
828  * \brief Mixing loop.
829  *
830  * \retval 0 on success
831  * \retval -1 on failure
832  */
833 static int softmix_mixing_loop(struct ast_bridge *bridge)
834 {
835         struct softmix_stats stats = { { 0 }, };
836         struct softmix_mixing_array mixing_array;
837         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
838         struct ast_timer *timer;
839         struct softmix_translate_helper trans_helper;
840         int16_t buf[MAX_DATALEN];
841         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
842         int timingfd;
843         int update_all_rates = 0; /* set this when the internal sample rate has changed */
844         unsigned int idx;
845         unsigned int x;
846         int res = -1;
847
848         timer = softmix_data->timer;
849         timingfd = ast_timer_fd(timer);
850         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
851         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
852
853         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
854         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10)) {
855                 goto softmix_cleanup;
856         }
857
858         /*
859          * XXX Softmix needs to use channel roles to determine who gets
860          * what audio mixed.
861          */
862         while (!softmix_data->stop && bridge->num_active) {
863                 struct ast_bridge_channel *bridge_channel;
864                 int timeout = -1;
865                 struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
866                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
867                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
868
869                 if (softmix_datalen > MAX_DATALEN) {
870                         /* This should NEVER happen, but if it does we need to know about it. Almost
871                          * all the memcpys used during this process depend on this assumption.  Rather
872                          * than checking this over and over again through out the code, this single
873                          * verification is done on each iteration. */
874                         ast_log(LOG_WARNING,
875                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
876                                 bridge->uniqueid);
877                         goto softmix_cleanup;
878                 }
879
880                 /* Grow the mixing array buffer as participants are added. */
881                 if (mixing_array.max_num_entries < bridge->num_channels
882                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5)) {
883                         goto softmix_cleanup;
884                 }
885
886                 /* init the number of buffers stored in the mixing array to 0.
887                  * As buffers are added for mixing, this number is incremented. */
888                 mixing_array.used_entries = 0;
889
890                 /* These variables help determine if a rate change is required */
891                 if (!stat_iteration_counter) {
892                         memset(&stats, 0, sizeof(stats));
893                         stats.locked_rate = bridge->softmix.internal_sample_rate;
894                 }
895
896                 /* If the sample rate has changed, update the translator helper */
897                 if (update_all_rates) {
898                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
899                 }
900
901                 /* Go through pulling audio from each factory that has it available */
902                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
903                         struct softmix_channel *sc = bridge_channel->tech_pvt;
904
905                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
906                         if (update_all_rates) {
907                                 set_softmix_bridge_data(softmix_data->internal_rate, softmix_data->internal_mixing_interval, bridge_channel, 1);
908                         }
909
910                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
911                         if (!stat_iteration_counter) {
912                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
913                         }
914
915                         /* if the channel is suspended, don't check for audio, but still gather stats */
916                         if (bridge_channel->suspended) {
917                                 continue;
918                         }
919
920                         /* Try to get audio from the factory if available */
921                         ast_mutex_lock(&sc->lock);
922                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
923                                 mixing_array.used_entries++;
924                         }
925                         ast_mutex_unlock(&sc->lock);
926                 }
927
928                 /* mix it like crazy */
929                 memset(buf, 0, softmix_datalen);
930                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
931                         for (x = 0; x < softmix_samples; ++x) {
932                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
933                         }
934                 }
935
936                 /* Next step go through removing the channel's own audio and creating a good frame... */
937                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
938                         struct softmix_channel *sc = bridge_channel->tech_pvt;
939
940                         if (bridge_channel->suspended) {
941                                 continue;
942                         }
943
944                         ast_mutex_lock(&sc->lock);
945
946                         /* Make SLINEAR write frame from local buffer */
947                         ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
948                                 "Replace softmix channel slin format");
949                         sc->write_frame.datalen = softmix_datalen;
950                         sc->write_frame.samples = softmix_samples;
951                         memcpy(sc->final_buf, buf, softmix_datalen);
952
953                         /* process the softmix channel's new write audio */
954                         softmix_process_write_audio(&trans_helper, ast_channel_rawwriteformat(bridge_channel->chan), sc);
955
956                         ast_mutex_unlock(&sc->lock);
957
958                         /* A frame is now ready for the channel. */
959                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
960                 }
961
962                 update_all_rates = 0;
963                 if (!stat_iteration_counter) {
964                         update_all_rates = analyse_softmix_stats(&stats, softmix_data);
965                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
966                 }
967                 stat_iteration_counter--;
968
969                 ast_bridge_unlock(bridge);
970                 /* cleanup any translation frame data from the previous mixing iteration. */
971                 softmix_translate_helper_cleanup(&trans_helper);
972                 /* Wait for the timing source to tell us to wake up and get things done */
973                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
974                 if (ast_timer_ack(timer, 1) < 0) {
975                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
976                                 bridge->uniqueid);
977                         ast_bridge_lock(bridge);
978                         goto softmix_cleanup;
979                 }
980                 ast_bridge_lock(bridge);
981
982                 /* make sure to detect mixing interval changes if they occur. */
983                 if (bridge->softmix.internal_mixing_interval
984                         && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
985                         softmix_data->internal_mixing_interval = bridge->softmix.internal_mixing_interval;
986                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
987                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
988                 }
989         }
990
991         res = 0;
992
993 softmix_cleanup:
994         softmix_translate_helper_destroy(&trans_helper);
995         softmix_mixing_array_destroy(&mixing_array);
996         return res;
997 }
998
999 /*!
1000  * \internal
1001  * \brief Mixing thread.
1002  * \since 12.0.0
1003  *
1004  * \note The thread does not have its own reference to the
1005  * bridge.  The lifetime of the thread is tied to the lifetime
1006  * of the mixing technology association with the bridge.
1007  */
1008 static void *softmix_mixing_thread(void *data)
1009 {
1010         struct softmix_bridge_data *softmix_data = data;
1011         struct ast_bridge *bridge = softmix_data->bridge;
1012
1013         ast_bridge_lock(bridge);
1014         if (bridge->callid) {
1015                 ast_callid_threadassoc_add(bridge->callid);
1016         }
1017
1018         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
1019
1020         while (!softmix_data->stop) {
1021                 if (!bridge->num_active) {
1022                         /* Wait for something to happen to the bridge. */
1023                         ast_bridge_unlock(bridge);
1024                         ast_mutex_lock(&softmix_data->lock);
1025                         if (!softmix_data->stop) {
1026                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1027                         }
1028                         ast_mutex_unlock(&softmix_data->lock);
1029                         ast_bridge_lock(bridge);
1030                         continue;
1031                 }
1032
1033                 if (softmix_mixing_loop(bridge)) {
1034                         /*
1035                          * A mixing error occurred.  Sleep and try again later so we
1036                          * won't flood the logs.
1037                          */
1038                         ast_bridge_unlock(bridge);
1039                         sleep(1);
1040                         ast_bridge_lock(bridge);
1041                 }
1042         }
1043
1044         ast_bridge_unlock(bridge);
1045
1046         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1047
1048         return NULL;
1049 }
1050
1051 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1052 {
1053         if (softmix_data->timer) {
1054                 ast_timer_close(softmix_data->timer);
1055                 softmix_data->timer = NULL;
1056         }
1057         ast_mutex_destroy(&softmix_data->lock);
1058         ast_cond_destroy(&softmix_data->cond);
1059         ast_free(softmix_data);
1060 }
1061
1062 /*! \brief Function called when a bridge is created */
1063 static int softmix_bridge_create(struct ast_bridge *bridge)
1064 {
1065         struct softmix_bridge_data *softmix_data;
1066
1067         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1068         if (!softmix_data) {
1069                 return -1;
1070         }
1071         softmix_data->bridge = bridge;
1072         ast_mutex_init(&softmix_data->lock);
1073         ast_cond_init(&softmix_data->cond, NULL);
1074         softmix_data->timer = ast_timer_open();
1075         if (!softmix_data->timer) {
1076                 ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
1077                 softmix_bridge_data_destroy(softmix_data);
1078                 return -1;
1079         }
1080         /* start at 8khz, let it grow from there */
1081         softmix_data->internal_rate = 8000;
1082         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1083
1084         bridge->tech_pvt = softmix_data;
1085
1086         /* Start the mixing thread. */
1087         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
1088                 softmix_data)) {
1089                 softmix_data->thread = AST_PTHREADT_NULL;
1090                 softmix_bridge_data_destroy(softmix_data);
1091                 bridge->tech_pvt = NULL;
1092                 return -1;
1093         }
1094
1095         return 0;
1096 }
1097
1098 /*!
1099  * \internal
1100  * \brief Request the softmix mixing thread stop.
1101  * \since 12.0.0
1102  *
1103  * \param bridge Which bridge is being stopped.
1104  *
1105  * \return Nothing
1106  */
1107 static void softmix_bridge_stop(struct ast_bridge *bridge)
1108 {
1109         struct softmix_bridge_data *softmix_data;
1110
1111         softmix_data = bridge->tech_pvt;
1112         if (!softmix_data) {
1113                 return;
1114         }
1115
1116         ast_mutex_lock(&softmix_data->lock);
1117         softmix_data->stop = 1;
1118         ast_mutex_unlock(&softmix_data->lock);
1119 }
1120
1121 /*! \brief Function called when a bridge is destroyed */
1122 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1123 {
1124         struct softmix_bridge_data *softmix_data;
1125         pthread_t thread;
1126
1127         softmix_data = bridge->tech_pvt;
1128         if (!softmix_data) {
1129                 return;
1130         }
1131
1132         /* Stop the mixing thread. */
1133         ast_mutex_lock(&softmix_data->lock);
1134         softmix_data->stop = 1;
1135         ast_cond_signal(&softmix_data->cond);
1136         thread = softmix_data->thread;
1137         softmix_data->thread = AST_PTHREADT_NULL;
1138         ast_mutex_unlock(&softmix_data->lock);
1139         if (thread != AST_PTHREADT_NULL) {
1140                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1141                 pthread_join(thread, NULL);
1142         }
1143
1144         softmix_bridge_data_destroy(softmix_data);
1145         bridge->tech_pvt = NULL;
1146 }
1147
1148 static struct ast_bridge_technology softmix_bridge = {
1149         .name = "softmix",
1150         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1151         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1152         .create = softmix_bridge_create,
1153         .stop = softmix_bridge_stop,
1154         .destroy = softmix_bridge_destroy,
1155         .join = softmix_bridge_join,
1156         .leave = softmix_bridge_leave,
1157         .unsuspend = softmix_bridge_unsuspend,
1158         .write = softmix_bridge_write,
1159 };
1160
1161 static int unload_module(void)
1162 {
1163         ao2_cleanup(softmix_bridge.format_capabilities);
1164         softmix_bridge.format_capabilities = NULL;
1165         return ast_bridge_technology_unregister(&softmix_bridge);
1166 }
1167
1168 static int load_module(void)
1169 {
1170         if (!(softmix_bridge.format_capabilities = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT))) {
1171                 return AST_MODULE_LOAD_DECLINE;
1172         }
1173         ast_format_cap_append(softmix_bridge.format_capabilities, ast_format_slin, 0);
1174         return ast_bridge_technology_register(&softmix_bridge);
1175 }
1176
1177 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");