Video support for ConfBridge.
[asterisk/asterisk.git] / bridges / bridge_softmix.c
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <jcolp@digium.com>
7  * David Vossel <dvossel@digium.com>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <jcolp@digium.com>
25  * \author David Vossel <dvossel@digium.com>
26  *
27  * \ingroup bridges
28  */
29
30 #include "asterisk.h"
31
32 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
33
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <string.h>
37 #include <sys/time.h>
38 #include <signal.h>
39 #include <errno.h>
40 #include <unistd.h>
41
42 #include "asterisk/module.h"
43 #include "asterisk/channel.h"
44 #include "asterisk/bridging.h"
45 #include "asterisk/bridging_technology.h"
46 #include "asterisk/frame.h"
47 #include "asterisk/options.h"
48 #include "asterisk/logger.h"
49 #include "asterisk/slinfactory.h"
50 #include "asterisk/astobj2.h"
51 #include "asterisk/timing.h"
52 #include "asterisk/translate.h"
53
54 #define MAX_DATALEN 8096
55
56 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
57 #define DEFAULT_SOFTMIX_INTERVAL 20
58
59 /*! \brief Size of the buffer used for sample manipulation */
60 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
61
62 /*! \brief Number of samples we are dealing with */
63 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
64
65 /*! \brief Number of mixing iterations to perform between gathering statistics. */
66 #define SOFTMIX_STAT_INTERVAL 100
67
68 /* This is the threshold in ms at which a channel's own audio will stop getting
69  * mixed out its own write audio stream because it is not talking. */
70 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
71 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
72
73 #define DEFAULT_ENERGY_HISTORY_LEN 150
74
75 struct video_follow_talker_data {
76         /*! audio energy history */
77         int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
78         /*! The current slot being used in the history buffer, this
79          *  increments and wraps around */
80         int energy_history_cur_slot;
81         /*! The current energy sum used for averages. */
82         int energy_accum;
83         /*! The current energy average */
84         int energy_average;
85 };
86
87 /*! \brief Structure which contains per-channel mixing information */
88 struct softmix_channel {
89         /*! Lock to protect this structure */
90         ast_mutex_t lock;
91         /*! Factory which contains audio read in from the channel */
92         struct ast_slinfactory factory;
93         /*! Frame that contains mixed audio to be written out to the channel */
94         struct ast_frame write_frame;
95         /*! Frame that contains mixed audio read from the channel */
96         struct ast_frame read_frame;
97         /*! DSP for detecting silence */
98         struct ast_dsp *dsp;
99         /*! Bit used to indicate if a channel is talking or not. This affects how
100          * the channel's audio is mixed back to it. */
101         int talking:1;
102         /*! Bit used to indicate that the channel provided audio for this mixing interval */
103         int have_audio:1;
104         /*! Bit used to indicate that a frame is available to be written out to the channel */
105         int have_frame:1;
106         /*! Buffer containing final mixed audio from all sources */
107         short final_buf[MAX_DATALEN];
108         /*! Buffer containing only the audio from the channel */
109         short our_buf[MAX_DATALEN];
110         /*! Data pertaining to talker mode for video conferencing */
111         struct video_follow_talker_data video_talker;
112 };
113
114 struct softmix_bridge_data {
115         struct ast_timer *timer;
116         unsigned int internal_rate;
117         unsigned int internal_mixing_interval;
118 };
119
120 struct softmix_stats {
121                 /*! Each index represents a sample rate used above the internal rate. */
122                 unsigned int sample_rates[16];
123                 /*! Each index represents the number of channels using the same index in the sample_rates array.  */
124                 unsigned int num_channels[16];
125                 /*! the number of channels above the internal sample rate */
126                 unsigned int num_above_internal_rate;
127                 /*! the number of channels at the internal sample rate */
128                 unsigned int num_at_internal_rate;
129                 /*! the absolute highest sample rate supported by any channel in the bridge */
130                 unsigned int highest_supported_rate;
131                 /*! Is the sample rate locked by the bridge, if so what is that rate.*/
132                 unsigned int locked_rate;
133 };
134
135 struct softmix_mixing_array {
136         int max_num_entries;
137         int used_entries;
138         int16_t **buffers;
139 };
140
141 struct softmix_translate_helper_entry {
142         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
143                                       and re-init if it was usable. */
144         struct ast_format dst_format; /*!< The destination format for this helper */
145         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
146         struct ast_frame *out_frame; /*!< The output frame from the last translation */
147         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
148 };
149
150 struct softmix_translate_helper {
151         struct ast_format slin_src; /*!< the source format expected for all the translators */
152         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
153 };
154
155 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
156 {
157         struct softmix_translate_helper_entry *entry;
158         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
159                 return NULL;
160         }
161         ast_format_copy(&entry->dst_format, dst);
162         return entry;
163 }
164
165 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
166 {
167         if (entry->trans_pvt) {
168                 ast_translator_free_path(entry->trans_pvt);
169         }
170         if (entry->out_frame) {
171                 ast_frfree(entry->out_frame);
172         }
173         ast_free(entry);
174         return NULL;
175 }
176
177 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
178 {
179         memset(trans_helper, 0, sizeof(*trans_helper));
180         ast_format_set(&trans_helper->slin_src, ast_format_slin_by_rate(sample_rate), 0);
181 }
182
183 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
184 {
185         struct softmix_translate_helper_entry *entry;
186
187         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
188                 softmix_translate_helper_free_entry(entry);
189         }
190 }
191
192 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
193 {
194         struct softmix_translate_helper_entry *entry;
195
196         ast_format_set(&trans_helper->slin_src, ast_format_slin_by_rate(sample_rate), 0);
197         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
198                 if (entry->trans_pvt) {
199                         ast_translator_free_path(entry->trans_pvt);
200                         if (!(entry->trans_pvt = ast_translator_build_path(&entry->dst_format, &trans_helper->slin_src))) {
201                                 AST_LIST_REMOVE_CURRENT(entry);
202                                 entry = softmix_translate_helper_free_entry(entry);
203                         }
204                 }
205         }
206         AST_LIST_TRAVERSE_SAFE_END;
207 }
208
209 /*!
210  * \internal
211  * \brief Get the next available audio on the softmix channel's read stream
212  * and determine if it should be mixed out or not on the write stream. 
213  *
214  * \retval pointer to buffer containing the exact number of samples requested on success.
215  * \retval NULL if no samples are present
216  */
217 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
218 {
219         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
220                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
221                 sc->have_audio = 1;
222                 return sc->our_buf;
223         }
224         sc->have_audio = 0;
225         return NULL;
226 }
227
228 /*!
229  * \internal
230  * \brief Process a softmix channel's write audio
231  *
232  * \details This function will remove the channel's talking from its own audio if present and
233  * possibly even do the channel's write translation for it depending on how many other
234  * channels use the same write format.
235  */
236 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
237         struct ast_format *raw_write_fmt,
238         struct softmix_channel *sc)
239 {
240         struct softmix_translate_helper_entry *entry = NULL;
241         int i;
242
243         /* If we provided audio that was not determined to be silence,
244          * then take it out while in slinear format. */
245         if (sc->have_audio && sc->talking) {
246                 for (i = 0; i < sc->write_frame.samples; i++) {
247                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
248                 }
249                 /* do not do any special write translate optimization if we had to make
250                  * a special mix for them to remove their own audio. */
251                 return;
252         }
253
254         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
255                 if (ast_format_cmp(&entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
256                         entry->num_times_requested++;
257                 } else {
258                         continue;
259                 }
260                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
261                         entry->trans_pvt = ast_translator_build_path(&entry->dst_format, &trans_helper->slin_src);
262                 }
263                 if (entry->trans_pvt && !entry->out_frame) {
264                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
265                 }
266                 if (entry->out_frame && (entry->out_frame->datalen < MAX_DATALEN)) {
267                         ast_format_copy(&sc->write_frame.subclass.format, &entry->out_frame->subclass.format);
268                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
269                         sc->write_frame.datalen = entry->out_frame->datalen;
270                         sc->write_frame.samples = entry->out_frame->samples;
271                 }
272                 break;
273         }
274
275         /* add new entry into list if this format destination was not matched. */
276         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
277                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
278         }
279 }
280
281 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
282 {
283         struct softmix_translate_helper_entry *entry = NULL;
284         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
285                 if (entry->out_frame) {
286                         ast_frfree(entry->out_frame);
287                         entry->out_frame = NULL;
288                 }
289                 entry->num_times_requested = 0;
290         }
291 }
292
293 static void softmix_bridge_data_destroy(void *obj)
294 {
295         struct softmix_bridge_data *softmix_data = obj;
296         ast_timer_close(softmix_data->timer);
297 }
298
299 /*! \brief Function called when a bridge is created */
300 static int softmix_bridge_create(struct ast_bridge *bridge)
301 {
302         struct softmix_bridge_data *softmix_data;
303
304         if (!(softmix_data = ao2_alloc(sizeof(*softmix_data), softmix_bridge_data_destroy))) {
305                 return -1;
306         }
307         if (!(softmix_data->timer = ast_timer_open())) {
308                 ao2_ref(softmix_data, -1);
309                 return -1;
310         }
311
312         /* start at 8khz, let it grow from there */
313         softmix_data->internal_rate = 8000;
314         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
315
316         bridge->bridge_pvt = softmix_data;
317         return 0;
318 }
319
320 /*! \brief Function called when a bridge is destroyed */
321 static int softmix_bridge_destroy(struct ast_bridge *bridge)
322 {
323         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
324         if (!bridge->bridge_pvt) {
325                 return -1;
326         }
327         ao2_ref(softmix_data, -1);
328         bridge->bridge_pvt = NULL;
329         return 0;
330 }
331
332 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset)
333 {
334         struct softmix_channel *sc = bridge_channel->bridge_pvt;
335         unsigned int channel_read_rate = ast_format_rate(&bridge_channel->chan->rawreadformat);
336
337         ast_mutex_lock(&sc->lock);
338         if (reset) {
339                 ast_slinfactory_destroy(&sc->factory);
340                 ast_dsp_free(sc->dsp);
341         }
342         /* Setup read/write frame parameters */
343         sc->write_frame.frametype = AST_FRAME_VOICE;
344         ast_format_set(&sc->write_frame.subclass.format, ast_format_slin_by_rate(rate), 0);
345         sc->write_frame.data.ptr = sc->final_buf;
346         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
347         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
348
349         sc->read_frame.frametype = AST_FRAME_VOICE;
350         ast_format_set(&sc->read_frame.subclass.format, ast_format_slin_by_rate(channel_read_rate), 0);
351         sc->read_frame.data.ptr = sc->our_buf;
352         sc->read_frame.datalen = SOFTMIX_DATALEN(channel_read_rate, interval);
353         sc->read_frame.samples = SOFTMIX_SAMPLES(channel_read_rate, interval);
354
355         /* Setup smoother */
356         ast_slinfactory_init_with_format(&sc->factory, &sc->write_frame.subclass.format);
357
358         /* set new read and write formats on channel. */
359         ast_set_read_format(bridge_channel->chan, &sc->read_frame.subclass.format);
360         ast_set_write_format(bridge_channel->chan, &sc->write_frame.subclass.format);
361
362         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
363         sc->dsp = ast_dsp_new_with_rate(channel_read_rate);
364         /* we want to aggressively detect silence to avoid feedback */
365         if (bridge_channel->tech_args.talking_threshold) {
366                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
367         } else {
368                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
369         }
370
371         ast_mutex_unlock(&sc->lock);
372 }
373
374 /*! \brief Function called when a channel is joined into the bridge */
375 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
376 {
377         struct softmix_channel *sc = NULL;
378         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
379
380         /* Create a new softmix_channel structure and allocate various things on it */
381         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
382                 return -1;
383         }
384
385         /* Can't forget the lock */
386         ast_mutex_init(&sc->lock);
387
388         /* Can't forget to record our pvt structure within the bridged channel structure */
389         bridge_channel->bridge_pvt = sc;
390
391         set_softmix_bridge_data(softmix_data->internal_rate,
392                 softmix_data->internal_mixing_interval ? softmix_data->internal_mixing_interval : DEFAULT_SOFTMIX_INTERVAL,
393                 bridge_channel, 0);
394
395         return 0;
396 }
397
398 /*! \brief Function called when a channel leaves the bridge */
399 static int softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
400 {
401         struct softmix_channel *sc = bridge_channel->bridge_pvt;
402
403         if (!(bridge_channel->bridge_pvt)) {
404                 return 0;
405         }
406         bridge_channel->bridge_pvt = NULL;
407
408         /* Drop mutex lock */
409         ast_mutex_destroy(&sc->lock);
410
411         /* Drop the factory */
412         ast_slinfactory_destroy(&sc->factory);
413
414         /* Drop the DSP */
415         ast_dsp_free(sc->dsp);
416
417         /* Eep! drop ourselves */
418         ast_free(sc);
419
420         return 0;
421 }
422
423 /*!
424  * \internal
425  * \brief If the bridging core passes DTMF to us, then they want it to be distributed out to all memebers. Do that here.
426  */
427 static void softmix_pass_dtmf(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
428 {
429         struct ast_bridge_channel *tmp;
430         AST_LIST_TRAVERSE(&bridge->channels, tmp, entry) {
431                 if (tmp == bridge_channel) {
432                         continue;
433                 }
434                 ast_write(tmp->chan, frame);
435         }
436 }
437
438 static void softmix_pass_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
439 {
440         struct ast_bridge_channel *tmp;
441         AST_LIST_TRAVERSE(&bridge->channels, tmp, entry) {
442                 if (tmp->suspended) {
443                         continue;
444                 }
445                 ast_write(tmp->chan, frame);
446         }
447 }
448
449 /*! \brief Function called when a channel writes a frame into the bridge */
450 static enum ast_bridge_write_result softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
451 {
452         struct softmix_channel *sc = bridge_channel->bridge_pvt;
453         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
454         int totalsilence = 0;
455         int cur_energy = 0;
456         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
457                 bridge_channel->tech_args.silence_threshold :
458                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
459         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
460         int res = AST_BRIDGE_WRITE_SUCCESS;
461
462         /* Only accept audio frames, all others are unsupported */
463         if (frame->frametype == AST_FRAME_DTMF_END || frame->frametype == AST_FRAME_DTMF_BEGIN) {
464                 softmix_pass_dtmf(bridge, bridge_channel, frame);
465                 goto bridge_write_cleanup;
466         } else if (frame->frametype != AST_FRAME_VOICE && frame->frametype != AST_FRAME_VIDEO) {
467                 res = AST_BRIDGE_WRITE_UNSUPPORTED;
468                 goto bridge_write_cleanup;
469         } else if (frame->datalen == 0) {
470                 goto bridge_write_cleanup;
471         }
472
473         /* Determine if this video frame should be distributed or not */
474         if (frame->frametype == AST_FRAME_VIDEO) {
475                 switch (bridge->video_mode.mode) {
476                 case AST_BRIDGE_VIDEO_MODE_NONE:
477                         break;
478                 case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
479                         if (ast_bridge_is_video_src(bridge, bridge_channel->chan)) {
480                                 softmix_pass_video(bridge, bridge_channel, frame);
481                         }
482                         break;
483                 case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
484                         ast_mutex_lock(&sc->lock);
485                         ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan, sc->video_talker.energy_average, ast_format_get_video_mark(&frame->subclass.format));
486                         ast_mutex_unlock(&sc->lock);
487                         if (ast_bridge_is_video_src(bridge, bridge_channel->chan)) {
488                                 softmix_pass_video(bridge, bridge_channel, frame);
489                         }
490                         break;
491                 }
492                 goto bridge_write_cleanup;
493         }
494
495         /* If we made it here, we are going to write the frame into the conference */
496         ast_mutex_lock(&sc->lock);
497         ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
498
499         if (bridge->video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
500                 int cur_slot = sc->video_talker.energy_history_cur_slot;
501                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
502                 sc->video_talker.energy_accum += cur_energy;
503                 sc->video_talker.energy_history[cur_slot] = cur_energy;
504                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
505                 sc->video_talker.energy_history_cur_slot++;
506                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
507                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
508                 }
509         }
510
511         if (totalsilence < silence_threshold) {
512                 if (!sc->talking) {
513                         update_talking = 1;
514                 }
515                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
516         } else {
517                 if (sc->talking) {
518                         update_talking = 0;
519                 }
520                 sc->talking = 0;
521         }
522
523         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
524          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
525          * the audio by flushing the buffer before adding new audio in. */
526         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
527                 ast_slinfactory_flush(&sc->factory);
528         }
529
530         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
531          * is not determined to be talking. */
532         if (!(bridge_channel->tech_args.drop_silence && !sc->talking) &&
533                 (frame->frametype == AST_FRAME_VOICE && ast_format_is_slinear(&frame->subclass.format))) {
534                 ast_slinfactory_feed(&sc->factory, frame);
535         }
536
537         /* If a frame is ready to be written out, do so */
538         if (sc->have_frame) {
539                 ast_write(bridge_channel->chan, &sc->write_frame);
540                 sc->have_frame = 0;
541         }
542
543         /* Alllll done */
544         ast_mutex_unlock(&sc->lock);
545
546         if (update_talking != -1) {
547                 ast_bridge_notify_talking(bridge, bridge_channel, update_talking);
548         }
549
550         return res;
551
552 bridge_write_cleanup:
553         /* Even though the frame is not being written into the conference because it is not audio,
554          * we should use this opportunity to check to see if a frame is ready to be written out from
555          * the conference to the channel. */
556         ast_mutex_lock(&sc->lock);
557         if (sc->have_frame) {
558                 ast_write(bridge_channel->chan, &sc->write_frame);
559                 sc->have_frame = 0;
560         }
561         ast_mutex_unlock(&sc->lock);
562
563         return res;
564 }
565
566 /*! \brief Function called when the channel's thread is poked */
567 static int softmix_bridge_poke(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
568 {
569         struct softmix_channel *sc = bridge_channel->bridge_pvt;
570
571         ast_mutex_lock(&sc->lock);
572
573         if (sc->have_frame) {
574                 ast_write(bridge_channel->chan, &sc->write_frame);
575                 sc->have_frame = 0;
576         }
577
578         ast_mutex_unlock(&sc->lock);
579
580         return 0;
581 }
582
583 static void gather_softmix_stats(struct softmix_stats *stats,
584         const struct softmix_bridge_data *softmix_data,
585         struct ast_bridge_channel *bridge_channel)
586 {
587         int channel_native_rate;
588         int i;
589         /* Gather stats about channel sample rates. */
590         channel_native_rate = MAX(ast_format_rate(&bridge_channel->chan->rawwriteformat),
591                 ast_format_rate(&bridge_channel->chan->rawreadformat));
592
593         if (channel_native_rate > stats->highest_supported_rate) {
594                 stats->highest_supported_rate = channel_native_rate;
595         }
596         if (channel_native_rate > softmix_data->internal_rate) {
597                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
598                         if (stats->sample_rates[i] == channel_native_rate) {
599                                 stats->num_channels[i]++;
600                                 break;
601                         } else if (!stats->sample_rates[i]) {
602                                 stats->sample_rates[i] = channel_native_rate;
603                                 stats->num_channels[i]++;
604                                 break;
605                         }
606                 }
607                 stats->num_above_internal_rate++;
608         } else if (channel_native_rate == softmix_data->internal_rate) {
609                 stats->num_at_internal_rate++;
610         }
611 }
612 /*!
613  * \internal
614  * \brief Analyse mixing statistics and change bridges internal rate
615  * if necessary.
616  *
617  * \retval 0, no changes to internal rate 
618  * \ratval 1, internal rate was changed, update all the channels on the next mixing iteration.
619  */
620 static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data)
621 {
622         int i;
623         /* Re-adjust the internal bridge sample rate if
624          * 1. The bridge's internal sample rate is locked in at a sample
625          *    rate other than the current sample rate being used.
626          * 2. two or more channels support a higher sample rate
627          * 3. no channels support the current sample rate or a higher rate
628          */
629         if (stats->locked_rate) {
630                 /* if the rate is locked by the bridge, only update it if it differs
631                  * from the current rate we are using. */
632                 if (softmix_data->internal_rate != stats->locked_rate) {
633                         softmix_data->internal_rate = stats->locked_rate;
634                         ast_debug(1, " Bridge is locked in at sample rate %d\n", softmix_data->internal_rate);
635                         return 1;
636                 }
637         } else if (stats->num_above_internal_rate >= 2) {
638                 /* the highest rate is just used as a starting point */
639                 unsigned int best_rate = stats->highest_supported_rate;
640                 int best_index = -1;
641
642                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
643                         if (stats->num_channels[i]) {
644                                 break;
645                         }
646                         /* best_rate starts out being the first sample rate
647                          * greater than the internal sample rate that 2 or
648                          * more channels support. */
649                         if (stats->num_channels[i] >= 2 && (best_index == -1)) {
650                                 best_rate = stats->sample_rates[i];
651                                 best_index = i;
652                         /* If it has been detected that multiple rates above
653                          * the internal rate are present, compare those rates
654                          * to each other and pick the highest one two or more
655                          * channels support. */
656                         } else if (((best_index != -1) &&
657                                 (stats->num_channels[i] >= 2) &&
658                                 (stats->sample_rates[best_index] < stats->sample_rates[i]))) {
659                                 best_rate = stats->sample_rates[i];
660                                 best_index = i;
661                         /* It is possible that multiple channels exist with native sample
662                          * rates above the internal sample rate, but none of those channels
663                          * have the same rate in common.  In this case, the lowest sample
664                          * rate among those channels is picked. Over time as additional
665                          * statistic runs are made the internal sample rate number will
666                          * adjust to the most optimal sample rate, but it may take multiple
667                          * iterations. */
668                         } else if (best_index == -1) {
669                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
670                         }
671                 }
672
673                 ast_debug(1, " Bridge changed from %d To %d\n", softmix_data->internal_rate, best_rate);
674                 softmix_data->internal_rate = best_rate;
675                 return 1;
676         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
677                 /* In this case, the highest supported rate is actually lower than the internal rate */
678                 softmix_data->internal_rate = stats->highest_supported_rate;
679                 ast_debug(1, " Bridge changed from %d to %d\n", softmix_data->internal_rate, stats->highest_supported_rate);
680                 return 1;
681         }
682         return 0;
683 }
684
685 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries)
686 {
687         memset(mixing_array, 0, sizeof(*mixing_array));
688         mixing_array->max_num_entries = starting_num_entries;
689         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
690                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure. \n");
691                 return -1;
692         }
693         return 0;
694 }
695
696 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array)
697 {
698         ast_free(mixing_array->buffers);
699 }
700
701 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries)
702 {
703         int16_t **tmp;
704         /* give it some room to grow since memory is cheap but allocations can be expensive */
705         mixing_array->max_num_entries = num_entries;
706         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
707                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure. \n");
708                 return -1;
709         }
710         mixing_array->buffers = tmp;
711         return 0;
712 }
713
714 /*! \brief Function which acts as the mixing thread */
715 static int softmix_bridge_thread(struct ast_bridge *bridge)
716 {
717         struct softmix_stats stats = { { 0 }, };
718         struct softmix_mixing_array mixing_array;
719         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
720         struct ast_timer *timer;
721         struct softmix_translate_helper trans_helper;
722         int16_t buf[MAX_DATALEN] = { 0, };
723         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
724         int timingfd;
725         int update_all_rates = 0; /* set this when the internal sample rate has changed */
726         int i, x;
727         int res = -1;
728
729         if (!(softmix_data = bridge->bridge_pvt)) {
730                 goto softmix_cleanup;
731         }
732
733         ao2_ref(softmix_data, 1);
734         timer = softmix_data->timer;
735         timingfd = ast_timer_fd(timer);
736         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
737         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
738
739         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
740         if (softmix_mixing_array_init(&mixing_array, bridge->num + 10)) {
741                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure. \n");
742                 goto softmix_cleanup;
743         }
744
745         while (!bridge->stop && !bridge->refresh && bridge->array_num) {
746                 struct ast_bridge_channel *bridge_channel = NULL;
747                 int timeout = -1;
748                 enum ast_format_id cur_slin_id = ast_format_slin_by_rate(softmix_data->internal_rate);
749                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
750                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
751
752                 if (softmix_datalen > MAX_DATALEN) {
753                         /* This should NEVER happen, but if it does we need to know about it. Almost
754                          * all the memcpys used during this process depend on this assumption.  Rather
755                          * than checking this over and over again through out the code, this single
756                          * verification is done on each iteration. */
757                         ast_log(LOG_WARNING, "Conference mixing error, requested mixing length greater than mixing buffer.\n");
758                         goto softmix_cleanup;
759                 }
760
761                 /* Grow the mixing array buffer as participants are added. */
762                 if (mixing_array.max_num_entries < bridge->num && softmix_mixing_array_grow(&mixing_array, bridge->num + 5)) {
763                         goto softmix_cleanup;
764                 }
765
766                 /* init the number of buffers stored in the mixing array to 0.
767                  * As buffers are added for mixing, this number is incremented. */
768                 mixing_array.used_entries = 0;
769
770                 /* These variables help determine if a rate change is required */
771                 if (!stat_iteration_counter) {
772                         memset(&stats, 0, sizeof(stats));
773                         stats.locked_rate = bridge->internal_sample_rate;
774                 }
775
776                 /* If the sample rate has changed, update the translator helper */
777                 if (update_all_rates) {
778                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
779                 }
780
781                 /* Go through pulling audio from each factory that has it available */
782                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
783                         struct softmix_channel *sc = bridge_channel->bridge_pvt;
784
785                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
786                         if (update_all_rates) {
787                                 set_softmix_bridge_data(softmix_data->internal_rate, softmix_data->internal_mixing_interval, bridge_channel, 1);
788                         }
789
790                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
791                         if (!stat_iteration_counter) {
792                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
793                         }
794
795                         /* if the channel is suspended, don't check for audio, but still gather stats */
796                         if (bridge_channel->suspended) {
797                                 continue;
798                         }
799
800                         /* Try to get audio from the factory if available */
801                         ast_mutex_lock(&sc->lock);
802                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
803                                 mixing_array.used_entries++;
804                         }
805                         ast_mutex_unlock(&sc->lock);
806                 }
807
808                 /* mix it like crazy */
809                 memset(buf, 0, softmix_datalen);
810                 for (i = 0; i < mixing_array.used_entries; i++) {
811                         for (x = 0; x < softmix_samples; x++) {
812                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[i] + x);
813                         }
814                 }
815
816                 /* Next step go through removing the channel's own audio and creating a good frame... */
817                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
818                         struct softmix_channel *sc = bridge_channel->bridge_pvt;
819
820                         if (bridge_channel->suspended) {
821                                 continue;
822                         }
823
824                         ast_mutex_lock(&sc->lock);
825
826                         /* Make SLINEAR write frame from local buffer */
827                         if (sc->write_frame.subclass.format.id != cur_slin_id) {
828                                 ast_format_set(&sc->write_frame.subclass.format, cur_slin_id, 0);
829                         }
830                         sc->write_frame.datalen = softmix_datalen;
831                         sc->write_frame.samples = softmix_samples;
832                         memcpy(sc->final_buf, buf, softmix_datalen);
833
834                         /* process the softmix channel's new write audio */
835                         softmix_process_write_audio(&trans_helper, &bridge_channel->chan->rawwriteformat, sc);
836
837                         /* The frame is now ready for use... */
838                         sc->have_frame = 1;
839
840                         ast_mutex_unlock(&sc->lock);
841
842                         /* Poke bridged channel thread just in case */
843                         pthread_kill(bridge_channel->thread, SIGURG);
844                 }
845
846                 update_all_rates = 0;
847                 if (!stat_iteration_counter) {
848                         update_all_rates = analyse_softmix_stats(&stats, softmix_data);
849                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
850                 }
851                 stat_iteration_counter--;
852
853                 ao2_unlock(bridge);
854                 /* cleanup any translation frame data from the previous mixing iteration. */
855                 softmix_translate_helper_cleanup(&trans_helper);
856                 /* Wait for the timing source to tell us to wake up and get things done */
857                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
858                 ast_timer_ack(timer, 1);
859                 ao2_lock(bridge);
860
861                 /* make sure to detect mixing interval changes if they occur. */
862                 if (bridge->internal_mixing_interval && (bridge->internal_mixing_interval != softmix_data->internal_mixing_interval)) {
863                         softmix_data->internal_mixing_interval = bridge->internal_mixing_interval;
864                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
865                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
866                 }
867         }
868
869         res = 0;
870
871 softmix_cleanup:
872         softmix_translate_helper_destroy(&trans_helper);
873         softmix_mixing_array_destroy(&mixing_array);
874         if (softmix_data) {
875                 ao2_ref(softmix_data, -1);
876         }
877         return res;
878 }
879
880 static struct ast_bridge_technology softmix_bridge = {
881         .name = "softmix",
882         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX | AST_BRIDGE_CAPABILITY_THREAD | AST_BRIDGE_CAPABILITY_MULTITHREADED | AST_BRIDGE_CAPABILITY_OPTIMIZE | AST_BRIDGE_CAPABILITY_VIDEO,
883         .preference = AST_BRIDGE_PREFERENCE_LOW,
884         .create = softmix_bridge_create,
885         .destroy = softmix_bridge_destroy,
886         .join = softmix_bridge_join,
887         .leave = softmix_bridge_leave,
888         .write = softmix_bridge_write,
889         .thread = softmix_bridge_thread,
890         .poke = softmix_bridge_poke,
891 };
892
893 static int unload_module(void)
894 {
895         ast_format_cap_destroy(softmix_bridge.format_capabilities);
896         return ast_bridge_technology_unregister(&softmix_bridge);
897 }
898
899 static int load_module(void)
900 {
901         struct ast_format tmp;
902         if (!(softmix_bridge.format_capabilities = ast_format_cap_alloc())) {
903                 return AST_MODULE_LOAD_DECLINE;
904         }
905         ast_format_cap_add(softmix_bridge.format_capabilities, ast_format_set(&tmp, AST_FORMAT_SLINEAR, 0));
906         return ast_bridge_technology_register(&softmix_bridge);
907 }
908
909 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");