bridges/bridge_softmix.c

   1 /*
   2  * Asterisk -- An open source telephony toolkit.
   3  *
   4  * Copyright (C) 2011, Digium, Inc.
   5  *
   6  * Joshua Colp <jcolp@digium.com>
   7  * David Vossel <dvossel@digium.com>
   8  *
   9  * See http://www.asterisk.org for more information about
  10  * the Asterisk project. Please do not directly contact
  11  * any of the maintainers of this project for assistance;
  12  * the project provides a web site, mailing lists and IRC
  13  * channels for your use.
  14  *
  15  * This program is free software, distributed under the terms of
  16  * the GNU General Public License Version 2. See the LICENSE file
  17  * at the top of the source tree.
  18  */
  19
  20 /*! \file
  21  *
  22  * \brief Multi-party software based channel mixing
  23  *
  24  * \author Joshua Colp <jcolp@digium.com>
  25  * \author David Vossel <dvossel@digium.com>
  26  *
  27  * \ingroup bridges
  28  */
  29
  30 #include "asterisk.h"
  31
  32 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <sys/time.h>
  38 #include <signal.h>
  39 #include <errno.h>
  40 #include <unistd.h>
  41
  42 #include "asterisk/module.h"
  43 #include "asterisk/channel.h"
  44 #include "asterisk/bridging.h"
  45 #include "asterisk/bridging_technology.h"
  46 #include "asterisk/frame.h"
  47 #include "asterisk/options.h"
  48 #include "asterisk/logger.h"
  49 #include "asterisk/slinfactory.h"
  50 #include "asterisk/astobj2.h"
  51 #include "asterisk/timing.h"
  52 #include "asterisk/translate.h"
  53
  54 #define MAX_DATALEN 8096
  55
  56 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
  57 #define DEFAULT_SOFTMIX_INTERVAL 20
  58
  59 /*! \brief Size of the buffer used for sample manipulation */
  60 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
  61
  62 /*! \brief Number of samples we are dealing with */
  63 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
  64
  65 /*! \brief Number of mixing iterations to perform between gathering statistics. */
  66 #define SOFTMIX_STAT_INTERVAL 100
  67
  68 /* This is the threshold in ms at which a channel's own audio will stop getting
  69  * mixed out its own write audio stream because it is not talking. */
  70 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
  71 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
  72
  73 #define DEFAULT_ENERGY_HISTORY_LEN 150
  74
  75 struct video_follow_talker_data {
  76         /*! audio energy history */
  77         int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
  78         /*! The current slot being used in the history buffer, this
  79          *  increments and wraps around */
  80         int energy_history_cur_slot;
  81         /*! The current energy sum used for averages. */
  82         int energy_accum;
  83         /*! The current energy average */
  84         int energy_average;
  85 };
  86
  87 /*! \brief Structure which contains per-channel mixing information */
  88 struct softmix_channel {
  89         /*! Lock to protect this structure */
  90         ast_mutex_t lock;
  91         /*! Factory which contains audio read in from the channel */
  92         struct ast_slinfactory factory;
  93         /*! Frame that contains mixed audio to be written out to the channel */
  94         struct ast_frame write_frame;
  95         /*! Frame that contains mixed audio read from the channel */
  96         struct ast_frame read_frame;
  97         /*! DSP for detecting silence */
  98         struct ast_dsp *dsp;
  99         /*! Bit used to indicate if a channel is talking or not. This affects how
 100          * the channel's audio is mixed back to it. */
 101         int talking:1;
 102         /*! Bit used to indicate that the channel provided audio for this mixing interval */
 103         int have_audio:1;
 104         /*! Bit used to indicate that a frame is available to be written out to the channel */
 105         int have_frame:1;
 106         /*! Buffer containing final mixed audio from all sources */
 107         short final_buf[MAX_DATALEN];
 108         /*! Buffer containing only the audio from the channel */
 109         short our_buf[MAX_DATALEN];
 110         /*! Data pertaining to talker mode for video conferencing */
 111         struct video_follow_talker_data video_talker;
 112 };
 113
 114 struct softmix_bridge_data {
 115         struct ast_timer *timer;
 116         unsigned int internal_rate;
 117         unsigned int internal_mixing_interval;
 118 };
 119
 120 struct softmix_stats {
 121                 /*! Each index represents a sample rate used above the internal rate. */
 122                 unsigned int sample_rates[16];
 123                 /*! Each index represents the number of channels using the same index in the sample_rates array.  */
 124                 unsigned int num_channels[16];
 125                 /*! the number of channels above the internal sample rate */
 126                 unsigned int num_above_internal_rate;
 127                 /*! the number of channels at the internal sample rate */
 128                 unsigned int num_at_internal_rate;
 129                 /*! the absolute highest sample rate supported by any channel in the bridge */
 130                 unsigned int highest_supported_rate;
 131                 /*! Is the sample rate locked by the bridge, if so what is that rate.*/
 132                 unsigned int locked_rate;
 133 };
 134
 135 struct softmix_mixing_array {
 136         int max_num_entries;
 137         int used_entries;
 138         int16_t **buffers;
 139 };
 140
 141 struct softmix_translate_helper_entry {
 142         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
 143                                       and re-init if it was usable. */
 144         struct ast_format dst_format; /*!< The destination format for this helper */
 145         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
 146         struct ast_frame *out_frame; /*!< The output frame from the last translation */
 147         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
 148 };
 149
 150 struct softmix_translate_helper {
 151         struct ast_format slin_src; /*!< the source format expected for all the translators */
 152         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
 153 };
 154
 155 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
 156 {
 157         struct softmix_translate_helper_entry *entry;
 158         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
 159                 return NULL;
 160         }
 161         ast_format_copy(&entry->dst_format, dst);
 162         return entry;
 163 }
 164
 165 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
 166 {
 167         if (entry->trans_pvt) {
 168                 ast_translator_free_path(entry->trans_pvt);
 169         }
 170         if (entry->out_frame) {
 171                 ast_frfree(entry->out_frame);
 172         }
 173         ast_free(entry);
 174         return NULL;
 175 }
 176
 177 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
 178 {
 179         memset(trans_helper, 0, sizeof(*trans_helper));
 180         ast_format_set(&trans_helper->slin_src, ast_format_slin_by_rate(sample_rate), 0);
 181 }
 182
 183 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
 184 {
 185         struct softmix_translate_helper_entry *entry;
 186
 187         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
 188                 softmix_translate_helper_free_entry(entry);
 189         }
 190 }
 191
 192 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
 193 {
 194         struct softmix_translate_helper_entry *entry;
 195
 196         ast_format_set(&trans_helper->slin_src, ast_format_slin_by_rate(sample_rate), 0);
 197         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
 198                 if (entry->trans_pvt) {
 199                         ast_translator_free_path(entry->trans_pvt);
 200                         if (!(entry->trans_pvt = ast_translator_build_path(&entry->dst_format, &trans_helper->slin_src))) {
 201                                 AST_LIST_REMOVE_CURRENT(entry);
 202                                 entry = softmix_translate_helper_free_entry(entry);
 203                         }
 204                 }
 205         }
 206         AST_LIST_TRAVERSE_SAFE_END;
 207 }
 208
 209 /*!
 210  * \internal
 211  * \brief Get the next available audio on the softmix channel's read stream
 212  * and determine if it should be mixed out or not on the write stream.
 213  *
 214  * \retval pointer to buffer containing the exact number of samples requested on success.
 215  * \retval NULL if no samples are present
 216  */
 217 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
 218 {
 219         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
 220                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
 221                 sc->have_audio = 1;
 222                 return sc->our_buf;
 223         }
 224         sc->have_audio = 0;
 225         return NULL;
 226 }
 227
 228 /*!
 229  * \internal
 230  * \brief Process a softmix channel's write audio
 231  *
 232  * \details This function will remove the channel's talking from its own audio if present and
 233  * possibly even do the channel's write translation for it depending on how many other
 234  * channels use the same write format.
 235  */
 236 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
 237         struct ast_format *raw_write_fmt,
 238         struct softmix_channel *sc)
 239 {
 240         struct softmix_translate_helper_entry *entry = NULL;
 241         int i;
 242
 243         /* If we provided audio that was not determined to be silence,
 244          * then take it out while in slinear format. */
 245         if (sc->have_audio && sc->talking) {
 246                 for (i = 0; i < sc->write_frame.samples; i++) {
 247                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
 248                 }
 249                 /* do not do any special write translate optimization if we had to make
 250                  * a special mix for them to remove their own audio. */
 251                 return;
 252         }
 253
 254         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
 255                 if (ast_format_cmp(&entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
 256                         entry->num_times_requested++;
 257                 } else {
 258                         continue;
 259                 }
 260                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
 261                         entry->trans_pvt = ast_translator_build_path(&entry->dst_format, &trans_helper->slin_src);
 262                 }
 263                 if (entry->trans_pvt && !entry->out_frame) {
 264                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
 265                 }
 266                 if (entry->out_frame && (entry->out_frame->datalen < MAX_DATALEN)) {
 267                         ast_format_copy(&sc->write_frame.subclass.format, &entry->out_frame->subclass.format);
 268                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
 269                         sc->write_frame.datalen = entry->out_frame->datalen;
 270                         sc->write_frame.samples = entry->out_frame->samples;
 271                 }
 272                 break;
 273         }
 274
 275         /* add new entry into list if this format destination was not matched. */
 276         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
 277                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
 278         }
 279 }
 280
 281 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
 282 {
 283         struct softmix_translate_helper_entry *entry = NULL;
 284         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
 285                 if (entry->out_frame) {
 286                         ast_frfree(entry->out_frame);
 287                         entry->out_frame = NULL;
 288                 }
 289                 entry->num_times_requested = 0;
 290         }
 291 }
 292
 293 static void softmix_bridge_data_destroy(void *obj)
 294 {
 295         struct softmix_bridge_data *softmix_data = obj;
 296         ast_timer_close(softmix_data->timer);
 297 }
 298
 299 /*! \brief Function called when a bridge is created */
 300 static int softmix_bridge_create(struct ast_bridge *bridge)
 301 {
 302         struct softmix_bridge_data *softmix_data;
 303
 304         if (!(softmix_data = ao2_alloc(sizeof(*softmix_data), softmix_bridge_data_destroy))) {
 305                 return -1;
 306         }
 307         if (!(softmix_data->timer = ast_timer_open())) {
 308                 ao2_ref(softmix_data, -1);
 309                 return -1;
 310         }
 311
 312         /* start at 8khz, let it grow from there */
 313         softmix_data->internal_rate = 8000;
 314         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
 315
 316         bridge->bridge_pvt = softmix_data;
 317         return 0;
 318 }
 319
 320 /*! \brief Function called when a bridge is destroyed */
 321 static int softmix_bridge_destroy(struct ast_bridge *bridge)
 322 {
 323         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
 324         if (!bridge->bridge_pvt) {
 325                 return -1;
 326         }
 327         ao2_ref(softmix_data, -1);
 328         bridge->bridge_pvt = NULL;
 329         return 0;
 330 }
 331
 332 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset)
 333 {
 334         struct softmix_channel *sc = bridge_channel->bridge_pvt;
 335         unsigned int channel_read_rate = ast_format_rate(&bridge_channel->chan->rawreadformat);
 336
 337         ast_mutex_lock(&sc->lock);
 338         if (reset) {
 339                 ast_slinfactory_destroy(&sc->factory);
 340                 ast_dsp_free(sc->dsp);
 341         }
 342         /* Setup read/write frame parameters */
 343         sc->write_frame.frametype = AST_FRAME_VOICE;
 344         ast_format_set(&sc->write_frame.subclass.format, ast_format_slin_by_rate(rate), 0);
 345         sc->write_frame.data.ptr = sc->final_buf;
 346         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
 347         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
 348
 349         sc->read_frame.frametype = AST_FRAME_VOICE;
 350         ast_format_set(&sc->read_frame.subclass.format, ast_format_slin_by_rate(channel_read_rate), 0);
 351         sc->read_frame.data.ptr = sc->our_buf;
 352         sc->read_frame.datalen = SOFTMIX_DATALEN(channel_read_rate, interval);
 353         sc->read_frame.samples = SOFTMIX_SAMPLES(channel_read_rate, interval);
 354
 355         /* Setup smoother */
 356         ast_slinfactory_init_with_format(&sc->factory, &sc->write_frame.subclass.format);
 357
 358         /* set new read and write formats on channel. */
 359         ast_set_read_format(bridge_channel->chan, &sc->read_frame.subclass.format);
 360         ast_set_write_format(bridge_channel->chan, &sc->write_frame.subclass.format);
 361
 362         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
 363         sc->dsp = ast_dsp_new_with_rate(channel_read_rate);
 364         /* we want to aggressively detect silence to avoid feedback */
 365         if (bridge_channel->tech_args.talking_threshold) {
 366                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
 367         } else {
 368                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
 369         }
 370
 371         ast_mutex_unlock(&sc->lock);
 372 }
 373
 374 /*! \brief Function called when a channel is joined into the bridge */
 375 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
 376 {
 377         struct softmix_channel *sc = NULL;
 378         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
 379
 380         /* Create a new softmix_channel structure and allocate various things on it */
 381         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
 382                 return -1;
 383         }
 384
 385         /* Can't forget the lock */
 386         ast_mutex_init(&sc->lock);
 387
 388         /* Can't forget to record our pvt structure within the bridged channel structure */
 389         bridge_channel->bridge_pvt = sc;
 390
 391         set_softmix_bridge_data(softmix_data->internal_rate,
 392                 softmix_data->internal_mixing_interval ? softmix_data->internal_mixing_interval : DEFAULT_SOFTMIX_INTERVAL,
 393                 bridge_channel, 0);
 394
 395         return 0;
 396 }
 397
 398 /*! \brief Function called when a channel leaves the bridge */
 399 static int softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
 400 {
 401         struct softmix_channel *sc = bridge_channel->bridge_pvt;
 402
 403         if (!(bridge_channel->bridge_pvt)) {
 404                 return 0;
 405         }
 406         bridge_channel->bridge_pvt = NULL;
 407
 408         /* Drop mutex lock */
 409         ast_mutex_destroy(&sc->lock);
 410
 411         /* Drop the factory */
 412         ast_slinfactory_destroy(&sc->factory);
 413
 414         /* Drop the DSP */
 415         ast_dsp_free(sc->dsp);
 416
 417         /* Eep! drop ourselves */
 418         ast_free(sc);
 419
 420         return 0;
 421 }
 422
 423 /*!
 424  * \internal
 425  * \brief If the bridging core passes DTMF to us, then they want it to be distributed out to all memebers. Do that here.
 426  */
 427 static void softmix_pass_dtmf(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
 428 {
 429         struct ast_bridge_channel *tmp;
 430         AST_LIST_TRAVERSE(&bridge->channels, tmp, entry) {
 431                 if (tmp == bridge_channel) {
 432                         continue;
 433                 }
 434                 ast_write(tmp->chan, frame);
 435         }
 436 }
 437
 438 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
 439 {
 440         struct ast_bridge_channel *tmp;
 441         AST_LIST_TRAVERSE(&bridge->channels, tmp, entry) {
 442                 if (tmp->suspended) {
 443                         continue;
 444                 }
 445                 if (ast_bridge_is_video_src(bridge, tmp->chan) == 1) {
 446                         ast_write(tmp->chan, frame);
 447                         break;
 448                 }
 449         }
 450 }
 451
 452 static void softmix_pass_video_all(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame, int echo)
 453 {
 454         struct ast_bridge_channel *tmp;
 455         AST_LIST_TRAVERSE(&bridge->channels, tmp, entry) {
 456                 if (tmp->suspended) {
 457                         continue;
 458                 }
 459                 if ((tmp->chan == bridge_channel->chan) && !echo) {
 460                         continue;
 461                 }
 462                 ast_write(tmp->chan, frame);
 463         }
 464 }
 465
 466 /*! \brief Function called when a channel writes a frame into the bridge */
 467 static enum ast_bridge_write_result softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
 468 {
 469         struct softmix_channel *sc = bridge_channel->bridge_pvt;
 470         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
 471         int totalsilence = 0;
 472         int cur_energy = 0;
 473         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
 474                 bridge_channel->tech_args.silence_threshold :
 475                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
 476         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
 477         int res = AST_BRIDGE_WRITE_SUCCESS;
 478
 479         /* Only accept audio frames, all others are unsupported */
 480         if (frame->frametype == AST_FRAME_DTMF_END || frame->frametype == AST_FRAME_DTMF_BEGIN) {
 481                 softmix_pass_dtmf(bridge, bridge_channel, frame);
 482                 goto bridge_write_cleanup;
 483         } else if (frame->frametype != AST_FRAME_VOICE && frame->frametype != AST_FRAME_VIDEO) {
 484                 res = AST_BRIDGE_WRITE_UNSUPPORTED;
 485                 goto bridge_write_cleanup;
 486         } else if (frame->datalen == 0) {
 487                 goto bridge_write_cleanup;
 488         }
 489
 490         /* Determine if this video frame should be distributed or not */
 491         if (frame->frametype == AST_FRAME_VIDEO) {
 492                 int num_src = ast_bridge_number_video_src(bridge);
 493                 int video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
 494
 495                 switch (bridge->video_mode.mode) {
 496                 case AST_BRIDGE_VIDEO_MODE_NONE:
 497                         break;
 498                 case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
 499                         if (video_src_priority == 1) {
 500                                 softmix_pass_video_all(bridge, bridge_channel, frame, 1);
 501                         }
 502                         break;
 503                 case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
 504                         ast_mutex_lock(&sc->lock);
 505                         ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan, sc->video_talker.energy_average, ast_format_get_video_mark(&frame->subclass.format));
 506                         ast_mutex_unlock(&sc->lock);
 507                         if (video_src_priority == 1) {
 508                                 int echo = num_src > 1 ? 0 : 1;
 509                                 softmix_pass_video_all(bridge, bridge_channel, frame, echo);
 510                         } else if (video_src_priority == 2) {
 511                                 softmix_pass_video_top_priority(bridge, frame);
 512                         }
 513                         break;
 514                 }
 515                 goto bridge_write_cleanup;
 516         }
 517
 518         /* If we made it here, we are going to write the frame into the conference */
 519         ast_mutex_lock(&sc->lock);
 520         ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
 521
 522         if (bridge->video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
 523                 int cur_slot = sc->video_talker.energy_history_cur_slot;
 524                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
 525                 sc->video_talker.energy_accum += cur_energy;
 526                 sc->video_talker.energy_history[cur_slot] = cur_energy;
 527                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
 528                 sc->video_talker.energy_history_cur_slot++;
 529                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
 530                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
 531                 }
 532         }
 533
 534         if (totalsilence < silence_threshold) {
 535                 if (!sc->talking) {
 536                         update_talking = 1;
 537                 }
 538                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
 539         } else {
 540                 if (sc->talking) {
 541                         update_talking = 0;
 542                 }
 543                 sc->talking = 0;
 544         }
 545
 546         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
 547          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
 548          * the audio by flushing the buffer before adding new audio in. */
 549         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
 550                 ast_slinfactory_flush(&sc->factory);
 551         }
 552
 553         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
 554          * is not determined to be talking. */
 555         if (!(bridge_channel->tech_args.drop_silence && !sc->talking) &&
 556                 (frame->frametype == AST_FRAME_VOICE && ast_format_is_slinear(&frame->subclass.format))) {
 557                 ast_slinfactory_feed(&sc->factory, frame);
 558         }
 559
 560         /* If a frame is ready to be written out, do so */
 561         if (sc->have_frame) {
 562                 ast_write(bridge_channel->chan, &sc->write_frame);
 563                 sc->have_frame = 0;
 564         }
 565
 566         /* Alllll done */
 567         ast_mutex_unlock(&sc->lock);
 568
 569         if (update_talking != -1) {
 570                 ast_bridge_notify_talking(bridge, bridge_channel, update_talking);
 571         }
 572
 573         return res;
 574
 575 bridge_write_cleanup:
 576         /* Even though the frame is not being written into the conference because it is not audio,
 577          * we should use this opportunity to check to see if a frame is ready to be written out from
 578          * the conference to the channel. */
 579         ast_mutex_lock(&sc->lock);
 580         if (sc->have_frame) {
 581                 ast_write(bridge_channel->chan, &sc->write_frame);
 582                 sc->have_frame = 0;
 583         }
 584         ast_mutex_unlock(&sc->lock);
 585
 586         return res;
 587 }
 588
 589 /*! \brief Function called when the channel's thread is poked */
 590 static int softmix_bridge_poke(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
 591 {
 592         struct softmix_channel *sc = bridge_channel->bridge_pvt;
 593
 594         ast_mutex_lock(&sc->lock);
 595
 596         if (sc->have_frame) {
 597                 ast_write(bridge_channel->chan, &sc->write_frame);
 598                 sc->have_frame = 0;
 599         }
 600
 601         ast_mutex_unlock(&sc->lock);
 602
 603         return 0;
 604 }
 605
 606 static void gather_softmix_stats(struct softmix_stats *stats,
 607         const struct softmix_bridge_data *softmix_data,
 608         struct ast_bridge_channel *bridge_channel)
 609 {
 610         int channel_native_rate;
 611         int i;
 612         /* Gather stats about channel sample rates. */
 613         channel_native_rate = MAX(ast_format_rate(&bridge_channel->chan->rawwriteformat),
 614                 ast_format_rate(&bridge_channel->chan->rawreadformat));
 615
 616         if (channel_native_rate > stats->highest_supported_rate) {
 617                 stats->highest_supported_rate = channel_native_rate;
 618         }
 619         if (channel_native_rate > softmix_data->internal_rate) {
 620                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
 621                         if (stats->sample_rates[i] == channel_native_rate) {
 622                                 stats->num_channels[i]++;
 623                                 break;
 624                         } else if (!stats->sample_rates[i]) {
 625                                 stats->sample_rates[i] = channel_native_rate;
 626                                 stats->num_channels[i]++;
 627                                 break;
 628                         }
 629                 }
 630                 stats->num_above_internal_rate++;
 631         } else if (channel_native_rate == softmix_data->internal_rate) {
 632                 stats->num_at_internal_rate++;
 633         }
 634 }
 635 /*!
 636  * \internal
 637  * \brief Analyse mixing statistics and change bridges internal rate
 638  * if necessary.
 639  *
 640  * \retval 0, no changes to internal rate
 641  * \ratval 1, internal rate was changed, update all the channels on the next mixing iteration.
 642  */
 643 static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data)
 644 {
 645         int i;
 646         /* Re-adjust the internal bridge sample rate if
 647          * 1. The bridge's internal sample rate is locked in at a sample
 648          *    rate other than the current sample rate being used.
 649          * 2. two or more channels support a higher sample rate
 650          * 3. no channels support the current sample rate or a higher rate
 651          */
 652         if (stats->locked_rate) {
 653                 /* if the rate is locked by the bridge, only update it if it differs
 654                  * from the current rate we are using. */
 655                 if (softmix_data->internal_rate != stats->locked_rate) {
 656                         softmix_data->internal_rate = stats->locked_rate;
 657                         ast_debug(1, " Bridge is locked in at sample rate %d\n", softmix_data->internal_rate);
 658                         return 1;
 659                 }
 660         } else if (stats->num_above_internal_rate >= 2) {
 661                 /* the highest rate is just used as a starting point */
 662                 unsigned int best_rate = stats->highest_supported_rate;
 663                 int best_index = -1;
 664
 665                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
 666                         if (stats->num_channels[i]) {
 667                                 break;
 668                         }
 669                         /* best_rate starts out being the first sample rate
 670                          * greater than the internal sample rate that 2 or
 671                          * more channels support. */
 672                         if (stats->num_channels[i] >= 2 && (best_index == -1)) {
 673                                 best_rate = stats->sample_rates[i];
 674                                 best_index = i;
 675                         /* If it has been detected that multiple rates above
 676                          * the internal rate are present, compare those rates
 677                          * to each other and pick the highest one two or more
 678                          * channels support. */
 679                         } else if (((best_index != -1) &&
 680                                 (stats->num_channels[i] >= 2) &&
 681                                 (stats->sample_rates[best_index] < stats->sample_rates[i]))) {
 682                                 best_rate = stats->sample_rates[i];
 683                                 best_index = i;
 684                         /* It is possible that multiple channels exist with native sample
 685                          * rates above the internal sample rate, but none of those channels
 686                          * have the same rate in common.  In this case, the lowest sample
 687                          * rate among those channels is picked. Over time as additional
 688                          * statistic runs are made the internal sample rate number will
 689                          * adjust to the most optimal sample rate, but it may take multiple
 690                          * iterations. */
 691                         } else if (best_index == -1) {
 692                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
 693                         }
 694                 }
 695
 696                 ast_debug(1, " Bridge changed from %d To %d\n", softmix_data->internal_rate, best_rate);
 697                 softmix_data->internal_rate = best_rate;
 698                 return 1;
 699         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
 700                 /* In this case, the highest supported rate is actually lower than the internal rate */
 701                 softmix_data->internal_rate = stats->highest_supported_rate;
 702                 ast_debug(1, " Bridge changed from %d to %d\n", softmix_data->internal_rate, stats->highest_supported_rate);
 703                 return 1;
 704         }
 705         return 0;
 706 }
 707
 708 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries)
 709 {
 710         memset(mixing_array, 0, sizeof(*mixing_array));
 711         mixing_array->max_num_entries = starting_num_entries;
 712         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
 713                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure. \n");
 714                 return -1;
 715         }
 716         return 0;
 717 }
 718
 719 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array)
 720 {
 721         ast_free(mixing_array->buffers);
 722 }
 723
 724 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries)
 725 {
 726         int16_t **tmp;
 727         /* give it some room to grow since memory is cheap but allocations can be expensive */
 728         mixing_array->max_num_entries = num_entries;
 729         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
 730                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure. \n");
 731                 return -1;
 732         }
 733         mixing_array->buffers = tmp;
 734         return 0;
 735 }
 736
 737 /*! \brief Function which acts as the mixing thread */
 738 static int softmix_bridge_thread(struct ast_bridge *bridge)
 739 {
 740         struct softmix_stats stats = { { 0 }, };
 741         struct softmix_mixing_array mixing_array;
 742         struct softmix_bridge_data *softmix_data = bridge->bridge_pvt;
 743         struct ast_timer *timer;
 744         struct softmix_translate_helper trans_helper;
 745         int16_t buf[MAX_DATALEN] = { 0, };
 746         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
 747         int timingfd;
 748         int update_all_rates = 0; /* set this when the internal sample rate has changed */
 749         int i, x;
 750         int res = -1;
 751
 752         if (!(softmix_data = bridge->bridge_pvt)) {
 753                 goto softmix_cleanup;
 754         }
 755
 756         ao2_ref(softmix_data, 1);
 757         timer = softmix_data->timer;
 758         timingfd = ast_timer_fd(timer);
 759         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
 760         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
 761
 762         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
 763         if (softmix_mixing_array_init(&mixing_array, bridge->num + 10)) {
 764                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure. \n");
 765                 goto softmix_cleanup;
 766         }
 767
 768         while (!bridge->stop && !bridge->refresh && bridge->array_num) {
 769                 struct ast_bridge_channel *bridge_channel = NULL;
 770                 int timeout = -1;
 771                 enum ast_format_id cur_slin_id = ast_format_slin_by_rate(softmix_data->internal_rate);
 772                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
 773                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
 774
 775                 if (softmix_datalen > MAX_DATALEN) {
 776                         /* This should NEVER happen, but if it does we need to know about it. Almost
 777                          * all the memcpys used during this process depend on this assumption.  Rather
 778                          * than checking this over and over again through out the code, this single
 779                          * verification is done on each iteration. */
 780                         ast_log(LOG_WARNING, "Conference mixing error, requested mixing length greater than mixing buffer.\n");
 781                         goto softmix_cleanup;
 782                 }
 783
 784                 /* Grow the mixing array buffer as participants are added. */
 785                 if (mixing_array.max_num_entries < bridge->num && softmix_mixing_array_grow(&mixing_array, bridge->num + 5)) {
 786                         goto softmix_cleanup;
 787                 }
 788
 789                 /* init the number of buffers stored in the mixing array to 0.
 790                  * As buffers are added for mixing, this number is incremented. */
 791                 mixing_array.used_entries = 0;
 792
 793                 /* These variables help determine if a rate change is required */
 794                 if (!stat_iteration_counter) {
 795                         memset(&stats, 0, sizeof(stats));
 796                         stats.locked_rate = bridge->internal_sample_rate;
 797                 }
 798
 799                 /* If the sample rate has changed, update the translator helper */
 800                 if (update_all_rates) {
 801                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
 802                 }
 803
 804                 /* Go through pulling audio from each factory that has it available */
 805                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
 806                         struct softmix_channel *sc = bridge_channel->bridge_pvt;
 807
 808                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
 809                         if (update_all_rates) {
 810                                 set_softmix_bridge_data(softmix_data->internal_rate, softmix_data->internal_mixing_interval, bridge_channel, 1);
 811                         }
 812
 813                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
 814                         if (!stat_iteration_counter) {
 815                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
 816                         }
 817
 818                         /* if the channel is suspended, don't check for audio, but still gather stats */
 819                         if (bridge_channel->suspended) {
 820                                 continue;
 821                         }
 822
 823                         /* Try to get audio from the factory if available */
 824                         ast_mutex_lock(&sc->lock);
 825                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
 826                                 mixing_array.used_entries++;
 827                         }
 828                         ast_mutex_unlock(&sc->lock);
 829                 }
 830
 831                 /* mix it like crazy */
 832                 memset(buf, 0, softmix_datalen);
 833                 for (i = 0; i < mixing_array.used_entries; i++) {
 834                         for (x = 0; x < softmix_samples; x++) {
 835                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[i] + x);
 836                         }
 837                 }
 838
 839                 /* Next step go through removing the channel's own audio and creating a good frame... */
 840                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
 841                         struct softmix_channel *sc = bridge_channel->bridge_pvt;
 842
 843                         if (bridge_channel->suspended) {
 844                                 continue;
 845                         }
 846
 847                         ast_mutex_lock(&sc->lock);
 848
 849                         /* Make SLINEAR write frame from local buffer */
 850                         if (sc->write_frame.subclass.format.id != cur_slin_id) {
 851                                 ast_format_set(&sc->write_frame.subclass.format, cur_slin_id, 0);
 852                         }
 853                         sc->write_frame.datalen = softmix_datalen;
 854                         sc->write_frame.samples = softmix_samples;
 855                         memcpy(sc->final_buf, buf, softmix_datalen);
 856
 857                         /* process the softmix channel's new write audio */
 858                         softmix_process_write_audio(&trans_helper, &bridge_channel->chan->rawwriteformat, sc);
 859
 860                         /* The frame is now ready for use... */
 861                         sc->have_frame = 1;
 862
 863                         ast_mutex_unlock(&sc->lock);
 864
 865                         /* Poke bridged channel thread just in case */
 866                         pthread_kill(bridge_channel->thread, SIGURG);
 867                 }
 868
 869                 update_all_rates = 0;
 870                 if (!stat_iteration_counter) {
 871                         update_all_rates = analyse_softmix_stats(&stats, softmix_data);
 872                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
 873                 }
 874                 stat_iteration_counter--;
 875
 876                 ao2_unlock(bridge);
 877                 /* cleanup any translation frame data from the previous mixing iteration. */
 878                 softmix_translate_helper_cleanup(&trans_helper);
 879                 /* Wait for the timing source to tell us to wake up and get things done */
 880                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
 881                 ast_timer_ack(timer, 1);
 882                 ao2_lock(bridge);
 883
 884                 /* make sure to detect mixing interval changes if they occur. */
 885                 if (bridge->internal_mixing_interval && (bridge->internal_mixing_interval != softmix_data->internal_mixing_interval)) {
 886                         softmix_data->internal_mixing_interval = bridge->internal_mixing_interval;
 887                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
 888                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
 889                 }
 890         }
 891
 892         res = 0;
 893
 894 softmix_cleanup:
 895         softmix_translate_helper_destroy(&trans_helper);
 896         softmix_mixing_array_destroy(&mixing_array);
 897         if (softmix_data) {
 898                 ao2_ref(softmix_data, -1);
 899         }
 900         return res;
 901 }
 902
 903 static struct ast_bridge_technology softmix_bridge = {
 904         .name = "softmix",
 905         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX | AST_BRIDGE_CAPABILITY_THREAD | AST_BRIDGE_CAPABILITY_MULTITHREADED | AST_BRIDGE_CAPABILITY_OPTIMIZE | AST_BRIDGE_CAPABILITY_VIDEO,
 906         .preference = AST_BRIDGE_PREFERENCE_LOW,
 907         .create = softmix_bridge_create,
 908         .destroy = softmix_bridge_destroy,
 909         .join = softmix_bridge_join,
 910         .leave = softmix_bridge_leave,
 911         .write = softmix_bridge_write,
 912         .thread = softmix_bridge_thread,
 913         .poke = softmix_bridge_poke,
 914 };
 915
 916 static int unload_module(void)
 917 {
 918         ast_format_cap_destroy(softmix_bridge.format_capabilities);
 919         return ast_bridge_technology_unregister(&softmix_bridge);
 920 }
 921
 922 static int load_module(void)
 923 {
 924         struct ast_format tmp;
 925         if (!(softmix_bridge.format_capabilities = ast_format_cap_alloc())) {
 926                 return AST_MODULE_LOAD_DECLINE;
 927         }
 928         ast_format_cap_add(softmix_bridge.format_capabilities, ast_format_set(&tmp, AST_FORMAT_SLINEAR, 0));
 929         return ast_bridge_technology_register(&softmix_bridge);
 930 }
 931
 932 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");