bridges/bridge_softmix.c

   1 /*
   2  * Asterisk -- An open source telephony toolkit.
   3  *
   4  * Copyright (C) 2011, Digium, Inc.
   5  *
   6  * Joshua Colp <jcolp@digium.com>
   7  * David Vossel <dvossel@digium.com>
   8  *
   9  * See http://www.asterisk.org for more information about
  10  * the Asterisk project. Please do not directly contact
  11  * any of the maintainers of this project for assistance;
  12  * the project provides a web site, mailing lists and IRC
  13  * channels for your use.
  14  *
  15  * This program is free software, distributed under the terms of
  16  * the GNU General Public License Version 2. See the LICENSE file
  17  * at the top of the source tree.
  18  */
  19
  20 /*! \file
  21  *
  22  * \brief Multi-party software based channel mixing
  23  *
  24  * \author Joshua Colp <jcolp@digium.com>
  25  * \author David Vossel <dvossel@digium.com>
  26  *
  27  * \ingroup bridges
  28  */
  29
  30 /*** MODULEINFO
  31         <support_level>core</support_level>
  32  ***/
  33
  34 #include "asterisk.h"
  35
  36 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
  37
  38 #include <stdio.h>
  39 #include <stdlib.h>
  40 #include <string.h>
  41 #include <sys/time.h>
  42 #include <signal.h>
  43 #include <errno.h>
  44 #include <unistd.h>
  45
  46 #include "asterisk/module.h"
  47 #include "asterisk/channel.h"
  48 #include "asterisk/bridging.h"
  49 #include "asterisk/bridging_technology.h"
  50 #include "asterisk/frame.h"
  51 #include "asterisk/options.h"
  52 #include "asterisk/logger.h"
  53 #include "asterisk/slinfactory.h"
  54 #include "asterisk/astobj2.h"
  55 #include "asterisk/timing.h"
  56 #include "asterisk/translate.h"
  57
  58 #define MAX_DATALEN 8096
  59
  60 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
  61 #define DEFAULT_SOFTMIX_INTERVAL 20
  62
  63 /*! \brief Size of the buffer used for sample manipulation */
  64 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
  65
  66 /*! \brief Number of samples we are dealing with */
  67 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
  68
  69 /*! \brief Number of mixing iterations to perform between gathering statistics. */
  70 #define SOFTMIX_STAT_INTERVAL 100
  71
  72 /* This is the threshold in ms at which a channel's own audio will stop getting
  73  * mixed out its own write audio stream because it is not talking. */
  74 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
  75 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
  76
  77 #define DEFAULT_ENERGY_HISTORY_LEN 150
  78
  79 struct video_follow_talker_data {
  80         /*! audio energy history */
  81         int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
  82         /*! The current slot being used in the history buffer, this
  83          *  increments and wraps around */
  84         int energy_history_cur_slot;
  85         /*! The current energy sum used for averages. */
  86         int energy_accum;
  87         /*! The current energy average */
  88         int energy_average;
  89 };
  90
  91 /*! \brief Structure which contains per-channel mixing information */
  92 struct softmix_channel {
  93         /*! Lock to protect this structure */
  94         ast_mutex_t lock;
  95         /*! Factory which contains audio read in from the channel */
  96         struct ast_slinfactory factory;
  97         /*! Frame that contains mixed audio to be written out to the channel */
  98         struct ast_frame write_frame;
  99         /*! Frame that contains mixed audio read from the channel */
 100         struct ast_frame read_frame;
 101         /*! DSP for detecting silence */
 102         struct ast_dsp *dsp;
 103         /*!
 104          * \brief TRUE if a channel is talking.
 105          *
 106          * \note This affects how the channel's audio is mixed back to
 107          * it.
 108          */
 109         unsigned int talking:1;
 110         /*! TRUE if the channel provided audio for this mixing interval */
 111         unsigned int have_audio:1;
 112         /*! Buffer containing final mixed audio from all sources */
 113         short final_buf[MAX_DATALEN];
 114         /*! Buffer containing only the audio from the channel */
 115         short our_buf[MAX_DATALEN];
 116         /*! Data pertaining to talker mode for video conferencing */
 117         struct video_follow_talker_data video_talker;
 118 };
 119
 120 struct softmix_bridge_data {
 121         struct ast_timer *timer;
 122         /*! Lock for signaling the mixing thread. */
 123         ast_mutex_t lock;
 124         /*! Condition, used if we need to wake up the mixing thread. */
 125         ast_cond_t cond;
 126         /*! Thread handling the mixing */
 127         pthread_t thread;
 128         unsigned int internal_rate;
 129         unsigned int internal_mixing_interval;
 130         /*! TRUE if the mixing thread should stop */
 131         unsigned int stop:1;
 132 };
 133
 134 struct softmix_stats {
 135         /*! Each index represents a sample rate used above the internal rate. */
 136         unsigned int sample_rates[16];
 137         /*! Each index represents the number of channels using the same index in the sample_rates array.  */
 138         unsigned int num_channels[16];
 139         /*! the number of channels above the internal sample rate */
 140         unsigned int num_above_internal_rate;
 141         /*! the number of channels at the internal sample rate */
 142         unsigned int num_at_internal_rate;
 143         /*! the absolute highest sample rate supported by any channel in the bridge */
 144         unsigned int highest_supported_rate;
 145         /*! Is the sample rate locked by the bridge, if so what is that rate.*/
 146         unsigned int locked_rate;
 147 };
 148
 149 struct softmix_mixing_array {
 150         unsigned int max_num_entries;
 151         unsigned int used_entries;
 152         int16_t **buffers;
 153 };
 154
 155 struct softmix_translate_helper_entry {
 156         int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
 157                                       and re-init if it was usable. */
 158         struct ast_format dst_format; /*!< The destination format for this helper */
 159         struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
 160         struct ast_frame *out_frame; /*!< The output frame from the last translation */
 161         AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
 162 };
 163
 164 struct softmix_translate_helper {
 165         struct ast_format slin_src; /*!< the source format expected for all the translators */
 166         AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
 167 };
 168
 169 static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
 170 {
 171         struct softmix_translate_helper_entry *entry;
 172         if (!(entry = ast_calloc(1, sizeof(*entry)))) {
 173                 return NULL;
 174         }
 175         ast_format_copy(&entry->dst_format, dst);
 176         return entry;
 177 }
 178
 179 static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
 180 {
 181         if (entry->trans_pvt) {
 182                 ast_translator_free_path(entry->trans_pvt);
 183         }
 184         if (entry->out_frame) {
 185                 ast_frfree(entry->out_frame);
 186         }
 187         ast_free(entry);
 188         return NULL;
 189 }
 190
 191 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
 192 {
 193         memset(trans_helper, 0, sizeof(*trans_helper));
 194         ast_format_set(&trans_helper->slin_src, ast_format_slin_by_rate(sample_rate), 0);
 195 }
 196
 197 static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
 198 {
 199         struct softmix_translate_helper_entry *entry;
 200
 201         while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
 202                 softmix_translate_helper_free_entry(entry);
 203         }
 204 }
 205
 206 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
 207 {
 208         struct softmix_translate_helper_entry *entry;
 209
 210         ast_format_set(&trans_helper->slin_src, ast_format_slin_by_rate(sample_rate), 0);
 211         AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
 212                 if (entry->trans_pvt) {
 213                         ast_translator_free_path(entry->trans_pvt);
 214                         if (!(entry->trans_pvt = ast_translator_build_path(&entry->dst_format, &trans_helper->slin_src))) {
 215                                 AST_LIST_REMOVE_CURRENT(entry);
 216                                 entry = softmix_translate_helper_free_entry(entry);
 217                         }
 218                 }
 219         }
 220         AST_LIST_TRAVERSE_SAFE_END;
 221 }
 222
 223 /*!
 224  * \internal
 225  * \brief Get the next available audio on the softmix channel's read stream
 226  * and determine if it should be mixed out or not on the write stream.
 227  *
 228  * \retval pointer to buffer containing the exact number of samples requested on success.
 229  * \retval NULL if no samples are present
 230  */
 231 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
 232 {
 233         if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
 234                 ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
 235                 sc->have_audio = 1;
 236                 return sc->our_buf;
 237         }
 238         sc->have_audio = 0;
 239         return NULL;
 240 }
 241
 242 /*!
 243  * \internal
 244  * \brief Process a softmix channel's write audio
 245  *
 246  * \details This function will remove the channel's talking from its own audio if present and
 247  * possibly even do the channel's write translation for it depending on how many other
 248  * channels use the same write format.
 249  */
 250 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
 251         struct ast_format *raw_write_fmt,
 252         struct softmix_channel *sc)
 253 {
 254         struct softmix_translate_helper_entry *entry = NULL;
 255         int i;
 256
 257         /* If we provided audio that was not determined to be silence,
 258          * then take it out while in slinear format. */
 259         if (sc->have_audio && sc->talking) {
 260                 for (i = 0; i < sc->write_frame.samples; i++) {
 261                         ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
 262                 }
 263                 /* do not do any special write translate optimization if we had to make
 264                  * a special mix for them to remove their own audio. */
 265                 return;
 266         }
 267
 268         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
 269                 if (ast_format_cmp(&entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
 270                         entry->num_times_requested++;
 271                 } else {
 272                         continue;
 273                 }
 274                 if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
 275                         entry->trans_pvt = ast_translator_build_path(&entry->dst_format, &trans_helper->slin_src);
 276                 }
 277                 if (entry->trans_pvt && !entry->out_frame) {
 278                         entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
 279                 }
 280                 if (entry->out_frame && (entry->out_frame->datalen < MAX_DATALEN)) {
 281                         ast_format_copy(&sc->write_frame.subclass.format, &entry->out_frame->subclass.format);
 282                         memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
 283                         sc->write_frame.datalen = entry->out_frame->datalen;
 284                         sc->write_frame.samples = entry->out_frame->samples;
 285                 }
 286                 break;
 287         }
 288
 289         /* add new entry into list if this format destination was not matched. */
 290         if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
 291                 AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
 292         }
 293 }
 294
 295 static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
 296 {
 297         struct softmix_translate_helper_entry *entry;
 298
 299         AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
 300                 if (entry->out_frame) {
 301                         ast_frfree(entry->out_frame);
 302                         entry->out_frame = NULL;
 303                 }
 304                 entry->num_times_requested = 0;
 305         }
 306 }
 307
 308 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset)
 309 {
 310         struct softmix_channel *sc = bridge_channel->tech_pvt;
 311         unsigned int channel_read_rate = ast_format_rate(ast_channel_rawreadformat(bridge_channel->chan));
 312
 313         ast_mutex_lock(&sc->lock);
 314         if (reset) {
 315                 ast_slinfactory_destroy(&sc->factory);
 316                 ast_dsp_free(sc->dsp);
 317         }
 318         /* Setup read/write frame parameters */
 319         sc->write_frame.frametype = AST_FRAME_VOICE;
 320         ast_format_set(&sc->write_frame.subclass.format, ast_format_slin_by_rate(rate), 0);
 321         sc->write_frame.data.ptr = sc->final_buf;
 322         sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
 323         sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
 324
 325         sc->read_frame.frametype = AST_FRAME_VOICE;
 326         ast_format_set(&sc->read_frame.subclass.format, ast_format_slin_by_rate(channel_read_rate), 0);
 327         sc->read_frame.data.ptr = sc->our_buf;
 328         sc->read_frame.datalen = SOFTMIX_DATALEN(channel_read_rate, interval);
 329         sc->read_frame.samples = SOFTMIX_SAMPLES(channel_read_rate, interval);
 330
 331         /* Setup smoother */
 332         ast_slinfactory_init_with_format(&sc->factory, &sc->write_frame.subclass.format);
 333
 334         /* set new read and write formats on channel. */
 335         ast_set_read_format(bridge_channel->chan, &sc->read_frame.subclass.format);
 336         ast_set_write_format(bridge_channel->chan, &sc->write_frame.subclass.format);
 337
 338         /* set up new DSP.  This is on the read side only right before the read frame enters the smoother.  */
 339         sc->dsp = ast_dsp_new_with_rate(channel_read_rate);
 340         /* we want to aggressively detect silence to avoid feedback */
 341         if (bridge_channel->tech_args.talking_threshold) {
 342                 ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
 343         } else {
 344                 ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
 345         }
 346
 347         ast_mutex_unlock(&sc->lock);
 348 }
 349
 350 /*!
 351  * \internal
 352  * \brief Poke the mixing thread in case it is waiting for an active channel.
 353  * \since 12.0.0
 354  *
 355  * \param softmix_data Bridge mixing data.
 356  *
 357  * \return Nothing
 358  */
 359 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
 360 {
 361         ast_mutex_lock(&softmix_data->lock);
 362         ast_cond_signal(&softmix_data->cond);
 363         ast_mutex_unlock(&softmix_data->lock);
 364 }
 365
 366 /*! \brief Function called when a channel is unsuspended from the bridge */
 367 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
 368 {
 369         if (bridge->tech_pvt) {
 370                 softmix_poke_thread(bridge->tech_pvt);
 371         }
 372 }
 373
 374 /*!
 375  * \internal
 376  * \brief Indicate a source change to the channel.
 377  * \since 12.0.0
 378  *
 379  * \param bridge_channel Which channel source is changing.
 380  *
 381  * \return Nothing
 382  */
 383 static void softmix_src_change(struct ast_bridge_channel *bridge_channel)
 384 {
 385         ast_bridge_channel_queue_control_data(bridge_channel, AST_CONTROL_SRCCHANGE, NULL, 0);
 386 }
 387
 388 /*! \brief Function called when a channel is joined into the bridge */
 389 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
 390 {
 391         struct softmix_channel *sc;
 392         struct softmix_bridge_data *softmix_data;
 393
 394         softmix_data = bridge->tech_pvt;
 395         if (!softmix_data) {
 396                 return -1;
 397         }
 398
 399         /* Create a new softmix_channel structure and allocate various things on it */
 400         if (!(sc = ast_calloc(1, sizeof(*sc)))) {
 401                 return -1;
 402         }
 403
 404         softmix_src_change(bridge_channel);
 405
 406         /* Can't forget the lock */
 407         ast_mutex_init(&sc->lock);
 408
 409         /* Can't forget to record our pvt structure within the bridged channel structure */
 410         bridge_channel->tech_pvt = sc;
 411
 412         set_softmix_bridge_data(softmix_data->internal_rate,
 413                 softmix_data->internal_mixing_interval
 414                         ? softmix_data->internal_mixing_interval
 415                         : DEFAULT_SOFTMIX_INTERVAL,
 416                 bridge_channel, 0);
 417
 418         softmix_poke_thread(softmix_data);
 419         return 0;
 420 }
 421
 422 /*! \brief Function called when a channel leaves the bridge */
 423 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
 424 {
 425         struct softmix_channel *sc = bridge_channel->tech_pvt;
 426
 427         if (!sc) {
 428                 return;
 429         }
 430         bridge_channel->tech_pvt = NULL;
 431
 432         softmix_src_change(bridge_channel);
 433
 434         /* Drop mutex lock */
 435         ast_mutex_destroy(&sc->lock);
 436
 437         /* Drop the factory */
 438         ast_slinfactory_destroy(&sc->factory);
 439
 440         /* Drop the DSP */
 441         ast_dsp_free(sc->dsp);
 442
 443         /* Eep! drop ourselves */
 444         ast_free(sc);
 445 }
 446
 447 /*!
 448  * \internal
 449  * \brief Pass the given frame to everyone else.
 450  * \since 12.0.0
 451  *
 452  * \param bridge What bridge to distribute frame.
 453  * \param bridge_channel Channel to optionally not pass frame to. (NULL to pass to everyone)
 454  * \param frame Frame to pass.
 455  *
 456  * \return Nothing
 457  */
 458 static void softmix_pass_everyone_else(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
 459 {
 460         struct ast_bridge_channel *cur;
 461
 462         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
 463                 if (cur == bridge_channel) {
 464                         continue;
 465                 }
 466                 ast_bridge_channel_queue_frame(cur, frame);
 467         }
 468 }
 469
 470 static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
 471 {
 472         struct ast_bridge_channel *cur;
 473
 474         AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
 475                 if (cur->suspended) {
 476                         continue;
 477                 }
 478                 if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
 479                         ast_bridge_channel_queue_frame(cur, frame);
 480                         break;
 481                 }
 482         }
 483 }
 484
 485 /*!
 486  * \internal
 487  * \brief Determine what to do with a video frame.
 488  * \since 12.0.0
 489  *
 490  * \param bridge Which bridge is getting the frame
 491  * \param bridge_channel Which channel is writing the frame.
 492  * \param frame What is being written.
 493  *
 494  * \return Nothing
 495  */
 496 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
 497 {
 498         struct softmix_channel *sc;
 499         int video_src_priority;
 500
 501         /* Determine if the video frame should be distributed or not */
 502         switch (bridge->video_mode.mode) {
 503         case AST_BRIDGE_VIDEO_MODE_NONE:
 504                 break;
 505         case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
 506                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
 507                 if (video_src_priority == 1) {
 508                         /* Pass to me and everyone else. */
 509                         softmix_pass_everyone_else(bridge, NULL, frame);
 510                 }
 511                 break;
 512         case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
 513                 sc = bridge_channel->tech_pvt;
 514                 ast_mutex_lock(&sc->lock);
 515                 ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
 516                         sc->video_talker.energy_average,
 517                         ast_format_get_video_mark(&frame->subclass.format));
 518                 ast_mutex_unlock(&sc->lock);
 519                 video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
 520                 if (video_src_priority == 1) {
 521                         int num_src = ast_bridge_number_video_src(bridge);
 522                         int echo = num_src > 1 ? 0 : 1;
 523
 524                         softmix_pass_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
 525                 } else if (video_src_priority == 2) {
 526                         softmix_pass_video_top_priority(bridge, frame);
 527                 }
 528                 break;
 529         }
 530 }
 531
 532 /*!
 533  * \internal
 534  * \brief Determine what to do with a voice frame.
 535  * \since 12.0.0
 536  *
 537  * \param bridge Which bridge is getting the frame
 538  * \param bridge_channel Which channel is writing the frame.
 539  * \param frame What is being written.
 540  *
 541  * \return Nothing
 542  */
 543 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
 544 {
 545         struct softmix_channel *sc = bridge_channel->tech_pvt;
 546         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
 547         int totalsilence = 0;
 548         int cur_energy = 0;
 549         int silence_threshold = bridge_channel->tech_args.silence_threshold ?
 550                 bridge_channel->tech_args.silence_threshold :
 551                 DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
 552         char update_talking = -1;  /* if this is set to 0 or 1, tell the bridge that the channel has started or stopped talking. */
 553
 554         /* Write the frame into the conference */
 555         ast_mutex_lock(&sc->lock);
 556         ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
 557
 558         if (bridge->video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
 559                 int cur_slot = sc->video_talker.energy_history_cur_slot;
 560
 561                 sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
 562                 sc->video_talker.energy_accum += cur_energy;
 563                 sc->video_talker.energy_history[cur_slot] = cur_energy;
 564                 sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
 565                 sc->video_talker.energy_history_cur_slot++;
 566                 if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
 567                         sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
 568                 }
 569         }
 570
 571         if (totalsilence < silence_threshold) {
 572                 if (!sc->talking) {
 573                         update_talking = 1;
 574                 }
 575                 sc->talking = 1; /* tell the write process we have audio to be mixed out */
 576         } else {
 577                 if (sc->talking) {
 578                         update_talking = 0;
 579                 }
 580                 sc->talking = 0;
 581         }
 582
 583         /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
 584          * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
 585          * the audio by flushing the buffer before adding new audio in. */
 586         if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
 587                 ast_slinfactory_flush(&sc->factory);
 588         }
 589
 590         /* If a frame was provided add it to the smoother, unless drop silence is enabled and this frame
 591          * is not determined to be talking. */
 592         if (!(bridge_channel->tech_args.drop_silence && !sc->talking) &&
 593                 (frame->frametype == AST_FRAME_VOICE && ast_format_is_slinear(&frame->subclass.format))) {
 594                 ast_slinfactory_feed(&sc->factory, frame);
 595         }
 596
 597         /* Alllll done */
 598         ast_mutex_unlock(&sc->lock);
 599
 600         if (update_talking != -1) {
 601                 ast_bridge_notify_talking(bridge_channel, update_talking);
 602         }
 603 }
 604
 605 /*!
 606  * \internal
 607  * \brief Determine what to do with a control frame.
 608  * \since 12.0.0
 609  *
 610  * \param bridge Which bridge is getting the frame
 611  * \param bridge_channel Which channel is writing the frame.
 612  * \param frame What is being written.
 613  *
 614  * \return Nothing
 615  */
 616 static void softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
 617 {
 618 /* BUGBUG need to look at channel roles to determine what to do with control frame. */
 619         /*! \todo BUGBUG softmix_bridge_write_control() not written */
 620 }
 621
 622 /*!
 623  * \internal
 624  * \brief Determine what to do with a frame written into the bridge.
 625  * \since 12.0.0
 626  *
 627  * \param bridge Which bridge is getting the frame
 628  * \param bridge_channel Which channel is writing the frame.
 629  * \param frame What is being written.
 630  *
 631  * \retval 0 on success
 632  * \retval -1 on failure
 633  *
 634  * \note On entry, bridge is already locked.
 635  */
 636 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
 637 {
 638         int res = 0;
 639
 640         if (!bridge->tech_pvt || !bridge_channel->tech_pvt) {
 641                 return -1;
 642         }
 643
 644         switch (frame->frametype) {
 645         case AST_FRAME_DTMF_BEGIN:
 646         case AST_FRAME_DTMF_END:
 647                 softmix_pass_everyone_else(bridge, bridge_channel, frame);
 648                 break;
 649         case AST_FRAME_VOICE:
 650                 softmix_bridge_write_voice(bridge, bridge_channel, frame);
 651                 break;
 652         case AST_FRAME_VIDEO:
 653                 softmix_bridge_write_video(bridge, bridge_channel, frame);
 654                 break;
 655         case AST_FRAME_CONTROL:
 656                 softmix_bridge_write_control(bridge, bridge_channel, frame);
 657                 break;
 658         case AST_FRAME_BRIDGE_ACTION:
 659                 softmix_pass_everyone_else(bridge, bridge_channel, frame);
 660                 break;
 661         default:
 662                 ast_debug(3, "Frame type %d unsupported\n", frame->frametype);
 663                 res = -1;
 664                 break;
 665         }
 666
 667         return res;
 668 }
 669
 670 static void gather_softmix_stats(struct softmix_stats *stats,
 671         const struct softmix_bridge_data *softmix_data,
 672         struct ast_bridge_channel *bridge_channel)
 673 {
 674         int channel_native_rate;
 675         int i;
 676         /* Gather stats about channel sample rates. */
 677         channel_native_rate = MAX(ast_format_rate(ast_channel_rawwriteformat(bridge_channel->chan)),
 678                 ast_format_rate(ast_channel_rawreadformat(bridge_channel->chan)));
 679
 680         if (channel_native_rate > stats->highest_supported_rate) {
 681                 stats->highest_supported_rate = channel_native_rate;
 682         }
 683         if (channel_native_rate > softmix_data->internal_rate) {
 684                 for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
 685                         if (stats->sample_rates[i] == channel_native_rate) {
 686                                 stats->num_channels[i]++;
 687                                 break;
 688                         } else if (!stats->sample_rates[i]) {
 689                                 stats->sample_rates[i] = channel_native_rate;
 690                                 stats->num_channels[i]++;
 691                                 break;
 692                         }
 693                 }
 694                 stats->num_above_internal_rate++;
 695         } else if (channel_native_rate == softmix_data->internal_rate) {
 696                 stats->num_at_internal_rate++;
 697         }
 698 }
 699 /*!
 700  * \internal
 701  * \brief Analyse mixing statistics and change bridges internal rate
 702  * if necessary.
 703  *
 704  * \retval 0, no changes to internal rate
 705  * \ratval 1, internal rate was changed, update all the channels on the next mixing iteration.
 706  */
 707 static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data)
 708 {
 709         int i;
 710         /* Re-adjust the internal bridge sample rate if
 711          * 1. The bridge's internal sample rate is locked in at a sample
 712          *    rate other than the current sample rate being used.
 713          * 2. two or more channels support a higher sample rate
 714          * 3. no channels support the current sample rate or a higher rate
 715          */
 716         if (stats->locked_rate) {
 717                 /* if the rate is locked by the bridge, only update it if it differs
 718                  * from the current rate we are using. */
 719                 if (softmix_data->internal_rate != stats->locked_rate) {
 720                         softmix_data->internal_rate = stats->locked_rate;
 721                         ast_debug(1, "Bridge is locked in at sample rate %d\n",
 722                                 softmix_data->internal_rate);
 723                         return 1;
 724                 }
 725         } else if (stats->num_above_internal_rate >= 2) {
 726                 /* the highest rate is just used as a starting point */
 727                 unsigned int best_rate = stats->highest_supported_rate;
 728                 int best_index = -1;
 729
 730                 for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
 731                         if (stats->num_channels[i]) {
 732                                 break;
 733                         }
 734                         /* best_rate starts out being the first sample rate
 735                          * greater than the internal sample rate that 2 or
 736                          * more channels support. */
 737                         if (stats->num_channels[i] >= 2 && (best_index == -1)) {
 738                                 best_rate = stats->sample_rates[i];
 739                                 best_index = i;
 740                         /* If it has been detected that multiple rates above
 741                          * the internal rate are present, compare those rates
 742                          * to each other and pick the highest one two or more
 743                          * channels support. */
 744                         } else if (((best_index != -1) &&
 745                                 (stats->num_channels[i] >= 2) &&
 746                                 (stats->sample_rates[best_index] < stats->sample_rates[i]))) {
 747                                 best_rate = stats->sample_rates[i];
 748                                 best_index = i;
 749                         /* It is possible that multiple channels exist with native sample
 750                          * rates above the internal sample rate, but none of those channels
 751                          * have the same rate in common.  In this case, the lowest sample
 752                          * rate among those channels is picked. Over time as additional
 753                          * statistic runs are made the internal sample rate number will
 754                          * adjust to the most optimal sample rate, but it may take multiple
 755                          * iterations. */
 756                         } else if (best_index == -1) {
 757                                 best_rate = MIN(best_rate, stats->sample_rates[i]);
 758                         }
 759                 }
 760
 761                 ast_debug(1, "Bridge changed from %d To %d\n",
 762                         softmix_data->internal_rate, best_rate);
 763                 softmix_data->internal_rate = best_rate;
 764                 return 1;
 765         } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
 766                 /* In this case, the highest supported rate is actually lower than the internal rate */
 767                 softmix_data->internal_rate = stats->highest_supported_rate;
 768                 ast_debug(1, "Bridge changed from %d to %d\n",
 769                         softmix_data->internal_rate, stats->highest_supported_rate);
 770                 return 1;
 771         }
 772         return 0;
 773 }
 774
 775 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries)
 776 {
 777         memset(mixing_array, 0, sizeof(*mixing_array));
 778         mixing_array->max_num_entries = starting_num_entries;
 779         if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
 780                 ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
 781                 return -1;
 782         }
 783         return 0;
 784 }
 785
 786 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array)
 787 {
 788         ast_free(mixing_array->buffers);
 789 }
 790
 791 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries)
 792 {
 793         int16_t **tmp;
 794         /* give it some room to grow since memory is cheap but allocations can be expensive */
 795         mixing_array->max_num_entries = num_entries;
 796         if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
 797                 ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
 798                 return -1;
 799         }
 800         mixing_array->buffers = tmp;
 801         return 0;
 802 }
 803
 804 /*!
 805  * \brief Mixing loop.
 806  *
 807  * \retval 0 on success
 808  * \retval -1 on failure
 809  */
 810 static int softmix_mixing_loop(struct ast_bridge *bridge)
 811 {
 812         struct softmix_stats stats = { { 0 }, };
 813         struct softmix_mixing_array mixing_array;
 814         struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
 815         struct ast_timer *timer;
 816         struct softmix_translate_helper trans_helper;
 817         int16_t buf[MAX_DATALEN];
 818         unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
 819         int timingfd;
 820         int update_all_rates = 0; /* set this when the internal sample rate has changed */
 821         unsigned int idx;
 822         unsigned int x;
 823         int res = -1;
 824
 825         timer = softmix_data->timer;
 826         timingfd = ast_timer_fd(timer);
 827         softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
 828         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
 829
 830         /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
 831         if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10)) {
 832                 goto softmix_cleanup;
 833         }
 834
 835         while (!softmix_data->stop && bridge->num_active) {
 836                 struct ast_bridge_channel *bridge_channel;
 837                 int timeout = -1;
 838                 enum ast_format_id cur_slin_id = ast_format_slin_by_rate(softmix_data->internal_rate);
 839                 unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
 840                 unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
 841
 842                 if (softmix_datalen > MAX_DATALEN) {
 843                         /* This should NEVER happen, but if it does we need to know about it. Almost
 844                          * all the memcpys used during this process depend on this assumption.  Rather
 845                          * than checking this over and over again through out the code, this single
 846                          * verification is done on each iteration. */
 847                         ast_log(LOG_WARNING,
 848                                 "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
 849                                 bridge->uniqueid);
 850                         goto softmix_cleanup;
 851                 }
 852
 853                 /* Grow the mixing array buffer as participants are added. */
 854                 if (mixing_array.max_num_entries < bridge->num_channels
 855                         && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5)) {
 856                         goto softmix_cleanup;
 857                 }
 858
 859                 /* init the number of buffers stored in the mixing array to 0.
 860                  * As buffers are added for mixing, this number is incremented. */
 861                 mixing_array.used_entries = 0;
 862
 863                 /* These variables help determine if a rate change is required */
 864                 if (!stat_iteration_counter) {
 865                         memset(&stats, 0, sizeof(stats));
 866                         stats.locked_rate = bridge->internal_sample_rate;
 867                 }
 868
 869                 /* If the sample rate has changed, update the translator helper */
 870                 if (update_all_rates) {
 871                         softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
 872                 }
 873
 874                 /* Go through pulling audio from each factory that has it available */
 875                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
 876                         struct softmix_channel *sc = bridge_channel->tech_pvt;
 877
 878                         /* Update the sample rate to match the bridge's native sample rate if necessary. */
 879                         if (update_all_rates) {
 880                                 set_softmix_bridge_data(softmix_data->internal_rate, softmix_data->internal_mixing_interval, bridge_channel, 1);
 881                         }
 882
 883                         /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
 884                         if (!stat_iteration_counter) {
 885                                 gather_softmix_stats(&stats, softmix_data, bridge_channel);
 886                         }
 887
 888                         /* if the channel is suspended, don't check for audio, but still gather stats */
 889                         if (bridge_channel->suspended) {
 890                                 continue;
 891                         }
 892
 893                         /* Try to get audio from the factory if available */
 894                         ast_mutex_lock(&sc->lock);
 895                         if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
 896                                 mixing_array.used_entries++;
 897                         }
 898                         ast_mutex_unlock(&sc->lock);
 899                 }
 900
 901                 /* mix it like crazy */
 902                 memset(buf, 0, softmix_datalen);
 903                 for (idx = 0; idx < mixing_array.used_entries; ++idx) {
 904                         for (x = 0; x < softmix_samples; ++x) {
 905                                 ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
 906                         }
 907                 }
 908
 909                 /* Next step go through removing the channel's own audio and creating a good frame... */
 910                 AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
 911                         struct softmix_channel *sc = bridge_channel->tech_pvt;
 912
 913                         if (bridge_channel->suspended) {
 914                                 continue;
 915                         }
 916
 917                         ast_mutex_lock(&sc->lock);
 918
 919                         /* Make SLINEAR write frame from local buffer */
 920                         if (sc->write_frame.subclass.format.id != cur_slin_id) {
 921                                 ast_format_set(&sc->write_frame.subclass.format, cur_slin_id, 0);
 922                         }
 923                         sc->write_frame.datalen = softmix_datalen;
 924                         sc->write_frame.samples = softmix_samples;
 925                         memcpy(sc->final_buf, buf, softmix_datalen);
 926
 927                         /* process the softmix channel's new write audio */
 928                         softmix_process_write_audio(&trans_helper, ast_channel_rawwriteformat(bridge_channel->chan), sc);
 929
 930                         ast_mutex_unlock(&sc->lock);
 931
 932                         /* A frame is now ready for the channel. */
 933                         ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
 934                 }
 935
 936                 update_all_rates = 0;
 937                 if (!stat_iteration_counter) {
 938                         update_all_rates = analyse_softmix_stats(&stats, softmix_data);
 939                         stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
 940                 }
 941                 stat_iteration_counter--;
 942
 943                 ast_bridge_unlock(bridge);
 944                 /* cleanup any translation frame data from the previous mixing iteration. */
 945                 softmix_translate_helper_cleanup(&trans_helper);
 946                 /* Wait for the timing source to tell us to wake up and get things done */
 947                 ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
 948                 if (ast_timer_ack(timer, 1) < 0) {
 949                         ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
 950                                 bridge->uniqueid);
 951                         ast_bridge_lock(bridge);
 952                         goto softmix_cleanup;
 953                 }
 954                 ast_bridge_lock(bridge);
 955
 956                 /* make sure to detect mixing interval changes if they occur. */
 957                 if (bridge->internal_mixing_interval && (bridge->internal_mixing_interval != softmix_data->internal_mixing_interval)) {
 958                         softmix_data->internal_mixing_interval = bridge->internal_mixing_interval;
 959                         ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
 960                         update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
 961                 }
 962         }
 963
 964         res = 0;
 965
 966 softmix_cleanup:
 967         softmix_translate_helper_destroy(&trans_helper);
 968         softmix_mixing_array_destroy(&mixing_array);
 969         return res;
 970 }
 971
 972 /*!
 973  * \internal
 974  * \brief Mixing thread.
 975  * \since 12.0.0
 976  *
 977  * \note The thread does not have its own reference to the
 978  * bridge.  The lifetime of the thread is tied to the lifetime
 979  * of the mixing technology association with the bridge.
 980  */
 981 static void *softmix_mixing_thread(void *data)
 982 {
 983         struct ast_bridge *bridge = data;
 984         struct softmix_bridge_data *softmix_data;
 985
 986         ast_bridge_lock(bridge);
 987         if (bridge->callid) {
 988                 ast_callid_threadassoc_add(bridge->callid);
 989         }
 990
 991         ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
 992
 993         softmix_data = bridge->tech_pvt;
 994         while (!softmix_data->stop) {
 995                 if (!bridge->num_active) {
 996                         /* Wait for something to happen to the bridge. */
 997                         ast_bridge_unlock(bridge);
 998                         ast_mutex_lock(&softmix_data->lock);
 999                         if (!softmix_data->stop) {
1000                                 ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
1001                         }
1002                         ast_mutex_unlock(&softmix_data->lock);
1003                         ast_bridge_lock(bridge);
1004                         continue;
1005                 }
1006
1007                 if (softmix_mixing_loop(bridge)) {
1008                         /*
1009                          * A mixing error occurred.  Sleep and try again later so we
1010                          * won't flood the logs.
1011                          */
1012                         ast_bridge_unlock(bridge);
1013                         sleep(1);
1014                         ast_bridge_lock(bridge);
1015                 }
1016         }
1017
1018         ast_bridge_unlock(bridge);
1019
1020         ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
1021
1022         return NULL;
1023 }
1024
1025 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
1026 {
1027         if (softmix_data->timer) {
1028                 ast_timer_close(softmix_data->timer);
1029                 softmix_data->timer = NULL;
1030         }
1031         ast_mutex_destroy(&softmix_data->lock);
1032         ast_free(softmix_data);
1033 }
1034
1035 /*! \brief Function called when a bridge is created */
1036 static int softmix_bridge_create(struct ast_bridge *bridge)
1037 {
1038         struct softmix_bridge_data *softmix_data;
1039
1040         softmix_data = ast_calloc(1, sizeof(*softmix_data));
1041         if (!softmix_data) {
1042                 return -1;
1043         }
1044         ast_mutex_init(&softmix_data->lock);
1045         softmix_data->timer = ast_timer_open();
1046         if (!softmix_data->timer) {
1047                 softmix_bridge_data_destroy(softmix_data);
1048                 return -1;
1049         }
1050         /* start at 8khz, let it grow from there */
1051         softmix_data->internal_rate = 8000;
1052         softmix_data->internal_mixing_interval = DEFAULT_SOFTMIX_INTERVAL;
1053
1054         bridge->tech_pvt = softmix_data;
1055
1056         /* Start the mixing thread. */
1057         if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread, bridge)) {
1058                 softmix_data->thread = AST_PTHREADT_NULL;
1059                 softmix_bridge_data_destroy(softmix_data);
1060                 bridge->tech_pvt = NULL;
1061                 return -1;
1062         }
1063
1064         return 0;
1065 }
1066
1067 /*! \brief Function called when a bridge is destroyed */
1068 static void softmix_bridge_destroy(struct ast_bridge *bridge)
1069 {
1070         struct softmix_bridge_data *softmix_data;
1071         pthread_t thread;
1072
1073         softmix_data = bridge->tech_pvt;
1074         if (!softmix_data) {
1075                 return;
1076         }
1077
1078         /* Stop the mixing thread. */
1079         ast_mutex_lock(&softmix_data->lock);
1080         softmix_data->stop = 1;
1081         ast_cond_signal(&softmix_data->cond);
1082         thread = softmix_data->thread;
1083         softmix_data->thread = AST_PTHREADT_NULL;
1084         ast_mutex_unlock(&softmix_data->lock);
1085         if (thread != AST_PTHREADT_NULL) {
1086                 ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
1087                 pthread_join(thread, NULL);
1088         }
1089
1090         softmix_bridge_data_destroy(softmix_data);
1091         bridge->tech_pvt = NULL;
1092 }
1093
1094 static struct ast_bridge_technology softmix_bridge = {
1095         .name = "softmix",
1096         .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
1097         .preference = AST_BRIDGE_PREFERENCE_BASE_MULTIMIX,
1098         .create = softmix_bridge_create,
1099         .destroy = softmix_bridge_destroy,
1100         .join = softmix_bridge_join,
1101         .leave = softmix_bridge_leave,
1102         .unsuspend = softmix_bridge_unsuspend,
1103         .write = softmix_bridge_write,
1104 };
1105
1106 static int unload_module(void)
1107 {
1108         ast_format_cap_destroy(softmix_bridge.format_capabilities);
1109         return ast_bridge_technology_unregister(&softmix_bridge);
1110 }
1111
1112 static int load_module(void)
1113 {
1114         struct ast_format tmp;
1115         if (!(softmix_bridge.format_capabilities = ast_format_cap_alloc())) {
1116                 return AST_MODULE_LOAD_DECLINE;
1117         }
1118         ast_format_cap_add(softmix_bridge.format_capabilities, ast_format_set(&tmp, AST_FORMAT_SLINEAR, 0));
1119         return ast_bridge_technology_register(&softmix_bridge);
1120 }
1121
1122 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");