app_dial: Allow macro/gosub pre-bridge execution to occur on priorities
[asterisk/asterisk.git] / main / utils.c
index eff270f..3d8e4c2 100644 (file)
  * Please consult the CODING GUIDELINES for more information.
  */
 
+/*** MODULEINFO
+       <support_level>core</support_level>
+ ***/
+
 #include "asterisk.h"
 
 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
 
 #include <ctype.h>
-#include <sys/stat.h>
-
-#ifdef HAVE_DEV_URANDOM
 #include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#if defined(__APPLE__)
+#include <mach/mach.h>
+#elif defined(HAVE_SYS_THR_H)
+#include <sys/thr.h>
 #endif
 
 #include "asterisk/network.h"
+#include "asterisk/ast_version.h"
 
 #define AST_API_MODULE         /* ensure that inlinable API functions will be built in lock.h if required */
 #include "asterisk/lock.h"
@@ -76,12 +85,12 @@ AST_MUTEX_DEFINE_STATIC(__mutex);
 
 /*! \brief Reentrant replacement for gethostbyname for BSD-based systems.
 \note This
-routine is derived from code originally written and placed in the public 
+routine is derived from code originally written and placed in the public
 domain by Enzo Michelangeli <em@em.no-ip.com> */
 
 static int gethostbyname_r (const char *name, struct hostent *ret, char *buf,
-                               size_t buflen, struct hostent **result, 
-                               int *h_errnop) 
+                               size_t buflen, struct hostent **result,
+                               int *h_errnop)
 {
        int hsave;
        struct hostent *ph;
@@ -176,7 +185,7 @@ static int gethostbyname_r (const char *name, struct hostent *ret, char *buf,
 
 #endif
 
-/*! \brief Re-entrant (thread safe) version of gethostbyname that replaces the 
+/*! \brief Re-entrant (thread safe) version of gethostbyname that replaces the
    standard gethostbyname (which is not thread safe)
 */
 struct hostent *ast_gethostbyname(const char *host, struct ast_hostent *hp)
@@ -207,10 +216,12 @@ struct hostent *ast_gethostbyname(const char *host, struct ast_hostent *hp)
                hp->hp.h_addrtype = AF_INET;
                hp->hp.h_addr_list = (void *) hp->buf;
                hp->hp.h_addr = hp->buf + sizeof(void *);
+               /* For AF_INET, this will always be 4 */
+               hp->hp.h_length = 4;
                if (inet_pton(AF_INET, host, hp->hp.h_addr) > 0)
                        return &hp->hp;
                return NULL;
-               
+
        }
 #ifdef HAVE_GETHOSTBYNAME_R_5
        result = gethostbyname_r(host, &hp->hp, hp->buf, sizeof(hp->buf), &herrno);
@@ -226,70 +237,8 @@ struct hostent *ast_gethostbyname(const char *host, struct ast_hostent *hp)
        return &hp->hp;
 }
 
-
-
-AST_MUTEX_DEFINE_STATIC(test_lock);
-AST_MUTEX_DEFINE_STATIC(test_lock2);
-static pthread_t test_thread; 
-static int lock_count = 0;
-static int test_errors = 0;
-
-/*! \brief This is a regression test for recursive mutexes.
-   test_for_thread_safety() will return 0 if recursive mutex locks are
-   working properly, and non-zero if they are not working properly. */
-static void *test_thread_body(void *data) 
-{ 
-       ast_mutex_lock(&test_lock);
-       lock_count += 10;
-       if (lock_count != 10) 
-               test_errors++;
-       ast_mutex_lock(&test_lock);
-       lock_count += 10;
-       if (lock_count != 20) 
-               test_errors++;
-       ast_mutex_lock(&test_lock2);
-       ast_mutex_unlock(&test_lock);
-       lock_count -= 10;
-       if (lock_count != 10) 
-               test_errors++;
-       ast_mutex_unlock(&test_lock);
-       lock_count -= 10;
-       ast_mutex_unlock(&test_lock2);
-       if (lock_count != 0) 
-               test_errors++;
-       return NULL;
-} 
-
-int test_for_thread_safety(void)
-{ 
-       ast_mutex_lock(&test_lock2);
-       ast_mutex_lock(&test_lock);
-       lock_count += 1;
-       ast_mutex_lock(&test_lock);
-       lock_count += 1;
-       ast_pthread_create(&test_thread, NULL, test_thread_body, NULL); 
-       usleep(100);
-       if (lock_count != 2) 
-               test_errors++;
-       ast_mutex_unlock(&test_lock);
-       lock_count -= 1;
-       usleep(100); 
-       if (lock_count != 1) 
-               test_errors++;
-       ast_mutex_unlock(&test_lock);
-       lock_count -= 1;
-       if (lock_count != 0) 
-               test_errors++;
-       ast_mutex_unlock(&test_lock2);
-       usleep(100);
-       if (lock_count != 0) 
-               test_errors++;
-       pthread_join(test_thread, NULL);
-       return(test_errors);          /* return 0 on success. */
-}
-
 /*! \brief Produce 32 char MD5 hash of value. */
-void ast_md5_hash(char *output, char *input)
+void ast_md5_hash(char *output, const char *input)
 {
        struct MD5Context md5;
        unsigned char digest[16];
@@ -297,7 +246,7 @@ void ast_md5_hash(char *output, char *input)
        int x;
 
        MD5Init(&md5);
-       MD5Update(&md5, (unsigned char *)input, strlen(input));
+       MD5Update(&md5, (const unsigned char *) input, strlen(input));
        MD5Final(digest, &md5);
        ptr = output;
        for (x = 0; x < 16; x++)
@@ -305,7 +254,7 @@ void ast_md5_hash(char *output, char *input)
 }
 
 /*! \brief Produce 40 char SHA1 hash of value. */
-void ast_sha1_hash(char *output, char *input)
+void ast_sha1_hash(char *output, const char *input)
 {
        struct SHA1Context sha;
        char *ptr;
@@ -313,7 +262,7 @@ void ast_sha1_hash(char *output, char *input)
        uint8_t Message_Digest[20];
 
        SHA1Reset(&sha);
-       
+
        SHA1Input(&sha, (const unsigned char *) input, strlen(input));
 
        SHA1Result(&sha, Message_Digest);
@@ -322,6 +271,18 @@ void ast_sha1_hash(char *output, char *input)
                ptr += sprintf(ptr, "%2.2x", Message_Digest[x]);
 }
 
+/*! \brief Produce a 20 byte SHA1 hash of value. */
+void ast_sha1_hash_uint(uint8_t *digest, const char *input)
+{
+        struct SHA1Context sha;
+
+        SHA1Reset(&sha);
+
+        SHA1Input(&sha, (const unsigned char *) input, strlen(input));
+
+        SHA1Result(&sha, digest);
+}
+
 /*! \brief decode BASE64 encoded text */
 int ast_base64decode(unsigned char *dst, const char *src, int max)
 {
@@ -329,14 +290,14 @@ int ast_base64decode(unsigned char *dst, const char *src, int max)
        unsigned int byte = 0;
        unsigned int bits = 0;
        int incnt = 0;
-       while (*src && (cnt < max)) {
+       while(*src && *src != '=' && (cnt < max)) {
                /* Shift in 6 bits of input */
                byte <<= 6;
                byte |= (b2a[(int)(*src)]) & 0x3f;
                bits += 6;
                src++;
                incnt++;
-               /* If we have at least 8 bits left over, take that character 
+               /* If we have at least 8 bits left over, take that character
                   off the top */
                if (bits >= 8)  {
                        bits -= 8;
@@ -345,7 +306,7 @@ int ast_base64decode(unsigned char *dst, const char *src, int max)
                        cnt++;
                }
        }
-       /* Dont worry about left over bits, they're extra anyway */
+       /* Don't worry about left over bits, they're extra anyway */
        return cnt;
 }
 
@@ -381,7 +342,7 @@ int ast_base64encode_full(char *dst, const unsigned char *src, int srclen, int m
                }
        }
        if (bits && (cnt + 4 <= max)) {
-               /* Add one last character for the remaining bits, 
+               /* Add one last character for the remaining bits,
                   padding the rest with 0 */
                byte <<= 24 - bits;
                *dst++ = base64[(byte >> 18) & 0x3f];
@@ -430,56 +391,59 @@ static void base64_init(void)
        b2a[(int)'/'] = 63;
 }
 
-/*! \brief  ast_uri_encode: Turn text string to URI-encoded %XX version
-\note  At this point, we're converting from ISO-8859-x (8-bit), not UTF8
-       as in the SIP protocol spec 
-       If doreserved == 1 we will convert reserved characters also.
-       RFC 2396, section 2.4
-       outbuf needs to have more memory allocated than the instring
-       to have room for the expansion. Every char that is converted
-       is replaced by three ASCII characters.
+const struct ast_flags ast_uri_http = {AST_URI_UNRESERVED};
+const struct ast_flags ast_uri_http_legacy = {AST_URI_LEGACY_SPACE | AST_URI_UNRESERVED};
+const struct ast_flags ast_uri_sip_user = {AST_URI_UNRESERVED | AST_URI_SIP_USER_UNRESERVED};
 
-       Note: The doreserved option is needed for replaces header in
-       SIP transfers.
-*/
-char *ast_uri_encode(const char *string, char *outbuf, int buflen, int doreserved) 
+char *ast_uri_encode(const char *string, char *outbuf, int buflen, struct ast_flags spec)
 {
-       char *reserved = ";/?:@&=+$,# ";        /* Reserved chars */
-
-       const char *ptr  = string;      /* Start with the string */
-       char *out = NULL;
-       char *buf = NULL;
-
-       ast_copy_string(outbuf, string, buflen);
-
-       /* If there's no characters to convert, just go through and don't do anything */
-       while (*ptr) {
-               if ((*ptr < 32 || (unsigned char) *ptr) > 127 || (doreserved && strchr(reserved, *ptr)) ) {
-                       /* Oops, we need to start working here */
-                       if (!buf) {
-                               buf = outbuf;
-                               out = buf + (ptr - string) ;    /* Set output ptr */
+       const char *ptr  = string;      /* Start with the string */
+       char *out = outbuf;
+       const char *mark = "-_.!~*'()"; /* no encode set, RFC 2396 section 2.3, RFC 3261 sec 25 */
+       const char *user_unreserved = "&=+$,;?/"; /* user-unreserved set, RFC 3261 sec 25 */
+
+       while (*ptr && out - outbuf < buflen - 1) {
+               if (ast_test_flag(&spec, AST_URI_LEGACY_SPACE) && *ptr == ' ') {
+                       /* for legacy encoding, encode spaces as '+' */
+                       *out = '+';
+                       out++;
+               } else if (!(ast_test_flag(&spec, AST_URI_MARK)
+                               && strchr(mark, *ptr))
+                       && !(ast_test_flag(&spec, AST_URI_ALPHANUM)
+                               && ((*ptr >= '0' && *ptr <= '9')
+                               || (*ptr >= 'A' && *ptr <= 'Z')
+                               || (*ptr >= 'a' && *ptr <= 'z')))
+                       && !(ast_test_flag(&spec, AST_URI_SIP_USER_UNRESERVED)
+                               && strchr(user_unreserved, *ptr))) {
+
+                       if (out - outbuf >= buflen - 3) {
+                               break;
                        }
-                       out += sprintf(out, "%%%02x", (unsigned char) *ptr);
-               } else if (buf) {
+                       out += sprintf(out, "%%%02X", (unsigned char) *ptr);
+               } else {
                        *out = *ptr;    /* Continue copying the string */
                        out++;
-               } 
+               }
                ptr++;
        }
-       if (buf)
+
+       if (buflen) {
                *out = '\0';
+       }
+
        return outbuf;
 }
 
-/*! \brief  ast_uri_decode: Decode SIP URI, URN, URL (overwrite the string)  */
-void ast_uri_decode(char *s) 
+void ast_uri_decode(char *s, struct ast_flags spec)
 {
        char *o;
        unsigned int tmp;
 
        for (o = s; *s; s++, o++) {
-               if (*s == '%' && strlen(s) > 2 && sscanf(s + 1, "%2x", &tmp) == 1) {
+               if (ast_test_flag(&spec, AST_URI_LEGACY_SPACE) && *s == '+') {
+                       /* legacy mode, decode '+' as space */
+                       *o = ' ';
+               } else if (*s == '%' && s[1] != '\0' && s[2] != '\0' && sscanf(s + 1, "%2x", &tmp) == 1) {
                        /* have '%', two chars and correct parsing */
                        *o = tmp;
                        s += 2; /* Will be incremented once more when we break out */
@@ -489,6 +453,98 @@ void ast_uri_decode(char *s)
        *o = '\0';
 }
 
+char *ast_escape_quoted(const char *string, char *outbuf, int buflen)
+{
+       const char *ptr  = string;
+       char *out = outbuf;
+       char *allow = "\t\v !"; /* allow LWS (minus \r and \n) and "!" */
+
+       while (*ptr && out - outbuf < buflen - 1) {
+               if (!(strchr(allow, *ptr))
+                       && !(*ptr >= '#' && *ptr <= '[') /* %x23 - %x5b */
+                       && !(*ptr >= ']' && *ptr <= '~') /* %x5d - %x7e */
+                       && !((unsigned char) *ptr > 0x7f)) {             /* UTF8-nonascii */
+
+                       if (out - outbuf >= buflen - 2) {
+                               break;
+                       }
+                       out += sprintf(out, "\\%c", (unsigned char) *ptr);
+               } else {
+                       *out = *ptr;
+                       out++;
+               }
+               ptr++;
+       }
+
+       if (buflen) {
+               *out = '\0';
+       }
+
+       return outbuf;
+}
+int ast_xml_escape(const char *string, char * const outbuf, const size_t buflen)
+{
+       char *dst = outbuf;
+       char *end = outbuf + buflen - 1; /* save one for the null terminator */
+
+       /* Handle the case for the empty output buffer */
+       if (buflen == 0) {
+               return -1;
+       }
+
+       /* Escaping rules from http://www.w3.org/TR/REC-xml/#syntax */
+       /* This also prevents partial entities at the end of a string */
+       while (*string && dst < end) {
+               const char *entity = NULL;
+               int len = 0;
+
+               switch (*string) {
+               case '<':
+                       entity = "&lt;";
+                       len = 4;
+                       break;
+               case '&':
+                       entity = "&amp;";
+                       len = 5;
+                       break;
+               case '>':
+                       /* necessary if ]]> is in the string; easier to escape them all */
+                       entity = "&gt;";
+                       len = 4;
+                       break;
+               case '\'':
+                       /* necessary in single-quoted strings; easier to escape them all */
+                       entity = "&apos;";
+                       len = 6;
+                       break;
+               case '"':
+                       /* necessary in double-quoted strings; easier to escape them all */
+                       entity = "&quot;";
+                       len = 6;
+                       break;
+               default:
+                       *dst++ = *string++;
+                       break;
+               }
+
+               if (entity) {
+                       ast_assert(len == strlen(entity));
+                       if (end - dst < len) {
+                               /* no room for the entity; stop */
+                               break;
+                       }
+                       /* just checked for length; strcpy is fine */
+                       strcpy(dst, entity);
+                       dst += len;
+                       ++string;
+               }
+       }
+       /* Write null terminator */
+       *dst = '\0';
+       /* If any chars are left in string, return failure */
+       return *string == '\0' ? 0 : -1;
+}
+
 /*! \brief  ast_inet_ntoa: Recursive thread safe replacement of inet_ntoa */
 const char *ast_inet_ntoa(struct in_addr ia)
 {
@@ -500,9 +556,7 @@ const char *ast_inet_ntoa(struct in_addr ia)
        return inet_ntop(AF_INET, &ia, buf, INET_ADDRSTRLEN);
 }
 
-#ifdef HAVE_DEV_URANDOM
 static int dev_urandom_fd;
-#endif
 
 #ifndef __linux__
 #undef pthread_create /* For ast_pthread_create function only */
@@ -522,8 +576,8 @@ static int dev_urandom_fd;
 #undef pthread_mutex_init
 #undef pthread_mutex_destroy
 
-/*! 
- * \brief Keep track of which locks a thread holds 
+/*!
+ * \brief Keep track of which locks a thread holds
  *
  * There is an instance of this struct for every active thread
  */
@@ -535,14 +589,16 @@ struct thr_lock_info {
        /*! This is the actual container of info for what locks this thread holds */
        struct {
                const char *file;
-               int line_num;
                const char *func;
                const char *lock_name;
                void *lock_addr;
                int times_locked;
+               int line_num;
                enum ast_lock_type type;
                /*! This thread is waiting on this lock */
                int pending:2;
+               /*! A condition has suspended this lock */
+               int suspended:1;
 #ifdef HAVE_BKTR
                struct ast_bt *backtrace;
 #endif
@@ -551,18 +607,20 @@ struct thr_lock_info {
         *  The index (num_locks - 1) has the info on the last one in the
         *  locks member */
        unsigned int num_locks;
-       /*! Protects the contents of the locks member 
+       /*! The LWP id (which GDB prints) */
+       int lwp;
+       /*! Protects the contents of the locks member
         * Intentionally not ast_mutex_t */
        pthread_mutex_t lock;
        AST_LIST_ENTRY(thr_lock_info) entry;
 };
 
-/*! 
- * \brief Locked when accessing the lock_infos list 
+/*!
+ * \brief Locked when accessing the lock_infos list
  */
 AST_MUTEX_DEFINE_STATIC(lock_infos_lock);
 /*!
- * \brief A list of each thread's lock info 
+ * \brief A list of each thread's lock info
  */
 static AST_LIST_HEAD_NOLOCK_STATIC(lock_infos, thr_lock_info);
 
@@ -582,8 +640,14 @@ static void lock_info_destroy(void *data)
 
 
        for (i = 0; i < lock_info->num_locks; i++) {
-               ast_log(LOG_ERROR, 
-                       "Thread '%s' still has a lock! - '%s' (%p) from '%s' in %s:%d!\n", 
+               if (lock_info->locks[i].pending == -1) {
+                       /* This just means that the last lock this thread went for was by
+                        * using trylock, and it failed.  This is fine. */
+                       break;
+               }
+
+               ast_log(LOG_ERROR,
+                       "Thread '%s' still has a lock! - '%s' (%p) from '%s' in %s:%d!\n",
                        lock_info->thread_name,
                        lock_info->locks[i].lock_name,
                        lock_info->locks[i].lock_addr,
@@ -594,9 +658,10 @@ static void lock_info_destroy(void *data)
        }
 
        pthread_mutex_destroy(&lock_info->lock);
-       if (lock_info->thread_name)
-               free((void *) lock_info->thread_name);
-       free(lock_info);
+       if (lock_info->thread_name) {
+               ast_free((void *) lock_info->thread_name);
+       }
+       ast_free(lock_info);
 }
 
 /*!
@@ -723,6 +788,60 @@ int ast_find_lock_info(void *lock_addr, char *filename, size_t filename_size, in
        return 0;
 }
 
+void ast_suspend_lock_info(void *lock_addr)
+{
+       struct thr_lock_info *lock_info;
+       int i = 0;
+
+       if (!(lock_info = ast_threadstorage_get(&thread_lock_info, sizeof(*lock_info)))) {
+               return;
+       }
+
+       pthread_mutex_lock(&lock_info->lock);
+
+       for (i = lock_info->num_locks - 1; i >= 0; i--) {
+               if (lock_info->locks[i].lock_addr == lock_addr)
+                       break;
+       }
+
+       if (i == -1) {
+               /* Lock not found :( */
+               pthread_mutex_unlock(&lock_info->lock);
+               return;
+       }
+
+       lock_info->locks[i].suspended = 1;
+
+       pthread_mutex_unlock(&lock_info->lock);
+}
+
+void ast_restore_lock_info(void *lock_addr)
+{
+       struct thr_lock_info *lock_info;
+       int i = 0;
+
+       if (!(lock_info = ast_threadstorage_get(&thread_lock_info, sizeof(*lock_info))))
+               return;
+
+       pthread_mutex_lock(&lock_info->lock);
+
+       for (i = lock_info->num_locks - 1; i >= 0; i--) {
+               if (lock_info->locks[i].lock_addr == lock_addr)
+                       break;
+       }
+
+       if (i == -1) {
+               /* Lock not found :( */
+               pthread_mutex_unlock(&lock_info->lock);
+               return;
+       }
+
+       lock_info->locks[i].suspended = 0;
+
+       pthread_mutex_unlock(&lock_info->lock);
+}
+
+
 #ifdef HAVE_BKTR
 void ast_remove_lock_info(void *lock_addr, struct ast_bt *bt)
 #else
@@ -759,7 +878,7 @@ void ast_remove_lock_info(void *lock_addr)
 
        if (i < lock_info->num_locks - 1) {
                /* Not the last one ... *should* be rare! */
-               memmove(&lock_info->locks[i], &lock_info->locks[i + 1], 
+               memmove(&lock_info->locks[i], &lock_info->locks[i + 1],
                        (lock_info->num_locks - (i + 1)) * sizeof(lock_info->locks[0]));
        }
 
@@ -786,20 +905,24 @@ static const char *locktype2str(enum ast_lock_type type)
 static void append_backtrace_information(struct ast_str **str, struct ast_bt *bt)
 {
        char **symbols;
+       int num_frames;
 
        if (!bt) {
                ast_str_append(str, 0, "\tNo backtrace to print\n");
                return;
        }
 
-       if ((symbols = backtrace_symbols(bt->addresses, bt->num_frames))) {
+       /* store frame count locally to avoid the memory corruption that
+        * sometimes happens on virtualized CentOS 6.x systems */
+       num_frames = bt->num_frames;
+       if ((symbols = ast_bt_get_symbols(bt->addresses, num_frames))) {
                int frame_iterator;
-               
-               for (frame_iterator = 0; frame_iterator < bt->num_frames; ++frame_iterator) {
+
+               for (frame_iterator = 0; frame_iterator < num_frames; ++frame_iterator) {
                        ast_str_append(str, 0, "\t%s\n", symbols[frame_iterator]);
                }
 
-               free(symbols);
+               ast_std_free(symbols);
        } else {
                ast_str_append(str, 0, "\tCouldn't retrieve backtrace symbols\n");
        }
@@ -811,42 +934,43 @@ static void append_lock_information(struct ast_str **str, struct thr_lock_info *
        int j;
        ast_mutex_t *lock;
        struct ast_lock_track *lt;
-       
-       ast_str_append(str, 0, "=== ---> %sLock #%d (%s): %s %d %s %s %p (%d)\n", 
-                                  lock_info->locks[i].pending > 0 ? "Waiting for " : 
+
+       ast_str_append(str, 0, "=== ---> %sLock #%d (%s): %s %d %s %s %p (%d%s)\n",
+                                  lock_info->locks[i].pending > 0 ? "Waiting for " :
                                   lock_info->locks[i].pending < 0 ? "Tried and failed to get " : "", i,
-                                  lock_info->locks[i].file, 
+                                  lock_info->locks[i].file,
                                   locktype2str(lock_info->locks[i].type),
                                   lock_info->locks[i].line_num,
                                   lock_info->locks[i].func, lock_info->locks[i].lock_name,
-                                  lock_info->locks[i].lock_addr, 
-                                  lock_info->locks[i].times_locked);
+                                  lock_info->locks[i].lock_addr,
+                                  lock_info->locks[i].times_locked,
+                                  lock_info->locks[i].suspended ? " - suspended" : "");
 #ifdef HAVE_BKTR
        append_backtrace_information(str, lock_info->locks[i].backtrace);
 #endif
-       
+
        if (!lock_info->locks[i].pending || lock_info->locks[i].pending == -1)
                return;
-       
+
        /* We only have further details for mutexes right now */
        if (lock_info->locks[i].type != AST_MUTEX)
                return;
-       
+
        lock = lock_info->locks[i].lock_addr;
-       lt = &lock->track;
+       lt = lock->track;
        ast_reentrancy_lock(lt);
        for (j = 0; *str && j < lt->reentrancy; j++) {
                ast_str_append(str, 0, "=== --- ---> Locked Here: %s line %d (%s)\n",
                                           lt->file[j], lt->lineno[j], lt->func[j]);
        }
-       ast_reentrancy_unlock(lt);      
+       ast_reentrancy_unlock(lt);
 }
 
 
-/*! This function can help you find highly temporal locks; locks that happen for a 
+/*! This function can help you find highly temporal locks; locks that happen for a
     short time, but at unexpected times, usually at times that create a deadlock,
        Why is this thing locked right then? Who is locking it? Who am I fighting
-    with for this lock? 
+    with for this lock?
 
        To answer such questions, just call this routine before you would normally try
        to aquire a lock. It doesn't do anything if the lock is not acquired. If the
@@ -862,7 +986,7 @@ static void append_lock_information(struct ast_str **str, struct thr_lock_info *
        which will give a stack trace and continue. -- that aught to do the job!
 
 */
-void log_show_lock(void *this_lock_addr)
+void ast_log_show_lock(void *this_lock_addr)
 {
        struct thr_lock_info *lock_info;
        struct ast_str *str;
@@ -871,7 +995,7 @@ void log_show_lock(void *this_lock_addr)
                ast_log(LOG_NOTICE,"Could not create str\n");
                return;
        }
-       
+
 
        pthread_mutex_lock(&lock_infos_lock.mutex);
        AST_LIST_TRAVERSE(&lock_infos, lock_info, entry) {
@@ -882,7 +1006,7 @@ void log_show_lock(void *this_lock_addr)
                           it's acquired... */
                        if (lock_info->locks[i].lock_addr == this_lock_addr) {
                                append_lock_information(&str, lock_info, i);
-                               ast_log(LOG_NOTICE, "%s", str->str);
+                               ast_log(LOG_NOTICE, "%s", ast_str_buffer(str));
                                break;
                        }
                }
@@ -893,69 +1017,99 @@ void log_show_lock(void *this_lock_addr)
 }
 
 
-static char *handle_show_locks(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
+struct ast_str *ast_dump_locks(void)
 {
        struct thr_lock_info *lock_info;
        struct ast_str *str;
 
-       if (!(str = ast_str_create(4096)))
-               return CLI_FAILURE;
-
-       switch (cmd) {
-       case CLI_INIT:
-               e->command = "core show locks";
-               e->usage =
-                       "Usage: core show locks\n"
-                       "       This command is for lock debugging.  It prints out which locks\n"
-                       "are owned by each active thread.\n";
-               return NULL;
-
-       case CLI_GENERATE:
+       if (!(str = ast_str_create(4096))) {
                return NULL;
        }
 
-       ast_str_append(&str, 0, "\n" 
+       ast_str_append(&str, 0, "\n"
                       "=======================================================================\n"
-                      "=== Currently Held Locks ==============================================\n"
+                      "=== %s\n"
+                      "=== Currently Held Locks\n"
                       "=======================================================================\n"
                       "===\n"
                       "=== <pending> <lock#> (<file>): <lock type> <line num> <function> <lock name> <lock addr> (times locked)\n"
-                      "===\n");
+                      "===\n", ast_get_version());
 
-       if (!str)
-               return CLI_FAILURE;
+       if (!str) {
+               return NULL;
+       }
 
        pthread_mutex_lock(&lock_infos_lock.mutex);
        AST_LIST_TRAVERSE(&lock_infos, lock_info, entry) {
                int i;
-               if (lock_info->num_locks) {
-                       ast_str_append(&str, 0, "=== Thread ID: %d (%s)\n", (int) lock_info->thread_id,
-                               lock_info->thread_name);
-                       pthread_mutex_lock(&lock_info->lock);
-                       for (i = 0; str && i < lock_info->num_locks; i++) {
-                               append_lock_information(&str, lock_info, i);
+               int header_printed = 0;
+               pthread_mutex_lock(&lock_info->lock);
+               for (i = 0; str && i < lock_info->num_locks; i++) {
+                       /* Don't show suspended locks */
+                       if (lock_info->locks[i].suspended) {
+                               continue;
                        }
-                       pthread_mutex_unlock(&lock_info->lock);
-                       if (!str)
-                               break;
+
+                       if (!header_printed) {
+                               if (lock_info->lwp != -1) {
+                                       ast_str_append(&str, 0, "=== Thread ID: 0x%lx LWP:%d (%s)\n",
+                                               (long) lock_info->thread_id, lock_info->lwp, lock_info->thread_name);
+                               } else {
+                                       ast_str_append(&str, 0, "=== Thread ID: 0x%lx (%s)\n",
+                                               (long) lock_info->thread_id, lock_info->thread_name);
+                               }
+                               header_printed = 1;
+                       }
+
+                       append_lock_information(&str, lock_info, i);
+               }
+               pthread_mutex_unlock(&lock_info->lock);
+               if (!str) {
+                       break;
+               }
+               if (header_printed) {
                        ast_str_append(&str, 0, "=== -------------------------------------------------------------------\n"
-                                      "===\n");
-                       if (!str)
-                               break;
+                               "===\n");
+               }
+               if (!str) {
+                       break;
                }
        }
        pthread_mutex_unlock(&lock_infos_lock.mutex);
 
-       if (!str)
-               return CLI_FAILURE;
+       if (!str) {
+               return NULL;
+       }
 
        ast_str_append(&str, 0, "=======================================================================\n"
                       "\n");
 
-       if (!str)
+       return str;
+}
+
+static char *handle_show_locks(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
+{
+       struct ast_str *str;
+
+       switch (cmd) {
+       case CLI_INIT:
+               e->command = "core show locks";
+               e->usage =
+                       "Usage: core show locks\n"
+                       "       This command is for lock debugging.  It prints out which locks\n"
+                       "are owned by each active thread.\n";
+               return NULL;
+
+       case CLI_GENERATE:
+               return NULL;
+       }
+
+       str = ast_dump_locks();
+       if (!str) {
                return CLI_FAILURE;
+       }
 
-       ast_cli(a->fd, "%s", str->str);
+       ast_cli(a->fd, "%s", ast_str_buffer(str));
 
        ast_free(str);
 
@@ -993,22 +1147,12 @@ static void *dummy_start(void *data)
 #ifdef DEBUG_THREADS
        struct thr_lock_info *lock_info;
        pthread_mutexattr_t mutex_attr;
-#endif
 
-       /* note that even though data->name is a pointer to allocated memory,
-          we are not freeing it here because ast_register_thread is going to
-          keep a copy of the pointer and then ast_unregister_thread will
-          free the memory
-       */
-       ast_free(data);
-       ast_register_thread(a.name);
-       pthread_cleanup_push(ast_unregister_thread, (void *) pthread_self());
-
-#ifdef DEBUG_THREADS
        if (!(lock_info = ast_threadstorage_get(&thread_lock_info, sizeof(*lock_info))))
                return NULL;
 
        lock_info->thread_id = pthread_self();
+       lock_info->lwp = ast_get_tid();
        lock_info->thread_name = strdup(a.name);
 
        pthread_mutexattr_init(&mutex_attr);
@@ -1021,6 +1165,15 @@ static void *dummy_start(void *data)
        pthread_mutex_unlock(&lock_infos_lock.mutex); /* Intentionally not the wrapper */
 #endif /* DEBUG_THREADS */
 
+       /* note that even though data->name is a pointer to allocated memory,
+          we are not freeing it here because ast_register_thread is going to
+          keep a copy of the pointer and then ast_unregister_thread will
+          free the memory
+       */
+       ast_free(data);
+       ast_register_thread(a.name);
+       pthread_cleanup_push(ast_unregister_thread, (void *) pthread_self());
+
        ret = a.start_routine(a.data);
 
        pthread_cleanup_pop(1);
@@ -1039,7 +1192,7 @@ int ast_pthread_create_stack(pthread_t *thread, pthread_attr_t *attr, void *(*st
 #endif
 
        if (!attr) {
-               attr = alloca(sizeof(*attr));
+               attr = ast_alloca(sizeof(*attr));
                pthread_attr_init(attr);
        }
 
@@ -1066,8 +1219,10 @@ int ast_pthread_create_stack(pthread_t *thread, pthread_attr_t *attr, void *(*st
                a->start_routine = start_routine;
                a->data = data;
                start_routine = dummy_start;
-               asprintf(&a->name, "%-20s started at [%5d] %s %s()",
-                        start_fn, line, file, caller);
+               if (ast_asprintf(&a->name, "%-20s started at [%5d] %s %s()",
+                            start_fn, line, file, caller) < 0) {
+                       a->name = NULL;
+               }
                data = a;
        }
 #endif /* !LOW_MEMORY */
@@ -1084,7 +1239,7 @@ int ast_pthread_create_detached_stack(pthread_t *thread, pthread_attr_t *attr, v
        int res;
 
        if (!attr) {
-               attr = alloca(sizeof(*attr));
+               attr = ast_alloca(sizeof(*attr));
                pthread_attr_init(attr);
                attr_destroy = 1;
        }
@@ -1092,7 +1247,7 @@ int ast_pthread_create_detached_stack(pthread_t *thread, pthread_attr_t *attr, v
        if ((errno = pthread_attr_setdetachstate(attr, PTHREAD_CREATE_DETACHED)))
                ast_log(LOG_WARNING, "pthread_attr_setdetachstate: %s\n", strerror(errno));
 
-       res = ast_pthread_create_stack(thread, attr, start_routine, data, 
+       res = ast_pthread_create_stack(thread, attr, start_routine, data,
                                       stacksize, file, caller, line, start_fn);
 
        if (attr_destroy)
@@ -1107,7 +1262,51 @@ int ast_wait_for_input(int fd, int ms)
        memset(pfd, 0, sizeof(pfd));
        pfd[0].fd = fd;
        pfd[0].events = POLLIN|POLLPRI;
-       return poll(pfd, 1, ms);
+       return ast_poll(pfd, 1, ms);
+}
+
+static int ast_wait_for_output(int fd, int timeoutms)
+{
+       struct pollfd pfd = {
+               .fd = fd,
+               .events = POLLOUT,
+       };
+       int res;
+       struct timeval start = ast_tvnow();
+       int elapsed = 0;
+
+       /* poll() until the fd is writable without blocking */
+       while ((res = ast_poll(&pfd, 1, timeoutms - elapsed)) <= 0) {
+               if (res == 0) {
+                       /* timed out. */
+#ifndef STANDALONE
+                       ast_debug(1, "Timed out trying to write\n");
+#endif
+                       return -1;
+               } else if (res == -1) {
+                       /* poll() returned an error, check to see if it was fatal */
+
+                       if (errno == EINTR || errno == EAGAIN) {
+                               elapsed = ast_tvdiff_ms(ast_tvnow(), start);
+                               if (elapsed >= timeoutms) {
+                                       return -1;
+                               }
+                               /* This was an acceptable error, go back into poll() */
+                               continue;
+                       }
+
+                       /* Fatal error, bail. */
+                       ast_log(LOG_ERROR, "poll returned error: %s\n", strerror(errno));
+
+                       return -1;
+               }
+               elapsed = ast_tvdiff_ms(ast_tvnow(), start);
+               if (elapsed >= timeoutms) {
+                       return -1;
+               }
+       }
+
+       return 0;
 }
 
 /*!
@@ -1118,41 +1317,103 @@ int ast_wait_for_input(int fd, int ms)
  * have a need to wait.  This way, we get better performance.
  * If the descriptor is blocking, all assumptions on the guaranteed
  * detail do not apply anymore.
- * Also note that in the current implementation, the delay is per-write,
- * so you still have no guarantees, anyways.
- * Fortunately the routine is only used in a few places (cli.c, manager.c,
- * res_agi.c) so it is reasonably easy to check how it behaves there.
- *
- * XXX We either need to fix the code, or fix the documentation.
  */
-int ast_carefulwrite(int fd, char *s, int len, int timeoutms) 
+int ast_carefulwrite(int fd, char *s, int len, int timeoutms)
 {
-       /* Try to write string, but wait no more than ms milliseconds
-          before timing out */
+       struct timeval start = ast_tvnow();
        int res = 0;
-       struct pollfd fds[1];
+       int elapsed = 0;
+
        while (len) {
+               if (ast_wait_for_output(fd, timeoutms - elapsed)) {
+                       return -1;
+               }
+
                res = write(fd, s, len);
-               if ((res < 0) && (errno != EAGAIN)) {
+
+               if (res < 0 && errno != EAGAIN && errno != EINTR) {
+                       /* fatal error from write() */
+                       ast_log(LOG_ERROR, "write() returned error: %s\n", strerror(errno));
                        return -1;
                }
-               if (res < 0)
+
+               if (res < 0) {
+                       /* It was an acceptable error */
                        res = 0;
+               }
+
+               /* Update how much data we have left to write */
                len -= res;
                s += res;
                res = 0;
-               if (len) {
-                       fds[0].fd = fd;
-                       fds[0].events = POLLOUT;
-                       /* Wait until writable again */
-                       res = poll(fds, 1, timeoutms);
-                       if (res < 1)
-                               return -1;
+
+               elapsed = ast_tvdiff_ms(ast_tvnow(), start);
+               if (elapsed >= timeoutms) {
+                       /* We've taken too long to write
+                        * This is only an error condition if we haven't finished writing. */
+                       res = len ? -1 : 0;
+                       break;
                }
        }
+
        return res;
 }
 
+int ast_careful_fwrite(FILE *f, int fd, const char *src, size_t len, int timeoutms)
+{
+       struct timeval start = ast_tvnow();
+       int n = 0;
+       int elapsed = 0;
+
+       while (len) {
+               if (ast_wait_for_output(fd, timeoutms - elapsed)) {
+                       /* poll returned a fatal error, so bail out immediately. */
+                       return -1;
+               }
+
+               /* Clear any errors from a previous write */
+               clearerr(f);
+
+               n = fwrite(src, 1, len, f);
+
+               if (ferror(f) && errno != EINTR && errno != EAGAIN) {
+                       /* fatal error from fwrite() */
+                       if (!feof(f)) {
+                               /* Don't spam the logs if it was just that the connection is closed. */
+                               ast_log(LOG_ERROR, "fwrite() returned error: %s\n", strerror(errno));
+                       }
+                       n = -1;
+                       break;
+               }
+
+               /* Update for data already written to the socket */
+               len -= n;
+               src += n;
+
+               elapsed = ast_tvdiff_ms(ast_tvnow(), start);
+               if (elapsed >= timeoutms) {
+                       /* We've taken too long to write
+                        * This is only an error condition if we haven't finished writing. */
+                       n = len ? -1 : 0;
+                       break;
+               }
+       }
+
+       while (fflush(f)) {
+               if (errno == EAGAIN || errno == EINTR) {
+                       continue;
+               }
+               if (!feof(f)) {
+                       /* Don't spam the logs if it was just that the connection is closed. */
+                       ast_log(LOG_ERROR, "fflush() returned error: %s\n", strerror(errno));
+               }
+               n = -1;
+               break;
+       }
+
+       return n < 0 ? -1 : 0;
+}
+
 char *ast_strip_quoted(char *s, const char *beg_quotes, const char *end_quotes)
 {
        char *e;
@@ -1255,6 +1516,26 @@ int ast_build_string(char **buffer, size_t *space, const char *fmt, ...)
        return result;
 }
 
+int ast_regex_string_to_regex_pattern(const char *regex_string, struct ast_str **regex_pattern)
+{
+       int regex_len = strlen(regex_string);
+       int ret = 3;
+
+       /* Chop off the leading / if there is one */
+       if ((regex_len >= 1) && (regex_string[0] == '/')) {
+               ast_str_set(regex_pattern, 0, "%s", regex_string + 1);
+               ret -= 2;
+       }
+
+       /* Chop off the ending / if there is one */
+       if ((regex_len > 1) && (regex_string[regex_len - 1] == '/')) {
+               ast_str_truncate(*regex_pattern, -1);
+               ret -= 1;
+       }
+
+       return ret;
+}
+
 int ast_true(const char *s)
 {
        if (ast_strlen_zero(s))
@@ -1336,10 +1617,33 @@ struct timeval ast_tvsub(struct timeval a, struct timeval b)
        }
        return a;
 }
-#undef ONE_MILLION
 
-/*! \brief glibc puts a lock inside random(3), so that the results are thread-safe.
- * BSD libc (and others) do not. */
+int ast_remaining_ms(struct timeval start, int max_ms)
+{
+       int ms;
+
+       if (max_ms < 0) {
+               ms = max_ms;
+       } else {
+               ms = max_ms - ast_tvdiff_ms(ast_tvnow(), start);
+               if (ms < 0) {
+                       ms = 0;
+               }
+       }
+
+       return ms;
+}
+
+void ast_format_duration_hh_mm_ss(int duration, char *buf, size_t length)
+{
+       int durh, durm, durs;
+       durh = duration / 3600;
+       durm = (duration % 3600) / 60;
+       durs = duration % 60;
+       snprintf(buf, length, "%02d:%02d:%02d", durh, durm, durs);
+}
+
+#undef ONE_MILLION
 
 #ifndef linux
 AST_MUTEX_DEFINE_STATIC(randomlock);
@@ -1348,7 +1652,7 @@ AST_MUTEX_DEFINE_STATIC(randomlock);
 long int ast_random(void)
 {
        long int res;
-#ifdef HAVE_DEV_URANDOM
+
        if (dev_urandom_fd >= 0) {
                int read_res = read(dev_urandom_fd, &res, sizeof(res));
                if (read_res > 0) {
@@ -1358,7 +1662,14 @@ long int ast_random(void)
                        return res % rm;
                }
        }
-#endif
+
+       /* XXX - Thread safety really depends on the libc, not the OS.
+        *
+        * But... popular Linux libc's (uClibc, glibc, eglibc), all have a
+        * somewhat thread safe random(3) (results are random, but not
+        * reproducible). The libc's for other systems (BSD, et al.), not so
+        * much.
+        */
 #ifdef linux
        res = random();
 #else
@@ -1369,9 +1680,18 @@ long int ast_random(void)
        return res;
 }
 
+void ast_replace_subargument_delimiter(char *s)
+{
+       for (; *s; s++) {
+               if (*s == '^') {
+                       *s = ',';
+               }
+       }
+}
+
 char *ast_process_quotes_and_slashes(char *start, char find, char replace_with)
 {
-       char *dataPut = start;
+       char *dataPut = start;
        int inEscape = 0;
        int inQuotes = 0;
 
@@ -1395,7 +1715,7 @@ char *ast_process_quotes_and_slashes(char *start, char find, char replace_with)
        return dataPut;
 }
 
-void ast_join(char *s, size_t len, char * const w[])
+void ast_join_delim(char *s, size_t len, const char * const w[], unsigned int size, char delim)
 {
        int x, ofs = 0;
        const char *src;
@@ -1403,9 +1723,9 @@ void ast_join(char *s, size_t len, char * const w[])
        /* Join words into a string */
        if (!s)
                return;
-       for (x = 0; ofs < len && w[x]; x++) {
+       for (x = 0; ofs < len && w[x] && x < size; x++) {
                if (x > 0)
-                       s[ofs++] = ' ';
+                       s[ofs++] = delim;
                for (src = w[x]; *src && ofs < len; src++)
                        s[ofs++] = *src;
        }
@@ -1414,29 +1734,79 @@ void ast_join(char *s, size_t len, char * const w[])
        s[ofs] = '\0';
 }
 
+char *ast_to_camel_case_delim(const char *s, const char *delim)
+{
+       char *res = ast_strdup(s);
+       char *front, *back, *buf = res;
+       int size;
+
+       front = strtok_r(buf, delim, &back);
+
+       while (front) {
+               size = strlen(front);
+               *front = toupper(*front);
+               ast_copy_string(buf, front, size + 1);
+               buf += size;
+               front = strtok_r(NULL, delim, &back);
+       }
+
+       return res;
+}
+
 /*
  * stringfields support routines.
  */
 
-const char __ast_string_field_empty[] = ""; /*!< the empty string */
+/* this is a little complex... string fields are stored with their
+   allocated size in the bytes preceding the string; even the
+   constant 'empty' string has to be this way, so the code that
+   checks to see if there is enough room for a new string doesn't
+   have to have any special case checks
+*/
+
+static const struct {
+       ast_string_field_allocation allocation;
+       char string[1];
+} __ast_string_field_empty_buffer;
+
+ast_string_field __ast_string_field_empty = __ast_string_field_empty_buffer.string;
+
+#define ALLOCATOR_OVERHEAD 48
+
+static size_t optimal_alloc_size(size_t size)
+{
+       unsigned int count;
+
+       size += ALLOCATOR_OVERHEAD;
+
+       for (count = 1; size; size >>= 1, count++);
+
+       return (1 << count) - ALLOCATOR_OVERHEAD;
+}
 
 /*! \brief add a new block to the pool.
  * We can only allocate from the topmost pool, so the
  * fields in *mgr reflect the size of that only.
  */
-static int add_string_pool(struct ast_string_field_mgr *mgr,
-                          struct ast_string_field_pool **pool_head,
-                          size_t size)
+static int add_string_pool(struct ast_string_field_mgr *mgr, struct ast_string_field_pool **pool_head,
+                          size_t size, const char *file, int lineno, const char *func)
 {
        struct ast_string_field_pool *pool;
+       size_t alloc_size = optimal_alloc_size(sizeof(*pool) + size);
 
-       if (!(pool = ast_calloc(1, sizeof(*pool) + size)))
+#if defined(__AST_DEBUG_MALLOC)
+       if (!(pool = __ast_calloc(1, alloc_size, file, lineno, func))) {
                return -1;
-       
+       }
+#else
+       if (!(pool = ast_calloc(1, alloc_size))) {
+               return -1;
+       }
+#endif
+
        pool->prev = *pool_head;
+       pool->size = alloc_size - sizeof(*pool);
        *pool_head = pool;
-       mgr->size = size;
-       mgr->used = 0;
        mgr->last_alloc = NULL;
 
        return 0;
@@ -1448,44 +1818,76 @@ static int add_string_pool(struct ast_string_field_mgr *mgr,
  * size > 0 means initialize the pool list with a pool of given size.
  *     This must be called right after allocating the object.
  * size = 0 means release all pools except the most recent one.
+ *      If the first pool was allocated via embedding in another
+ *      object, that pool will be preserved instead.
  *     This is useful to e.g. reset an object to the initial value.
  * size < 0 means release all pools.
  *     This must be done before destroying the object.
  */
-int __ast_string_field_init(struct ast_string_field_mgr *mgr,
-                           struct ast_string_field_pool **pool_head,
-                           int size)
+int __ast_string_field_init(struct ast_string_field_mgr *mgr, struct ast_string_field_pool **pool_head,
+                           int needed, const char *file, int lineno, const char *func)
 {
        const char **p = (const char **) pool_head + 1;
-       struct ast_string_field_pool *cur = *pool_head;
+       struct ast_string_field_pool *cur = NULL;
+       struct ast_string_field_pool *preserve = NULL;
 
        /* clear fields - this is always necessary */
-       while ((struct ast_string_field_mgr *) p != mgr)
+       while ((struct ast_string_field_mgr *) p != mgr) {
                *p++ = __ast_string_field_empty;
+       }
+
        mgr->last_alloc = NULL;
-       if (size > 0) {                 /* allocate the initial pool */
+#if defined(__AST_DEBUG_MALLOC)
+       mgr->owner_file = file;
+       mgr->owner_func = func;
+       mgr->owner_line = lineno;
+#endif
+       if (needed > 0) {               /* allocate the initial pool */
                *pool_head = NULL;
-               return add_string_pool(mgr, pool_head, size);
+               mgr->embedded_pool = NULL;
+               return add_string_pool(mgr, pool_head, needed, file, lineno, func);
        }
-       if (size < 0) {                 /* reset all pools */
-               *pool_head = NULL;
-       } else {                        /* preserve the first pool */
-               if (cur == NULL) {
+
+       /* if there is an embedded pool, we can't actually release *all*
+        * pools, we must keep the embedded one. if the caller is about
+        * to free the structure that contains the stringfield manager
+        * and embedded pool anyway, it will be freed as part of that
+        * operation.
+        */
+       if ((needed < 0) && mgr->embedded_pool) {
+               needed = 0;
+       }
+
+       if (needed < 0) {               /* reset all pools */
+               cur = *pool_head;
+       } else if (mgr->embedded_pool) { /* preserve the embedded pool */
+               preserve = mgr->embedded_pool;
+               cur = *pool_head;
+       } else {                        /* preserve the last pool */
+               if (*pool_head == NULL) {
                        ast_log(LOG_WARNING, "trying to reset empty pool\n");
                        return -1;
                }
-               cur = cur->prev;
-               (*pool_head)->prev = NULL;
-               mgr->used = 0;
+               preserve = *pool_head;
+               cur = preserve->prev;
+       }
+
+       if (preserve) {
+               preserve->prev = NULL;
+               preserve->used = preserve->active = 0;
        }
 
        while (cur) {
                struct ast_string_field_pool *prev = cur->prev;
 
-               ast_free(cur);
+               if (cur != preserve) {
+                       ast_free(cur);
+               }
                cur = prev;
        }
 
+       *pool_head = preserve;
+
        return 0;
 }
 
@@ -1493,33 +1895,49 @@ ast_string_field __ast_string_field_alloc_space(struct ast_string_field_mgr *mgr
                                                struct ast_string_field_pool **pool_head, size_t needed)
 {
        char *result = NULL;
-       size_t space = mgr->size - mgr->used;
+       size_t space = (*pool_head)->size - (*pool_head)->used;
+       size_t to_alloc;
+
+       /* Make room for ast_string_field_allocation and make it a multiple of that. */
+       to_alloc = ast_make_room_for(needed, ast_string_field_allocation);
+       ast_assert(to_alloc % ast_alignof(ast_string_field_allocation) == 0);
 
-       if (__builtin_expect(needed > space, 0)) {
-               size_t new_size = mgr->size * 2;
+       if (__builtin_expect(to_alloc > space, 0)) {
+               size_t new_size = (*pool_head)->size;
 
-               while (new_size < needed)
+               while (new_size < to_alloc) {
                        new_size *= 2;
+               }
 
-               if (add_string_pool(mgr, pool_head, new_size))
+#if defined(__AST_DEBUG_MALLOC)
+               if (add_string_pool(mgr, pool_head, new_size, mgr->owner_file, mgr->owner_line, mgr->owner_func))
                        return NULL;
+#else
+               if (add_string_pool(mgr, pool_head, new_size, __FILE__, __LINE__, __FUNCTION__))
+                       return NULL;
+#endif
        }
 
-       result = (*pool_head)->base + mgr->used;
-       mgr->used += needed;
+       /* pool->base is always aligned (gcc aligned attribute). We ensure that
+        * to_alloc is also a multiple of ast_alignof(ast_string_field_allocation)
+        * causing result to always be aligned as well; which in turn fixes that
+        * AST_STRING_FIELD_ALLOCATION(result) is aligned. */
+       result = (*pool_head)->base + (*pool_head)->used;
+       (*pool_head)->used += to_alloc;
+       (*pool_head)->active += needed;
+       result += ast_alignof(ast_string_field_allocation);
+       AST_STRING_FIELD_ALLOCATION(result) = needed;
        mgr->last_alloc = result;
+
        return result;
 }
 
-int __ast_string_field_ptr_grow(struct ast_string_field_mgr *mgr, size_t needed,
+int __ast_string_field_ptr_grow(struct ast_string_field_mgr *mgr,
+                               struct ast_string_field_pool **pool_head, size_t needed,
                                const ast_string_field *ptr)
 {
-       int grow = needed - (strlen(*ptr) + 1);
-       size_t space = mgr->size - mgr->used;
-
-       if (grow <= 0) {
-               return 0;
-       }
+       ssize_t grow = needed - AST_STRING_FIELD_ALLOCATION(*ptr);
+       size_t space = (*pool_head)->size - (*pool_head)->used;
 
        if (*ptr != mgr->last_alloc) {
                return 1;
@@ -1529,58 +1947,157 @@ int __ast_string_field_ptr_grow(struct ast_string_field_mgr *mgr, size_t needed,
                return 1;
        }
 
-       mgr->used += grow;
+       (*pool_head)->used += grow;
+       (*pool_head)->active += grow;
+       AST_STRING_FIELD_ALLOCATION(*ptr) += grow;
 
        return 0;
 }
 
-__attribute((format (printf, 4, 0)))
-void __ast_string_field_ptr_build_va(struct ast_string_field_mgr *mgr,
-                                    struct ast_string_field_pool **pool_head,
-                                    const ast_string_field *ptr, const char *format, va_list ap1, va_list ap2)
+void __ast_string_field_release_active(struct ast_string_field_pool *pool_head,
+                                      const ast_string_field ptr)
 {
-       size_t needed;
-       char *dst = (*pool_head)->base + mgr->used;
-       const char **p = (const char **) ptr;
-       size_t space = mgr->size - mgr->used;
+       struct ast_string_field_pool *pool, *prev;
 
-       /* try to write using available space */
-       needed = vsnprintf(dst, space, format, ap1) + 1;
+       if (ptr == __ast_string_field_empty) {
+               return;
+       }
 
-       va_end(ap1);
+       for (pool = pool_head, prev = NULL; pool; prev = pool, pool = pool->prev) {
+               if ((ptr >= pool->base) && (ptr <= (pool->base + pool->size))) {
+                       pool->active -= AST_STRING_FIELD_ALLOCATION(ptr);
+                       if ((pool->active == 0) && prev) {
+                               prev->prev = pool->prev;
+                               ast_free(pool);
+                       }
+                       break;
+               }
+       }
+}
 
-       if (needed > space) {   /* if it fails, reallocate */
-               size_t new_size = mgr->size * 2;
+void __ast_string_field_ptr_build_va(struct ast_string_field_mgr *mgr,
+                                    struct ast_string_field_pool **pool_head,
+                                    ast_string_field *ptr, const char *format, va_list ap)
+{
+       size_t needed;
+       size_t available;
+       size_t space = (*pool_head)->size - (*pool_head)->used;
+       ssize_t grow;
+       char *target;
+       va_list ap2;
+
+       /* if the field already has space allocated, try to reuse it;
+          otherwise, try to use the empty space at the end of the current
+          pool
+       */
+       if (*ptr != __ast_string_field_empty) {
+               target = (char *) *ptr;
+               available = AST_STRING_FIELD_ALLOCATION(*ptr);
+               if (*ptr == mgr->last_alloc) {
+                       available += space;
+               }
+       } else {
+               /* pool->used is always a multiple of ast_alignof(ast_string_field_allocation)
+                * so we don't need to re-align anything here.
+                */
+               target = (*pool_head)->base + (*pool_head)->used + ast_alignof(ast_string_field_allocation);
+               available = space - ast_alignof(ast_string_field_allocation);
+       }
 
-               while (new_size < needed)
-                       new_size *= 2;
+       va_copy(ap2, ap);
+       needed = vsnprintf(target, available, format, ap2) + 1;
+       va_end(ap2);
 
-               if (add_string_pool(mgr, pool_head, new_size))
+       if (needed > available) {
+               /* the allocation could not be satisfied using the field's current allocation
+                  (if it has one), or the space available in the pool (if it does not). allocate
+                  space for it, adding a new string pool if necessary.
+               */
+               if (!(target = (char *) __ast_string_field_alloc_space(mgr, pool_head, needed))) {
                        return;
-
-               dst = (*pool_head)->base + mgr->used;
-               vsprintf(dst, format, ap2);
+               }
+               vsprintf(target, format, ap);
+               va_end(ap); /* XXX va_end without va_start? */
+               __ast_string_field_release_active(*pool_head, *ptr);
+               *ptr = target;
+       } else if (*ptr != target) {
+               /* the allocation was satisfied using available space in the pool, but not
+                  using the space already allocated to the field
+               */
+               __ast_string_field_release_active(*pool_head, *ptr);
+               mgr->last_alloc = *ptr = target;
+               AST_STRING_FIELD_ALLOCATION(target) = needed;
+               (*pool_head)->used += ast_make_room_for(needed, ast_string_field_allocation);
+               (*pool_head)->active += needed;
+       } else if ((grow = (needed - AST_STRING_FIELD_ALLOCATION(*ptr))) > 0) {
+               /* the allocation was satisfied by using available space in the pool *and*
+                  the field was the last allocated field from the pool, so it grew
+               */
+               AST_STRING_FIELD_ALLOCATION(*ptr) += grow;
+               (*pool_head)->used += ast_align_for(grow, ast_string_field_allocation);
+               (*pool_head)->active += grow;
        }
-
-       mgr->last_alloc = *p = dst;
-       mgr->used += needed;
 }
 
-__attribute((format (printf, 4, 5)))
 void __ast_string_field_ptr_build(struct ast_string_field_mgr *mgr,
                                  struct ast_string_field_pool **pool_head,
-                                 const ast_string_field *ptr, const char *format, ...)
+                                 ast_string_field *ptr, const char *format, ...)
 {
-       va_list ap1, ap2;
+       va_list ap;
 
-       va_start(ap1, format);
-       va_start(ap2, format);          /* va_copy does not exist on FreeBSD */
+       va_start(ap, format);
+       __ast_string_field_ptr_build_va(mgr, pool_head, ptr, format, ap);
+       va_end(ap);
+}
 
-       __ast_string_field_ptr_build_va(mgr, pool_head, ptr, format, ap1, ap2);
+void *__ast_calloc_with_stringfields(unsigned int num_structs, size_t struct_size, size_t field_mgr_offset,
+                                    size_t field_mgr_pool_offset, size_t pool_size, const char *file,
+                                    int lineno, const char *func)
+{
+       struct ast_string_field_mgr *mgr;
+       struct ast_string_field_pool *pool;
+       struct ast_string_field_pool **pool_head;
+       size_t pool_size_needed = sizeof(*pool) + pool_size;
+       size_t size_to_alloc = optimal_alloc_size(struct_size + pool_size_needed);
+       void *allocation;
+       unsigned int x;
+
+#if defined(__AST_DEBUG_MALLOC)
+       if (!(allocation = __ast_calloc(num_structs, size_to_alloc, file, lineno, func))) {
+               return NULL;
+       }
+#else
+       if (!(allocation = ast_calloc(num_structs, size_to_alloc))) {
+               return NULL;
+       }
+#endif
 
-       va_end(ap1);
-       va_end(ap2);
+       for (x = 0; x < num_structs; x++) {
+               void *base = allocation + (size_to_alloc * x);
+               const char **p;
+
+               mgr = base + field_mgr_offset;
+               pool_head = base + field_mgr_pool_offset;
+               pool = base + struct_size;
+
+               p = (const char **) pool_head + 1;
+               while ((struct ast_string_field_mgr *) p != mgr) {
+                       *p++ = __ast_string_field_empty;
+               }
+
+               mgr->embedded_pool = pool;
+               *pool_head = pool;
+               pool->size = size_to_alloc - struct_size - sizeof(*pool);
+#if defined(__AST_DEBUG_MALLOC)
+               mgr->owner_file = file;
+               mgr->owner_func = func;
+               mgr->owner_line = lineno;
+#endif
+       }
+
+       return allocation;
 }
+
 /* end of stringfields support */
 
 AST_MUTEX_DEFINE_STATIC(fetchadd_m); /* used for all fetc&add ops */
@@ -1612,7 +2129,7 @@ int ast_get_timeval(const char *src, struct timeval *dst, struct timeval _defaul
                return -1;
 
        /* only integer at the moment, but one day we could accept more formats */
-       if (sscanf(src, "%Lf%n", &dtv, &scanned) > 0) {
+       if (sscanf(src, "%30Lf%n", &dtv, &scanned) > 0) {
                dst->tv_sec = dtv;
                dst->tv_usec = (dtv - dst->tv_sec) * 1000000.0;
                if (consumed)
@@ -1639,7 +2156,7 @@ int ast_get_time_t(const char *src, time_t *dst, time_t _default, int *consumed)
                return -1;
 
        /* only integer at the moment, but one day we could accept more formats */
-       if (sscanf(src, "%ld%n", &t, &scanned) == 1) {
+       if (sscanf(src, "%30ld%n", &t, &scanned) == 1) {
                *dst = t;
                if (consumed)
                        *consumed = scanned;
@@ -1648,64 +2165,11 @@ int ast_get_time_t(const char *src, time_t *dst, time_t _default, int *consumed)
                return -1;
 }
 
-/*!
- * core handler for dynamic strings.
- * This is not meant to be called directly, but rather through the
- * various wrapper macros
- *     ast_str_set(...)
- *     ast_str_append(...)
- *     ast_str_set_va(...)
- *     ast_str_append_va(...)
- */
-
-__attribute__((format (printf, 4, 0)))
-int __ast_str_helper(struct ast_str **buf, size_t max_len,
-       int append, const char *fmt, va_list ap)
-{
-       int res, need;
-       int offset = (append && (*buf)->len) ? (*buf)->used : 0;
-
-       if (max_len < 0)
-               max_len = (*buf)->len;  /* don't exceed the allocated space */
-       /*
-        * Ask vsnprintf how much space we need. Remember that vsnprintf
-        * does not count the final '\0' so we must add 1.
-        */
-       res = vsnprintf((*buf)->str + offset, (*buf)->len - offset, fmt, ap);
-
-       need = res + offset + 1;
-       /*
-        * If there is not enough space and we are below the max length,
-        * reallocate the buffer and return a message telling to retry.
-        */
-       if (need > (*buf)->len && (max_len == 0 || (*buf)->len < max_len) ) {
-               if (max_len && max_len < need)  /* truncate as needed */
-                       need = max_len;
-               else if (max_len == 0)  /* if unbounded, give more room for next time */
-                       need += 16 + need/4;
-               if (0)  /* debugging */
-                       ast_verbose("extend from %d to %d\n", (int)(*buf)->len, need);
-               if (ast_str_make_space(buf, need)) {
-                       ast_verbose("failed to extend from %d to %d\n", (int)(*buf)->len, need);
-                       return AST_DYNSTR_BUILD_FAILED;
-               }
-               (*buf)->str[offset] = '\0';     /* Truncate the partial write. */
-
-               /* va_end() and va_start() must be done before calling
-                * vsnprintf() again. */
-               return AST_DYNSTR_BUILD_RETRY;
-       }
-       /* update space used, keep in mind the truncation */
-       (*buf)->used = (res + offset > (*buf)->len) ? (*buf)->len : res + offset;
-
-       return res;
-}
-
 void ast_enable_packet_fragmentation(int sock)
 {
 #if defined(HAVE_IP_MTU_DISCOVER)
        int val = IP_PMTUDISC_DONT;
-       
+
        if (setsockopt(sock, IPPROTO_IP, IP_MTU_DISCOVER, &val, sizeof(val)))
                ast_log(LOG_WARNING, "Unable to disable PMTU discovery. Large UDP packets may fail to be delivered when sent from this socket.\n");
 #endif /* HAVE_IP_MTU_DISCOVER */
@@ -1717,7 +2181,7 @@ int ast_mkdir(const char *path, int mode)
        int len = strlen(path), count = 0, x, piececount = 0;
        char *tmp = ast_strdupa(path);
        char **pieces;
-       char *fullpath = alloca(len + 1);
+       char *fullpath = ast_alloca(len + 1);
        int res = 0;
 
        for (ptr = tmp; *ptr; ptr++) {
@@ -1726,7 +2190,7 @@ int ast_mkdir(const char *path, int mode)
        }
 
        /* Count the components to the directory path */
-       pieces = alloca(count * sizeof(*pieces));
+       pieces = ast_alloca(count * sizeof(*pieces));
        for (ptr = tmp; *ptr; ptr++) {
                if (*ptr == '/') {
                        *ptr = '\0';
@@ -1746,17 +2210,241 @@ int ast_mkdir(const char *path, int mode)
        return 0;
 }
 
+static int safe_mkdir(const char *base_path, char *path, int mode)
+{
+       RAII_VAR(char *, absolute_path, NULL, ast_std_free);
+
+       absolute_path = realpath(path, NULL);
+
+       if (absolute_path) {
+               /* Path exists, but is it in the right place? */
+               if (!ast_begins_with(absolute_path, base_path)) {
+                       return EPERM;
+               }
+
+               /* It is in the right place! */
+               return 0;
+       } else {
+               /* Path doesn't exist. */
+
+               /* The slash terminating the subpath we're checking */
+               char *path_term = strchr(path, '/');
+               /* True indicates the parent path is within base_path */
+               int parent_is_safe = 0;
+               int res;
+
+               while (path_term) {
+                       RAII_VAR(char *, absolute_subpath, NULL, ast_std_free);
+
+                       /* Truncate the path one past the slash */
+                       char c = *(path_term + 1);
+                       *(path_term + 1) = '\0';
+                       absolute_subpath = realpath(path, NULL);
+
+                       if (absolute_subpath) {
+                               /* Subpath exists, but is it safe? */
+                               parent_is_safe = ast_begins_with(
+                                       absolute_subpath, base_path);
+                       } else if (parent_is_safe) {
+                               /* Subpath does not exist, but parent is safe
+                                * Create it */
+                               res = mkdir(path, mode);
+                               if (res != 0) {
+                                       ast_assert(errno != EEXIST);
+                                       return errno;
+                               }
+                       } else {
+                               /* Subpath did not exist, parent was not safe
+                                * Fail! */
+                               errno = EPERM;
+                               return errno;
+                       }
+                       /* Restore the path */
+                       *(path_term + 1) = c;
+                       /* Move on to the next slash */
+                       path_term = strchr(path_term + 1, '/');
+               }
+
+               /* Now to build the final path, but only if it's safe */
+               if (!parent_is_safe) {
+                       errno = EPERM;
+                       return errno;
+               }
+
+               res = mkdir(path, mode);
+               if (res != 0 && errno != EEXIST) {
+                       return errno;
+               }
+
+               return 0;
+       }
+}
+
+int ast_safe_mkdir(const char *base_path, const char *path, int mode)
+{
+       RAII_VAR(char *, absolute_base_path, NULL, ast_std_free);
+       RAII_VAR(char *, p, NULL, ast_free);
+
+       if (base_path == NULL || path == NULL) {
+               errno = EFAULT;
+               return errno;
+       }
+
+       p = ast_strdup(path);
+       if (p == NULL) {
+               errno = ENOMEM;
+               return errno;
+       }
+
+       absolute_base_path = realpath(base_path, NULL);
+       if (absolute_base_path == NULL) {
+               return errno;
+       }
+
+       return safe_mkdir(absolute_base_path, p, mode);
+}
+
+static void utils_shutdown(void)
+{
+       close(dev_urandom_fd);
+       dev_urandom_fd = -1;
+#if defined(DEBUG_THREADS) && !defined(LOW_MEMORY)
+       ast_cli_unregister_multiple(utils_cli, ARRAY_LEN(utils_cli));
+#endif
+}
+
 int ast_utils_init(void)
 {
-#ifdef HAVE_DEV_URANDOM
        dev_urandom_fd = open("/dev/urandom", O_RDONLY);
-#endif
        base64_init();
 #ifdef DEBUG_THREADS
 #if !defined(LOW_MEMORY)
        ast_cli_register_multiple(utils_cli, ARRAY_LEN(utils_cli));
 #endif
 #endif
+       ast_register_atexit(utils_shutdown);
+       return 0;
+}
+
+
+/*!
+ *\brief Parse digest authorization header.
+ *\return Returns -1 if we have no auth or something wrong with digest.
+ *\note        This function may be used for Digest request and responce header.
+ * request arg is set to nonzero, if we parse Digest Request.
+ * pedantic arg can be set to nonzero if we need to do addition Digest check.
+ */
+int ast_parse_digest(const char *digest, struct ast_http_digest *d, int request, int pedantic) {
+       char *c;
+       struct ast_str *str = ast_str_create(16);
+
+       /* table of recognised keywords, and places where they should be copied */
+       const struct x {
+               const char *key;
+               const ast_string_field *field;
+       } *i, keys[] = {
+               { "username=", &d->username },
+               { "realm=", &d->realm },
+               { "nonce=", &d->nonce },
+               { "uri=", &d->uri },
+               { "domain=", &d->domain },
+               { "response=", &d->response },
+               { "cnonce=", &d->cnonce },
+               { "opaque=", &d->opaque },
+               /* Special cases that cannot be directly copied */
+               { "algorithm=", NULL },
+               { "qop=", NULL },
+               { "nc=", NULL },
+               { NULL, 0 },
+       };
+
+       if (ast_strlen_zero(digest) || !d || !str) {
+               ast_free(str);
+               return -1;
+       }
+
+       ast_str_set(&str, 0, "%s", digest);
+
+       c = ast_skip_blanks(ast_str_buffer(str));
+
+       if (strncasecmp(c, "Digest ", strlen("Digest "))) {
+               ast_log(LOG_WARNING, "Missing Digest.\n");
+               ast_free(str);
+               return -1;
+       }
+       c += strlen("Digest ");
+
+       /* lookup for keys/value pair */
+       while (c && *c && *(c = ast_skip_blanks(c))) {
+               /* find key */
+               for (i = keys; i->key != NULL; i++) {
+                       char *src, *separator;
+                       int unescape = 0;
+                       if (strncasecmp(c, i->key, strlen(i->key)) != 0) {
+                               continue;
+                       }
+
+                       /* Found. Skip keyword, take text in quotes or up to the separator. */
+                       c += strlen(i->key);
+                       if (*c == '"') {
+                               src = ++c;
+                               separator = "\"";
+                               unescape = 1;
+                       } else {
+                               src = c;
+                               separator = ",";
+                       }
+                       strsep(&c, separator); /* clear separator and move ptr */
+                       if (unescape) {
+                               ast_unescape_c(src);
+                       }
+                       if (i->field) {
+                               ast_string_field_ptr_set(d, i->field, src);
+                       } else {
+                               /* Special cases that require additional procesing */
+                               if (!strcasecmp(i->key, "algorithm=")) {
+                                       if (strcasecmp(src, "MD5")) {
+                                               ast_log(LOG_WARNING, "Digest algorithm: \"%s\" not supported.\n", src);
+                                               ast_free(str);
+                                               return -1;
+                                       }
+                               } else if (!strcasecmp(i->key, "qop=") && !strcasecmp(src, "auth")) {
+                                       d->qop = 1;
+                               } else if (!strcasecmp(i->key, "nc=")) {
+                                       unsigned long u;
+                                       if (sscanf(src, "%30lx", &u) != 1) {
+                                               ast_log(LOG_WARNING, "Incorrect Digest nc value: \"%s\".\n", src);
+                                               ast_free(str);
+                                               return -1;
+                                       }
+                                       ast_string_field_set(d, nc, src);
+                               }
+                       }
+                       break;
+               }
+               if (i->key == NULL) { /* not found, try ',' */
+                       strsep(&c, ",");
+               }
+       }
+       ast_free(str);
+
+       /* Digest checkout */
+       if (ast_strlen_zero(d->realm) || ast_strlen_zero(d->nonce)) {
+               /* "realm" and "nonce" MUST be always exist */
+               return -1;
+       }
+
+       if (!request) {
+               /* Additional check for Digest response */
+               if (ast_strlen_zero(d->username) || ast_strlen_zero(d->uri) || ast_strlen_zero(d->response)) {
+                       return -1;
+               }
+
+               if (pedantic && d->qop && (ast_strlen_zero(d->cnonce) || ast_strlen_zero(d->nc))) {
+                       return -1;
+               }
+       }
+
        return 0;
 }
 
@@ -1775,3 +2463,72 @@ int _ast_asprintf(char **ret, const char *file, int lineno, const char *func, co
        return res;
 }
 #endif
+
+int ast_get_tid(void)
+{
+       int ret = -1;
+#if defined (__linux) && defined(SYS_gettid)
+       ret = syscall(SYS_gettid); /* available since Linux 1.4.11 */
+#elif defined(__sun)
+       ret = pthread_self();
+#elif defined(__APPLE__)
+       ret = mach_thread_self();
+       mach_port_deallocate(mach_task_self(), ret);
+#elif defined(__FreeBSD__) && defined(HAVE_SYS_THR_H)
+       long lwpid;
+       thr_self(&lwpid); /* available since sys/thr.h creation 2003 */
+       ret = lwpid;
+#endif
+       return ret;
+}
+
+char *ast_utils_which(const char *binary, char *fullpath, size_t fullpath_size)
+{
+       const char *envPATH = getenv("PATH");
+       char *tpath, *path;
+       struct stat unused;
+       if (!envPATH) {
+               return NULL;
+       }
+       tpath = ast_strdupa(envPATH);
+       while ((path = strsep(&tpath, ":"))) {
+               snprintf(fullpath, fullpath_size, "%s/%s", path, binary);
+               if (!stat(fullpath, &unused)) {
+                       return fullpath;
+               }
+       }
+       return NULL;
+}
+
+void ast_do_crash(void)
+{
+#if defined(DO_CRASH)
+       abort();
+       /*
+        * Just in case abort() doesn't work or something else super
+        * silly, and for Qwell's amusement.
+        */
+       *((int *) 0) = 0;
+#endif /* defined(DO_CRASH) */
+}
+
+#if defined(AST_DEVMODE)
+void __ast_assert_failed(int condition, const char *condition_str, const char *file, int line, const char *function)
+{
+       /*
+        * Attempt to put it into the logger, but hope that at least
+        * someone saw the message on stderr ...
+        */
+       ast_log(__LOG_ERROR, file, line, function, "FRACK!, Failed assertion %s (%d)\n",
+               condition_str, condition);
+       fprintf(stderr, "FRACK!, Failed assertion %s (%d) at line %d in %s of %s\n",
+               condition_str, condition, line, function, file);
+       /*
+        * Give the logger a chance to get the message out, just in case
+        * we abort(), or Asterisk crashes due to whatever problem just
+        * happened after we exit ast_assert().
+        */
+       usleep(1);
+       ast_do_crash();
+}
+#endif /* defined(AST_DEVMODE) */