b2ab4ace6a7c79625a751428f3f939af5656858a
[asterisk/asterisk.git] / contrib / scripts / ast_coredumper
1 #!/usr/bin/env bash
2 # Turn on extended globbing
3 shopt -s extglob
4 # Bail on any error
5 set -e
6
7 prog=$(basename $0)
8
9 print_help() {
10 cat <<EOF
11 NAME
12         $prog - Dump and/or format asterisk coredump files
13
14 SYNOPSIS
15         $prog [ --help ] [ --running | --RUNNING ] [ --latest ]
16                 [ --tarball-coredumps ] [ --delete-coredumps-after ]
17                 [ --tarball-results ] [ --delete-results-after ]
18                 [ --tarball-config ] [ --tarball-uniqueid="<uniqueid>" ]
19                 [ --no-default-search ] [ --append-coredumps ]
20                 [ --asterisk-bin="path" ]
21                 [ <coredump> | <pattern> ... ]
22
23 DESCRIPTION
24
25         Extracts backtraces and lock tables from Asterisk coredump files.
26         For each coredump found, 4 new result files are created:
27         - <coredump>.brief.txt: The output of "thread apply all bt".
28
29         - <coredump>.thread1.txt: The output of "thread apply 1 bt full".
30
31         - <coredump>.full.txt: The output of "thread apply all bt full".
32
33         - <coredump>.locks.txt: If asterisk was compiled with
34                 "DEBUG_THREADS", this file will contain a dump of the locks
35                 table similar to doing a "core show locks" from the asterisk
36                 CLI.
37
38         Optional features:
39         - The running asterisk process can be suspended and dumped.
40         - The coredumps can be merged into a tarball.
41         - The coredumps can be deleted after processing.
42         - The results files can be merged into a tarball.
43         - The results files can be deleted after processing.
44
45         Options:
46
47         --help
48                 Print this help.
49
50         --running
51                 Create a coredump from the running asterisk instance and
52                 process it along with any other coredumps found (if any).
53                 WARNING: This WILL interrupt call processing.  You will be
54                 asked to confirm.  The coredump will be written to /tmp if
55                 $OUTPUTDIR is not defined.
56
57         --RUNNING
58                 Same as --running but without the confirmation prompt.
59                 DANGEROUS!!
60
61         --latest
62                 Process only the latest coredump from those specified (based
63                 on last-modified time).  If a dump of the running process was
64                 requested, it is always included in addition to the latest
65                 from the existing coredumps.
66
67         --tarball-coredumps
68                 Creates a gzipped tarball of coredumps processed, their
69                 results txt files and copies of /etc/os-release,
70                 /usr/sbin/asterisk, /usr/lib(64)/libasterisk* and
71                 /usr/lib(64)/asterisk as those files are needed to properly
72                 examine the coredump.  The file will be named
73                 $OUTPUTDIR/asterisk.<timestamp>.coredumps.tar.gz or
74                 $OUTPUTDIR/asterisk-<uniqueid>.coredumps.tar.gz if
75                 --tarball-uniqueid was specified.
76                 WARNING:  This file could 1gb in size!
77                 Mutually exclusive with --tartball-results
78
79         --delete-coredumps-after
80                 Deletes all processed coredumps regardless of whether
81                 a tarball was created.
82
83         --tarball-results
84                 Creates a gzipped tarball of all result files produced.
85                 The tarball name will be:
86                 $OUTPUTDIR/asterisk.<timestamp>.results.tar.gz
87                 Mutually exclusive with --tartball-coredumps
88
89         --delete-results-after
90                 Deletes all processed results regardless of whether
91                 a tarball was created.  It probably doesn't make sense
92                 to use this option unless you have also specified
93                 --tarball-results.
94
95         --tarball-config
96                 Adds the contents of /etc/asterisk to the tarball created
97                 with --tarball-coredumps or --tarball-results.
98
99         --tarball-uniqueid="<uniqueid>"
100                 Normally DATEFORMAT is used to make the tarballs unique
101                 but you can use your own unique id in the tarball names
102                 such as the Jira issue id.
103
104         --no-default-search
105                 Ignore COREDUMPS from the config files and process only
106                 coredumps listed on the command line (if any) and/or
107                 the running asterisk instance (if requested).
108
109         --append-coredumps
110                 Append any coredumps specified on the command line to the
111                 config file specified ones instead of overriding them.
112
113         --asterisk-binary
114                 Path to the asterisk binary. Default: look for asterisk
115                 in the PATH.
116
117         <coredump> | <pattern>
118                 A list of coredumps or coredump search patterns.  Unless
119                 --append-coredumps was specified, these entries will override
120                 those specified in the config files.
121
122                 Any resulting file that isn't actually a coredump is silently
123                 ignored.  If your patterns contains spaces be sure to only
124                 quote the portion of the pattern that DOESN'T contain wildcard
125                 expressions.  If you quote the whole pattern, it won't be
126                 expanded.
127
128                 If --no-default-search is specified and no files are specified
129                 on the command line, then the only the running asterisk process
130                 will be dumped (if requested).  Otherwise if no files are
131                 specified on the command line the value of COREDUMPS from
132                 ast_debug_tools.conf will be used.  Failing that, the following
133                 patterns will be used:
134                 /tmp/core[-._]asterisk!(*.txt)
135                 /tmp/core[-._]\$(hostname)!(*.txt)
136
137 NOTES
138         You must be root to use $prog.
139
140         $OUTPUTDIR can be read from the current environment or from the
141         ast_debug_tools.conf file described below.  If not specified,
142         work products are placed in the same directory as the core file.
143
144         The script relies on not only bash, but also recent GNU date and
145         gdb with python support.  *BSD operating systems may require
146         installation of the 'coreutils' and 'devel/gdb' packagess and minor
147         tweaking of the ast_debug_tools.conf file.
148
149         Any files output will have ':' characters changed to '-'.  This is
150         to facilitate uploading those files to Jira which doesn't like the
151         colons.
152
153 FILES
154         /etc/asterisk/ast_debug_tools.conf
155         ~/ast_debug_tools.conf
156         ./ast_debug_tools.conf
157
158         #
159         # This file is used by the Asterisk debug tools.
160         # Unlike other Asterisk config files, this one is
161         # "sourced" by bash and must adhere to bash semantics.
162         #
163
164         # A list of coredumps and/or coredump search patterns.
165         # Bash extended globs are enabled and any resulting files
166         # that aren't actually coredumps are silently ignored
167         # so you can be liberal with the globs.
168         #
169         # If your patterns contains spaces be sure to only quote
170         # the portion of the pattern that DOESN'T contain wildcard
171         # expressions.  If you quote the whole pattern, it won't
172         # be expanded and the glob characters will be treated as
173         # literals.
174         #
175         # The exclusion of files ending ".txt" is just for
176         # demonstration purposes as non-coredumps will be ignored
177         # anyway.
178         COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]\$(hostname)!(*.txt))
179
180         # The directory to contain output files and work directories.
181         # For output from existing core files, the default is the
182         # directory that the core file is found in.  For core files
183         # produced from a running process, the default is /tmp.
184         OUTPUTDIR=/some/directory
185
186         # Date command for the "running" coredump and tarballs.
187         # DATEFORMAT will be executed to get the timestamp.
188         # Don't put quotes around the format string or they'll be
189         # treated as literal characters.  Also be aware of colons
190         # in the output as you can't upload files with colons in
191         # the name to Jira.
192         #
193         # Unix timestamp
194         #DATEFORMAT='date +%s.%N'
195         #
196         # *BSD/MacOS doesn't support %N but after installing GNU
197         # coreutils...
198         #DATEFORMAT='gdate +%s.%N'
199         #
200         # Readable GMT
201         #DATEFORMAT='date -u +%FT%H-%M-%S%z'
202         #
203         # Readable Local time
204         DATEFORMAT='date +%FT%H-%M-%S%z'
205
206 EOF
207         exit 1
208 }
209
210 if [ $EUID -ne 0 ] ; then
211         echo "You must be root to use $prog."
212         exit 1
213 fi
214
215 running=false
216 RUNNING=false
217 latest=false
218 tarball_coredumps=false
219 tarball_config=false
220 delete_coredumps_after=false
221 tarball_results=false
222 delete_results_after=false
223 append_coredumps=false
224
225 declare -a COREDUMPS
226 declare -a ARGS_COREDUMPS
227
228 # readconf reads a bash-sourceable file and sets variables
229 # that havn't already been set.  This allows variables set
230 # on the command line or that are already in the environment
231 # to take precedence over those read from the file.
232 #
233 # Setting the values can't be done in a subshell so you can't
234 # just pipe the output of sed into the while.
235
236 readconf() {
237         while read line ; do
238                 declare -n v=${line%%=*}
239                 [ -z "${v}" ] && eval $line || :
240         done <<EOF
241 $( sed -r -e "/\s*#/d" -e "/^\s*$/d" $1 )
242 EOF
243 }
244
245 # Read config files from most important to least important.
246 # Variable set on the command line or environment always take precedence.
247 [ -f ./ast_debug_tools.conf ] && readconf ./ast_debug_tools.conf
248 [ -f ~/ast_debug_tools.conf ] && readconf ~/ast_debug_tools.conf
249 [ -f /etc/asterisk/ast_debug_tools.conf ] && readconf /etc/asterisk/ast_debug_tools.conf
250
251 # For *BSD, the preferred gdb may be in /usr/local/bin so we
252 # need to search for one that supports python.
253 for g in $(which -a gdb) ; do
254         result=$($g --batch --ex "python print('hello')" 2>/dev/null || : )
255         if [[ "$result" =~ ^hello$ ]] ; then
256                 GDB=$g
257                 break
258         fi
259 done
260
261 if [ -z "$GDB" ] ; then
262         echo "No suitable gdb was found in $PATH"
263         exit 1
264 fi
265
266 if [ -n "$OUTPUTDIR" ] ; then
267         if [ ! -d "$OUTPUTDIR" ] ; then
268                 echo "OUTPUTDIR $OUTPUTDIR doesn't exists or is not a directory"
269                 exit 1
270         fi
271 fi
272
273 if [ ${#COREDUMPS[@]} -eq 0 ] ; then
274         COREDUMPS+=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt))
275 fi
276
277 DATEFORMAT=${DATEFORMAT:-'date +%FT%H-%M-%S%z'}
278
279 # Use "$@" (with the quotes) so spaces in patterns or
280 # file names are preserved.
281 # Later on when we have to iterate over COREDUMPS, we always
282 # use the indexes rather than trying to expand the values of COREDUMPS
283 # just in case.
284
285 for a in "$@" ; do
286         case "$a" in
287         --running)
288                 running=true
289                 ;;
290         --RUNNING)
291                 RUNNING=true
292                 ;;
293         --no-default-search)
294                 # Clean out COREDUMPS from config files
295                 COREDUMPS=()
296                 ;;
297         --latest)
298                 latest=true
299                 ;;
300         --tarball-coredumps)
301                 tarball_coredumps=true
302                 ;;
303         --tarball-config)
304                 tarball_config=true
305                 ;;
306         --delete-coredumps-after)
307                 delete_coredumps_after=true
308                 ;;
309         --tarball-results)
310                 tarball_results=true
311                 ;;
312         --delete-results-after)
313                 delete_results_after=true
314                 ;;
315         --append-coredumps)
316                 append_coredumps=true
317                 ;;
318         --tarball-uniqueid=*)
319                 tarball_uniqueid=${a#*=}
320                 ;;
321         --asterisk-bin=*)
322                 asterisk_bin=${a#*=}
323                 ;;
324         --help|-*)
325                 print_help
326                 ;;
327         *)
328                 ARGS_COREDUMPS+=("$a")
329                 # If any files are specified on the command line, ignore those
330                 # specified in the config files unless append-coredumps was specified.
331                 if ! $append_coredumps ; then
332                         COREDUMPS=()
333                 fi
334         esac
335 done
336
337 # append coredumps/patterns specified as command line arguments to COREDUMPS.
338 for i in ${!ARGS_COREDUMPS[@]} ; do
339         COREDUMPS+=("${ARGS_COREDUMPS[$i]}")
340 done
341
342 # At this point, all glob entries that match files should be expanded.
343 # Any entries that don't exist are probably globs that didn't match anything
344 # and need to be pruned.  Any non coredumps are also pruned.
345
346 for i in ${!COREDUMPS[@]} ; do
347         if [ ! -f "${COREDUMPS[$i]}" ] ; then
348                 unset COREDUMPS[$i]
349                 continue
350         fi
351         # Some versions of 'file' don't allow only the first n bytes of the
352         # file to be processed so we use dd to grab just the first 32 bytes.
353         mimetype=$(dd if="${COREDUMPS[$i]}" bs=32 count=1 2>/dev/null | file -bi -)
354         if [[ ! "$mimetype" =~ coredump ]] ; then
355                 unset COREDUMPS[$i]
356                 continue
357         fi
358 done
359
360 # Sort and weed out any dups
361 IFS=$'\x0a'
362 readarray -t COREDUMPS < <(echo -n "${COREDUMPS[*]}" | sort -u )
363 unset IFS
364
365 # If --latest, get the last modified timestamp of each file,
366 # sort them, then return the latest.
367 if [ ${#COREDUMPS[@]} -gt 0 ] && $latest ; then
368         lf=$(find "${COREDUMPS[@]}" -printf '%T@ %p\n' | sort -n | tail -1)
369         COREDUMPS=("${lf#* }")
370 fi
371
372 # Timestamp to use for output files
373 df=${tarball_uniqueid:-$(${DATEFORMAT})}
374
375 if [ -z "$asterisk_bin" ]; then
376         asterisk_bin=$(which asterisk)
377 fi
378
379 if $running || $RUNNING ; then
380         # We need to go through some gyrations to find the pid of the running
381         # MAIN asterisk process and not someone or something running asterisk -r.
382         # The pid file may NOT be in /var/run/asterisk so we need to find any
383         # running asterisk process and see if -C was specified on the command
384         # line.  The chances of more than 1 asterisk instance running with
385         # different -C options is so unlikely that we're going to ignore it.
386         #
387         # 'ps axo command' should work on Linux (back to CentOS6) and FreeBSD.
388         # If asterisk was started with -C, get the asterisk.conf file.
389         # If it wasn't, assume /etc/asterisk/asterisk.conf
390         astetcconf=`ps axo command | sed -n -r -e "s/.*asterisk\s+.*-C\s+([^ ]+).*/\1/gp" | tail -1`
391         [ x$astetcconf = x ] && astetcconf=/etc/asterisk/asterisk.conf
392         # Now parse out astrundir and cat asterisk.pid
393         astrundir=$(sed -n -r -e "s/astrundir\s+[=>]+\s+(.*)/\1/gp" $astetcconf)
394         pid=$(cat $astrundir/asterisk.pid 2>/dev/null || : )
395         if [ x$pid = x ] ; then
396                 echo "Asterisk is not running"
397         else
398                 if $RUNNING ; then
399                         answer=Y
400                 else
401                         read -p "WARNING:  Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved.  Do you wish to continue? (y/N) " answer
402                 fi
403                 if [[ "$answer" =~ ^[Yy] ]] ; then
404                         cf="${OUTPUTDIR:-/tmp}/core-asterisk-running-$df"
405                         echo "Dumping running asterisk process to $cf"
406                         ${GDB} ${asterisk_bin} -p $pid -q --batch --ex "gcore $cf" >/dev/null 2>&1
407                         COREDUMPS+=("$cf")
408                 else
409                         echo "Skipping dump of running process"
410                 fi
411         fi
412 fi
413
414 if [ "${#COREDUMPS[@]}" -eq 0 ] ; then
415         echo "No coredumps found"
416         print_help
417 fi
418
419 # Extract the gdb scripts from the end of this script
420 # and save them to /tmp/.gdbinit
421
422 ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
423 tail -n +${ss} $0 >${OUTPUTDIR:-/tmp}/.ast_coredumper.gdbinit
424
425 # Now iterate over the coredumps and dump the debugging info
426 for i in ${!COREDUMPS[@]} ; do
427         cf=${COREDUMPS[$i]}
428         echo "Processing $cf"
429
430         cfdir=`dirname ${cf}`
431         cfname=`basename ${cf}`
432         outputdir=${OUTPUTDIR:-${cfdir}}
433
434         ${GDB} -n --batch -q --ex "source ${OUTPUTDIR:-/tmp}/.ast_coredumper.gdbinit" "$asterisk_bin" "$cf" 2>/dev/null | (
435                 of=/dev/null
436                 while IFS= read line ; do
437                         if [[ "$line" =~ !@!@!@!\ ([^\ ]+)\ !@!@!@! ]] ; then
438                                 of=${outputdir}/${cfname}-${BASH_REMATCH[1]}
439                                 of=${of//:/-}
440                                 rm -f "$of"
441                                 echo "Creating $of"
442                         fi
443                         echo -e $"$line" >> "$of"
444                 done
445         )
446
447         if $tarball_coredumps ; then
448                 cfname=${cfname//:/-}
449                 tf=${outputdir}/${cfname}.tar.gz
450                 echo "Creating ${tf}"
451
452                 dest=${outputdir}/${cfname}.output
453                 rm -rf ${dest} 2>/dev/null || :
454
455                 libdir=usr/lib
456                 [ -d /usr/lib64 ] && libdir+=64
457                 mkdir -p ${dest}/tmp ${dest}/${libdir}/asterisk ${dest}/etc ${dest}/usr/sbin
458
459                 ln -s ${cf} ${dest}/tmp/${cfname}
460                 cp ${outputdir}/${cfname}*.txt ${dest}/tmp/
461                 [ -f /etc/os-release ] && cp /etc/os-release ${dest}/etc/
462                 if $tarball_config ; then
463                         cp -a /etc/asterisk ${dest}/etc/
464                 fi
465                 cp -a /${libdir}/libasterisk* ${dest}/${libdir}/
466                 cp -a /${libdir}/asterisk/* ${dest}/${libdir}/asterisk/
467                 cp -a /usr/sbin/asterisk ${dest}/usr/sbin
468                 rm -rf ${tf}
469                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
470                 sleep 3
471                 rm -rf ${dest}
472                 echo "Created $tf"
473         elif $tarball_results ; then
474                 cfname=${cfname//:/-}
475                 tf=${outputdir}/${cfname}.tar.gz
476                 echo "Creating ${tf}"
477
478                 dest=${outputdir}/${cfname}.output
479                 rm -rf ${dest} 2>/dev/null || :
480                 mkdir -p ${dest}
481                 cp ${outputdir}/${cfname}*.txt ${dest}/
482                 if $tarball_config ; then
483                         mkdir -p ${dest}/etc
484                         cp -a /etc/asterisk ${dest}/etc/
485                 fi
486                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
487                 rm -rf ${dest}
488                 echo "Created $tf"
489         fi
490
491 if $delete_coredumps_after ; then
492                 rm -rf "${cf}"
493         fi
494
495         if $delete_results_after ; then
496                 rm -rf "${cf//:/-}"-{brief,full,thread1,locks}.txt
497         fi
498 done
499
500 exit
501
502 # Be careful editng the inline scripts.
503 # They're space-indented.
504
505 # We need the python bit because lock_infos isn't
506 # a valid symbol in asterisk unless DEBUG_THREADS was
507 # used during the compile.  Also, interrupt and continue
508 # are only valid for a running program.
509
510 #@@@SCRIPTSTART@@@
511 python
512 class DumpAsteriskCommand(gdb.Command):
513
514     def __init__(self):
515         super(DumpAsteriskCommand, self).__init__ ("dump-asterisk",
516             gdb.COMMAND_OBSCURE, gdb.COMPLETE_COMMAND)
517
518     def invoke(self, arg, from_tty):
519         try:
520             gdb.execute("interrupt", from_tty)
521         except:
522             pass
523         print("!@!@!@! thread1.txt !@!@!@!\n")
524         try:
525             gdb.execute("p $_siginfo", from_tty)
526             gdb.execute("info signal $_siginfo.si_signo")
527         except:
528             pass
529         try:
530             gdb.execute("thread apply 1 bt full", from_tty)
531         except:
532             pass
533         print("!@!@!@! brief.txt !@!@!@!\n")
534         try:
535             gdb.execute("p $_siginfo", from_tty)
536             gdb.execute("info signal $_siginfo.si_signo")
537         except:
538             pass
539         try:
540             gdb.execute("thread apply all bt", from_tty)
541         except:
542             pass
543         print("!@!@!@! full.txt !@!@!@!\n")
544         try:
545             gdb.execute("p $_siginfo", from_tty)
546             gdb.execute("info signal $_siginfo.si_signo")
547         except:
548             pass
549         try:
550             gdb.execute("thread apply all bt full", from_tty)
551         except:
552             pass
553         print("!@!@!@! locks.txt !@!@!@!\n")
554         try:
555             gdb.execute("p $_siginfo", from_tty)
556             gdb.execute("info signal $_siginfo.si_signo")
557         except:
558             pass
559         try:
560             gdb.execute("show_locks", from_tty)
561         except:
562             pass
563         try:
564             gdb.execute("continue", from_tty)
565         except:
566             pass
567
568 DumpAsteriskCommand ()
569 end
570
571 define show_locks
572    set $n = lock_infos.first
573
574    if $argc == 0
575       printf "                                                                                                                    where_held count-|\n"
576       printf "                                                                                                                         suspended-| |\n"
577       printf "                                                                                                        type- |     times locked-| | |\n"
578       printf "thread         status   file                   line function                             lock name            | lock addr        | | |\n"
579    else
580       printf "thread,status,file,line,function,lock_name,lock_type,lock_addr,times_locked,suspended,where_held_count,where_held_file,where_held_line,where_held_function,there_held_thread\n"
581    end
582
583    while $n
584       if $n->num_locks > 0
585       set $i = 0
586       while $i < $n->num_locks
587          if $n->locks[$i]->suspended == 0
588             if ((ast_mutex_t *)$n->locks[$i]->lock_addr)->tracking
589                if $n->locks[$i]->type > 0
590                   set $track = ((ast_rwlock_t *)$n->locks[$i]->lock_addr)->track
591                else
592                   set $track = ((ast_mutex_t *)$n->locks[$i]->lock_addr)->track
593                end
594             end
595             set $reentrancy = $track->reentrancy
596             set $pending = $n->locks[$i]->pending
597             if $argc > 0
598                printf "%p,%d,%s,%d,%s,%s,%d,%p,%d,%d,%d",\
599                   $n->thread_id, $n->locks[$i]->pending, $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
600                   $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
601                   $n->locks[$i]->suspended, $track->reentrancy
602                if $reentrancy
603                   if $pending
604                      printf ",%s,%d,%s,%p", $track->file[0], $track->lineno[0], $track->func[0], $track->thread[0]
605                   end
606                end
607             else
608                if $n->locks[$i]->pending < 0
609                   printf "%p failed   %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
610                      $n->thread_id,\
611                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
612                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
613                      $n->locks[$i]->suspended, $track->reentrancy
614                end
615                if $n->locks[$i]->pending == 0
616                   printf "%p holding  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
617                      $n->thread_id,\
618                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
619                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
620                      $n->locks[$i]->suspended, $track->reentrancy
621                end
622                if $n->locks[$i]->pending > 0
623                   printf "%p waiting  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
624                      $n->thread_id,\
625                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
626                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
627                      $n->locks[$i]->suspended, $track->reentrancy
628                end
629                if $reentrancy
630                   if $pending
631                      printf "\n               held at: %-20s %6d %-36s by 0x%08lx", $track->file[0], $track->lineno[0], $track->func[0], $track->thread_id[0]
632                   end
633                end
634             end
635             printf "\n"
636          end
637          set $i = $i + 1
638       end
639     end
640     set $n = $n->entry->next
641   end
642 end
643
644 dump-asterisk