ast_coredumper: Remove .gdbinit file on exit
[asterisk/asterisk.git] / contrib / scripts / ast_coredumper
1 #!/usr/bin/env bash
2 # Turn on extended globbing
3 shopt -s extglob
4 # Bail on any error
5 set -e
6
7 prog=$(basename $0)
8
9 print_help() {
10 cat <<EOF
11 NAME
12         $prog - Dump and/or format asterisk coredump files
13
14 SYNOPSIS
15         $prog [ --help ] [ --running | --RUNNING ] [ --latest ]
16                 [ --tarball-coredumps ] [ --delete-coredumps-after ]
17                 [ --tarball-results ] [ --delete-results-after ]
18                 [ --tarball-config ] [ --tarball-uniqueid="<uniqueid>" ]
19                 [ --no-default-search ] [ --append-coredumps ]
20                 [ --asterisk-bin="path" ]
21                 [ <coredump> | <pattern> ... ]
22
23 DESCRIPTION
24
25         Extracts backtraces and lock tables from Asterisk coredump files.
26         For each coredump found, 4 new result files are created:
27         - <coredump>.brief.txt: The output of "thread apply all bt".
28
29         - <coredump>.thread1.txt: The output of "thread apply 1 bt full".
30
31         - <coredump>.full.txt: The output of "thread apply all bt full".
32
33         - <coredump>.locks.txt: If asterisk was compiled with
34                 "DEBUG_THREADS", this file will contain a dump of the locks
35                 table similar to doing a "core show locks" from the asterisk
36                 CLI.
37
38         Optional features:
39         - The running asterisk process can be suspended and dumped.
40         - The coredumps can be merged into a tarball.
41         - The coredumps can be deleted after processing.
42         - The results files can be merged into a tarball.
43         - The results files can be deleted after processing.
44
45         Options:
46
47         --help
48                 Print this help.
49
50         --running
51                 Create a coredump from the running asterisk instance and
52                 process it along with any other coredumps found (if any).
53                 WARNING: This WILL interrupt call processing.  You will be
54                 asked to confirm.  The coredump will be written to /tmp if
55                 $OUTPUTDIR is not defined.
56
57         --RUNNING
58                 Same as --running but without the confirmation prompt.
59                 DANGEROUS!!
60
61         --latest
62                 Process only the latest coredump from those specified (based
63                 on last-modified time).  If a dump of the running process was
64                 requested, it is always included in addition to the latest
65                 from the existing coredumps.
66
67         --tarball-coredumps
68                 Creates a gzipped tarball of coredumps processed, their
69                 results txt files and copies of /etc/os-release,
70                 /usr/sbin/asterisk, /usr/lib(64)/libasterisk* and
71                 /usr/lib(64)/asterisk as those files are needed to properly
72                 examine the coredump.  The file will be named
73                 $OUTPUTDIR/asterisk.<timestamp>.coredumps.tar.gz or
74                 $OUTPUTDIR/asterisk-<uniqueid>.coredumps.tar.gz if
75                 --tarball-uniqueid was specified.
76                 WARNING:  This file could 1gb in size!
77                 Mutually exclusive with --tartball-results
78
79         --delete-coredumps-after
80                 Deletes all processed coredumps regardless of whether
81                 a tarball was created.
82
83         --tarball-results
84                 Creates a gzipped tarball of all result files produced.
85                 The tarball name will be:
86                 $OUTPUTDIR/asterisk.<timestamp>.results.tar.gz
87                 Mutually exclusive with --tartball-coredumps
88
89         --delete-results-after
90                 Deletes all processed results regardless of whether
91                 a tarball was created.  It probably doesn't make sense
92                 to use this option unless you have also specified
93                 --tarball-results.
94
95         --tarball-config
96                 Adds the contents of /etc/asterisk to the tarball created
97                 with --tarball-coredumps or --tarball-results.
98
99         --tarball-uniqueid="<uniqueid>"
100                 Normally DATEFORMAT is used to make the tarballs unique
101                 but you can use your own unique id in the tarball names
102                 such as the Jira issue id.
103
104         --no-default-search
105                 Ignore COREDUMPS from the config files and process only
106                 coredumps listed on the command line (if any) and/or
107                 the running asterisk instance (if requested).
108
109         --append-coredumps
110                 Append any coredumps specified on the command line to the
111                 config file specified ones instead of overriding them.
112
113         --asterisk-binary
114                 Path to the asterisk binary. Default: look for asterisk
115                 in the PATH.
116
117         <coredump> | <pattern>
118                 A list of coredumps or coredump search patterns.  Unless
119                 --append-coredumps was specified, these entries will override
120                 those specified in the config files.
121
122                 Any resulting file that isn't actually a coredump is silently
123                 ignored.  If your patterns contains spaces be sure to only
124                 quote the portion of the pattern that DOESN'T contain wildcard
125                 expressions.  If you quote the whole pattern, it won't be
126                 expanded.
127
128                 If --no-default-search is specified and no files are specified
129                 on the command line, then the only the running asterisk process
130                 will be dumped (if requested).  Otherwise if no files are
131                 specified on the command line the value of COREDUMPS from
132                 ast_debug_tools.conf will be used.  Failing that, the following
133                 patterns will be used:
134                 /tmp/core[-._]asterisk!(*.txt)
135                 /tmp/core[-._]\$(hostname)!(*.txt)
136
137 NOTES
138         You must be root to use $prog.
139
140         $OUTPUTDIR can be read from the current environment or from the
141         ast_debug_tools.conf file described below.  If not specified,
142         work products are placed in the same directory as the core file.
143
144         The script relies on not only bash, but also recent GNU date and
145         gdb with python support.  *BSD operating systems may require
146         installation of the 'coreutils' and 'devel/gdb' packagess and minor
147         tweaking of the ast_debug_tools.conf file.
148
149         Any files output will have ':' characters changed to '-'.  This is
150         to facilitate uploading those files to Jira which doesn't like the
151         colons.
152
153 FILES
154         /etc/asterisk/ast_debug_tools.conf
155         ~/ast_debug_tools.conf
156         ./ast_debug_tools.conf
157
158         #
159         # This file is used by the Asterisk debug tools.
160         # Unlike other Asterisk config files, this one is
161         # "sourced" by bash and must adhere to bash semantics.
162         #
163
164         # A list of coredumps and/or coredump search patterns.
165         # Bash extended globs are enabled and any resulting files
166         # that aren't actually coredumps are silently ignored
167         # so you can be liberal with the globs.
168         #
169         # If your patterns contains spaces be sure to only quote
170         # the portion of the pattern that DOESN'T contain wildcard
171         # expressions.  If you quote the whole pattern, it won't
172         # be expanded and the glob characters will be treated as
173         # literals.
174         #
175         # The exclusion of files ending ".txt" is just for
176         # demonstration purposes as non-coredumps will be ignored
177         # anyway.
178         COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]\$(hostname)!(*.txt))
179
180         # The directory to contain output files and work directories.
181         # For output from existing core files, the default is the
182         # directory that the core file is found in.  For core files
183         # produced from a running process, the default is /tmp.
184         OUTPUTDIR=/some/directory
185
186         # Date command for the "running" coredump and tarballs.
187         # DATEFORMAT will be executed to get the timestamp.
188         # Don't put quotes around the format string or they'll be
189         # treated as literal characters.  Also be aware of colons
190         # in the output as you can't upload files with colons in
191         # the name to Jira.
192         #
193         # Unix timestamp
194         #DATEFORMAT='date +%s.%N'
195         #
196         # *BSD/MacOS doesn't support %N but after installing GNU
197         # coreutils...
198         #DATEFORMAT='gdate +%s.%N'
199         #
200         # Readable GMT
201         #DATEFORMAT='date -u +%FT%H-%M-%S%z'
202         #
203         # Readable Local time
204         DATEFORMAT='date +%FT%H-%M-%S%z'
205
206 EOF
207         exit 1
208 }
209
210 if [ $EUID -ne 0 ] ; then
211         echo "You must be root to use $prog."
212         exit 1
213 fi
214
215 running=false
216 RUNNING=false
217 latest=false
218 tarball_coredumps=false
219 tarball_config=false
220 delete_coredumps_after=false
221 tarball_results=false
222 delete_results_after=false
223 append_coredumps=false
224
225 declare -a COREDUMPS
226 declare -a ARGS_COREDUMPS
227
228 # readconf reads a bash-sourceable file and sets variables
229 # that havn't already been set.  This allows variables set
230 # on the command line or that are already in the environment
231 # to take precedence over those read from the file.
232 #
233 # Setting the values can't be done in a subshell so you can't
234 # just pipe the output of sed into the while.
235
236 readconf() {
237         while read line ; do
238                 declare -n v=${line%%=*}
239                 [ -z "${v}" ] && eval $line || :
240         done <<EOF
241 $( sed -r -e "/\s*#/d" -e "/^\s*$/d" $1 )
242 EOF
243 }
244
245 # Read config files from most important to least important.
246 # Variable set on the command line or environment always take precedence.
247 [ -f ./ast_debug_tools.conf ] && readconf ./ast_debug_tools.conf
248 [ -f ~/ast_debug_tools.conf ] && readconf ~/ast_debug_tools.conf
249 [ -f /etc/asterisk/ast_debug_tools.conf ] && readconf /etc/asterisk/ast_debug_tools.conf
250
251 # For *BSD, the preferred gdb may be in /usr/local/bin so we
252 # need to search for one that supports python.
253 for g in $(which -a gdb) ; do
254         result=$($g --batch --ex "python print('hello')" 2>/dev/null || : )
255         if [[ "$result" =~ ^hello$ ]] ; then
256                 GDB=$g
257                 break
258         fi
259 done
260
261 if [ -z "$GDB" ] ; then
262         echo "No suitable gdb was found in $PATH"
263         exit 1
264 fi
265
266 if [ -n "$OUTPUTDIR" ] ; then
267         if [ ! -d "$OUTPUTDIR" ] ; then
268                 echo "OUTPUTDIR $OUTPUTDIR doesn't exists or is not a directory"
269                 exit 1
270         fi
271 fi
272
273 if [ ${#COREDUMPS[@]} -eq 0 ] ; then
274         COREDUMPS+=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt))
275 fi
276
277 DATEFORMAT=${DATEFORMAT:-'date +%FT%H-%M-%S%z'}
278
279 # Use "$@" (with the quotes) so spaces in patterns or
280 # file names are preserved.
281 # Later on when we have to iterate over COREDUMPS, we always
282 # use the indexes rather than trying to expand the values of COREDUMPS
283 # just in case.
284
285 for a in "$@" ; do
286         case "$a" in
287         --running)
288                 running=true
289                 ;;
290         --RUNNING)
291                 RUNNING=true
292                 ;;
293         --no-default-search)
294                 # Clean out COREDUMPS from config files
295                 COREDUMPS=()
296                 ;;
297         --latest)
298                 latest=true
299                 ;;
300         --tarball-coredumps)
301                 tarball_coredumps=true
302                 ;;
303         --tarball-config)
304                 tarball_config=true
305                 ;;
306         --delete-coredumps-after)
307                 delete_coredumps_after=true
308                 ;;
309         --tarball-results)
310                 tarball_results=true
311                 ;;
312         --delete-results-after)
313                 delete_results_after=true
314                 ;;
315         --append-coredumps)
316                 append_coredumps=true
317                 ;;
318         --tarball-uniqueid=*)
319                 tarball_uniqueid=${a#*=}
320                 ;;
321         --asterisk-bin=*)
322                 asterisk_bin=${a#*=}
323                 ;;
324         --help|-*)
325                 print_help
326                 ;;
327         *)
328                 ARGS_COREDUMPS+=("$a")
329                 # If any files are specified on the command line, ignore those
330                 # specified in the config files unless append-coredumps was specified.
331                 if ! $append_coredumps ; then
332                         COREDUMPS=()
333                 fi
334         esac
335 done
336
337 # append coredumps/patterns specified as command line arguments to COREDUMPS.
338 for i in ${!ARGS_COREDUMPS[@]} ; do
339         COREDUMPS+=("${ARGS_COREDUMPS[$i]}")
340 done
341
342 # At this point, all glob entries that match files should be expanded.
343 # Any entries that don't exist are probably globs that didn't match anything
344 # and need to be pruned.  Any non coredumps are also pruned.
345
346 for i in ${!COREDUMPS[@]} ; do
347         if [ ! -f "${COREDUMPS[$i]}" ] ; then
348                 unset COREDUMPS[$i]
349                 continue
350         fi
351         # Some versions of 'file' don't allow only the first n bytes of the
352         # file to be processed so we use dd to grab just the first 32 bytes.
353         mimetype=$(dd if="${COREDUMPS[$i]}" bs=32 count=1 2>/dev/null | file -bi -)
354         if [[ ! "$mimetype" =~ coredump ]] ; then
355                 unset COREDUMPS[$i]
356                 continue
357         fi
358 done
359
360 # Sort and weed out any dups
361 IFS=$'\x0a'
362 readarray -t COREDUMPS < <(echo -n "${COREDUMPS[*]}" | sort -u )
363 unset IFS
364
365 # If --latest, get the last modified timestamp of each file,
366 # sort them, then return the latest.
367 if [ ${#COREDUMPS[@]} -gt 0 ] && $latest ; then
368         lf=$(find "${COREDUMPS[@]}" -printf '%T@ %p\n' | sort -n | tail -1)
369         COREDUMPS=("${lf#* }")
370 fi
371
372 # Timestamp to use for output files
373 df=${tarball_uniqueid:-$(${DATEFORMAT})}
374
375 if [ -z "$asterisk_bin" ]; then
376         asterisk_bin=$(which asterisk)
377 fi
378
379 if $running || $RUNNING ; then
380         # We need to go through some gyrations to find the pid of the running
381         # MAIN asterisk process and not someone or something running asterisk -r.
382         # The pid file may NOT be in /var/run/asterisk so we need to find any
383         # running asterisk process and see if -C was specified on the command
384         # line.  The chances of more than 1 asterisk instance running with
385         # different -C options is so unlikely that we're going to ignore it.
386         #
387         # 'ps axo command' should work on Linux (back to CentOS6) and FreeBSD.
388         # If asterisk was started with -C, get the asterisk.conf file.
389         # If it wasn't, assume /etc/asterisk/asterisk.conf
390         astetcconf=`ps axo command | sed -n -r -e "s/.*asterisk\s+.*-C\s+([^ ]+).*/\1/gp" | tail -1`
391         [ x$astetcconf = x ] && astetcconf=/etc/asterisk/asterisk.conf
392         # Now parse out astrundir and cat asterisk.pid
393         astrundir=$(sed -n -r -e "s/astrundir\s+[=>]+\s+(.*)/\1/gp" $astetcconf)
394         pid=$(cat $astrundir/asterisk.pid 2>/dev/null || : )
395         if [ x$pid = x ] ; then
396                 echo "Asterisk is not running"
397         else
398                 if $RUNNING ; then
399                         answer=Y
400                 else
401                         read -p "WARNING:  Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved.  Do you wish to continue? (y/N) " answer
402                 fi
403                 if [[ "$answer" =~ ^[Yy] ]] ; then
404                         cf="${OUTPUTDIR:-/tmp}/core-asterisk-running-$df"
405                         echo "Dumping running asterisk process to $cf"
406                         ${GDB} ${asterisk_bin} -p $pid -q --batch --ex "gcore $cf" >/dev/null 2>&1
407                         COREDUMPS+=("$cf")
408                 else
409                         echo "Skipping dump of running process"
410                 fi
411         fi
412 fi
413
414 if [ "${#COREDUMPS[@]}" -eq 0 ] ; then
415         echo "No coredumps found"
416         print_help
417 fi
418
419 # Extract the gdb scripts from the end of this script
420 # and save them to /tmp/.gdbinit
421
422 gdbinit=${OUTPUTDIR:-/tmp}/.ast_coredumper.gdbinit
423
424 trap "rm $gdbinit" EXIT
425
426 ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
427 tail -n +${ss} $0 >$gdbinit
428
429 # Now iterate over the coredumps and dump the debugging info
430 for i in ${!COREDUMPS[@]} ; do
431         cf=${COREDUMPS[$i]}
432         echo "Processing $cf"
433
434         cfdir=`dirname ${cf}`
435         cfname=`basename ${cf}`
436         outputdir=${OUTPUTDIR:-${cfdir}}
437
438         ${GDB} -n --batch -q --ex "source $gdbinit" "$asterisk_bin" "$cf" 2>/dev/null | (
439                 of=/dev/null
440                 while IFS= read line ; do
441                         if [[ "$line" =~ !@!@!@!\ ([^\ ]+)\ !@!@!@! ]] ; then
442                                 of=${outputdir}/${cfname}-${BASH_REMATCH[1]}
443                                 of=${of//:/-}
444                                 rm -f "$of"
445                                 echo "Creating $of"
446                         fi
447                         echo -e $"$line" >> "$of"
448                 done
449         )
450
451         if $tarball_coredumps ; then
452                 cfname=${cfname//:/-}
453                 tf=${outputdir}/${cfname}.tar.gz
454                 echo "Creating ${tf}"
455
456                 dest=${outputdir}/${cfname}.output
457                 rm -rf ${dest} 2>/dev/null || :
458
459                 libdir=usr/lib
460                 [ -d /usr/lib64 ] && libdir+=64
461                 mkdir -p ${dest}/tmp ${dest}/${libdir}/asterisk ${dest}/etc ${dest}/usr/sbin
462
463                 ln -s ${cf} ${dest}/tmp/${cfname}
464                 cp ${outputdir}/${cfname}*.txt ${dest}/tmp/
465                 [ -f /etc/os-release ] && cp /etc/os-release ${dest}/etc/
466                 if $tarball_config ; then
467                         cp -a /etc/asterisk ${dest}/etc/
468                 fi
469                 cp -a /${libdir}/libasterisk* ${dest}/${libdir}/
470                 cp -a /${libdir}/asterisk/* ${dest}/${libdir}/asterisk/
471                 cp -a /usr/sbin/asterisk ${dest}/usr/sbin
472                 rm -rf ${tf}
473                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
474                 sleep 3
475                 rm -rf ${dest}
476                 echo "Created $tf"
477         elif $tarball_results ; then
478                 cfname=${cfname//:/-}
479                 tf=${outputdir}/${cfname}.tar.gz
480                 echo "Creating ${tf}"
481
482                 dest=${outputdir}/${cfname}.output
483                 rm -rf ${dest} 2>/dev/null || :
484                 mkdir -p ${dest}
485                 cp ${outputdir}/${cfname}*.txt ${dest}/
486                 if $tarball_config ; then
487                         mkdir -p ${dest}/etc
488                         cp -a /etc/asterisk ${dest}/etc/
489                 fi
490                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
491                 rm -rf ${dest}
492                 echo "Created $tf"
493         fi
494
495 if $delete_coredumps_after ; then
496                 rm -rf "${cf}"
497         fi
498
499         if $delete_results_after ; then
500                 rm -rf "${cf//:/-}"-{brief,full,thread1,locks}.txt
501         fi
502 done
503
504 exit
505
506 # Be careful editng the inline scripts.
507 # They're space-indented.
508
509 # We need the python bit because lock_infos isn't
510 # a valid symbol in asterisk unless DEBUG_THREADS was
511 # used during the compile.  Also, interrupt and continue
512 # are only valid for a running program.
513
514 #@@@SCRIPTSTART@@@
515 python
516 class DumpAsteriskCommand(gdb.Command):
517
518     def __init__(self):
519         super(DumpAsteriskCommand, self).__init__ ("dump-asterisk",
520             gdb.COMMAND_OBSCURE, gdb.COMPLETE_COMMAND)
521
522     def invoke(self, arg, from_tty):
523         try:
524             gdb.execute("interrupt", from_tty)
525         except:
526             pass
527         print("!@!@!@! thread1.txt !@!@!@!\n")
528         try:
529             gdb.execute("p $_siginfo", from_tty)
530             gdb.execute("info signal $_siginfo.si_signo")
531         except:
532             pass
533         try:
534             gdb.execute("thread apply 1 bt full", from_tty)
535         except:
536             pass
537         print("!@!@!@! brief.txt !@!@!@!\n")
538         try:
539             gdb.execute("p $_siginfo", from_tty)
540             gdb.execute("info signal $_siginfo.si_signo")
541         except:
542             pass
543         try:
544             gdb.execute("thread apply all bt", from_tty)
545         except:
546             pass
547         print("!@!@!@! full.txt !@!@!@!\n")
548         try:
549             gdb.execute("p $_siginfo", from_tty)
550             gdb.execute("info signal $_siginfo.si_signo")
551         except:
552             pass
553         try:
554             gdb.execute("thread apply all bt full", from_tty)
555         except:
556             pass
557         print("!@!@!@! locks.txt !@!@!@!\n")
558         try:
559             gdb.execute("p $_siginfo", from_tty)
560             gdb.execute("info signal $_siginfo.si_signo")
561         except:
562             pass
563         try:
564             gdb.execute("show_locks", from_tty)
565         except:
566             pass
567         try:
568             gdb.execute("continue", from_tty)
569         except:
570             pass
571
572 DumpAsteriskCommand ()
573 end
574
575 define show_locks
576    set $n = lock_infos.first
577
578    if $argc == 0
579       printf "                                                                                                                    where_held count-|\n"
580       printf "                                                                                                                         suspended-| |\n"
581       printf "                                                                                                        type- |     times locked-| | |\n"
582       printf "thread         status   file                   line function                             lock name            | lock addr        | | |\n"
583    else
584       printf "thread,status,file,line,function,lock_name,lock_type,lock_addr,times_locked,suspended,where_held_count,where_held_file,where_held_line,where_held_function,there_held_thread\n"
585    end
586
587    while $n
588       if $n->num_locks > 0
589       set $i = 0
590       while $i < $n->num_locks
591          if $n->locks[$i]->suspended == 0
592             if ((ast_mutex_t *)$n->locks[$i]->lock_addr)->tracking
593                if $n->locks[$i]->type > 0
594                   set $track = ((ast_rwlock_t *)$n->locks[$i]->lock_addr)->track
595                else
596                   set $track = ((ast_mutex_t *)$n->locks[$i]->lock_addr)->track
597                end
598             end
599             set $reentrancy = $track->reentrancy
600             set $pending = $n->locks[$i]->pending
601             if $argc > 0
602                printf "%p,%d,%s,%d,%s,%s,%d,%p,%d,%d,%d",\
603                   $n->thread_id, $n->locks[$i]->pending, $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
604                   $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
605                   $n->locks[$i]->suspended, $track->reentrancy
606                if $reentrancy
607                   if $pending
608                      printf ",%s,%d,%s,%p", $track->file[0], $track->lineno[0], $track->func[0], $track->thread[0]
609                   end
610                end
611             else
612                if $n->locks[$i]->pending < 0
613                   printf "%p failed   %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
614                      $n->thread_id,\
615                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
616                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
617                      $n->locks[$i]->suspended, $track->reentrancy
618                end
619                if $n->locks[$i]->pending == 0
620                   printf "%p holding  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
621                      $n->thread_id,\
622                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
623                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
624                      $n->locks[$i]->suspended, $track->reentrancy
625                end
626                if $n->locks[$i]->pending > 0
627                   printf "%p waiting  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
628                      $n->thread_id,\
629                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
630                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
631                      $n->locks[$i]->suspended, $track->reentrancy
632                end
633                if $reentrancy
634                   if $pending
635                      printf "\n               held at: %-20s %6d %-36s by 0x%08lx", $track->file[0], $track->lineno[0], $track->func[0], $track->thread_id[0]
636                   end
637                end
638             end
639             printf "\n"
640          end
641          set $i = $i + 1
642       end
643     end
644     set $n = $n->entry->next
645   end
646 end
647
648 dump-asterisk