install_prereq: Download latest Jansson.
[asterisk/asterisk.git] / contrib / scripts / ast_coredumper
1 #!/usr/bin/env bash
2 # Turn on extended globbing
3 shopt -s extglob
4 # Bail on any error
5 set -e
6
7 prog=$(basename $0)
8
9 print_help() {
10 cat <<EOF
11 NAME
12         $prog - Dump and/or format asterisk coredump files
13
14 SYNOPSIS
15         $prog [ --help ] [ --running | --RUNNING ] [ --latest ]
16                 [ --tarball-coredumps ] [ --delete-coredumps-after ]
17                 [ --tarball-results ] [ --delete-results-after ]
18                 [ --tarball-config ] [ --tarball-uniqueid="<uniqueid>" ]
19                 [ --no-default-search ] [ --append-coredumps ]
20                 [ --asterisk-bin="path" ]
21                 [ <coredump> | <pattern> ... ]
22
23 DESCRIPTION
24
25         Extracts backtraces and lock tables from Asterisk coredump files.
26         For each coredump found, 4 new result files are created:
27         - <coredump>.brief.txt: The output of "thread apply all bt".
28
29         - <coredump>.thread1.txt: The output of "thread apply 1 bt full".
30
31         - <coredump>.full.txt: The output of "thread apply all bt full".
32
33         - <coredump>.locks.txt: If asterisk was compiled with
34                 "DEBUG_THREADS", this file will contain a dump of the locks
35                 table similar to doing a "core show locks" from the asterisk
36                 CLI.
37
38         Optional features:
39         - The running asterisk process can be suspended and dumped.
40         - The coredumps can be merged into a tarball.
41         - The coredumps can be deleted after processing.
42         - The results files can be merged into a tarball.
43         - The results files can be deleted after processing.
44
45         Options:
46
47         --help
48                 Print this help.
49
50         --running
51                 Create a coredump from the running asterisk instance and
52                 process it along with any other coredumps found (if any).
53                 WARNING: This WILL interrupt call processing.  You will be
54                 asked to confirm.  The coredump will be written to /tmp if
55                 $OUTPUTDIR is not defined.
56
57         --RUNNING
58                 Same as --running but without the confirmation prompt.
59                 DANGEROUS!!
60
61         --latest
62                 Process only the latest coredump from those specified (based
63                 on last-modified time).  If a dump of the running process was
64                 requested, it is always included in addition to the latest
65                 from the existing coredumps.
66
67         --tarball-coredumps
68                 Creates a gzipped tarball of coredumps processed, their
69                 results txt files and copies of /etc/os-release,
70                 /usr/sbin/asterisk, /usr/lib(64)/libasterisk* and
71                 /usr/lib(64)/asterisk as those files are needed to properly
72                 examine the coredump.  The file will be named
73                 $OUTPUTDIR/asterisk.<timestamp>.coredumps.tar.gz or
74                 $OUTPUTDIR/asterisk-<uniqueid>.coredumps.tar.gz if
75                 --tarball-uniqueid was specified.
76                 WARNING:  This file could 1gb in size!
77                 Mutually exclusive with --tartball-results
78
79         --delete-coredumps-after
80                 Deletes all processed coredumps regardless of whether
81                 a tarball was created.
82
83         --tarball-results
84                 Creates a gzipped tarball of all result files produced.
85                 The tarball name will be:
86                 $OUTPUTDIR/asterisk.<timestamp>.results.tar.gz
87                 Mutually exclusive with --tartball-coredumps
88
89         --delete-results-after
90                 Deletes all processed results regardless of whether
91                 a tarball was created.  It probably doesn't make sense
92                 to use this option unless you have also specified
93                 --tarball-results.
94
95         --tarball-config
96                 Adds the contents of /etc/asterisk to the tarball created
97                 with --tarball-coredumps or --tarball-results.
98
99         --tarball-uniqueid="<uniqueid>"
100                 Normally DATEFORMAT is used to make the tarballs unique
101                 but you can use your own unique id in the tarball names
102                 such as the Jira issue id.
103
104         --no-default-search
105                 Ignore COREDUMPS from the config files and process only
106                 coredumps listed on the command line (if any) and/or
107                 the running asterisk instance (if requested).
108
109         --append-coredumps
110                 Append any coredumps specified on the command line to the
111                 config file specified ones instead of overriding them.
112
113         --asterisk-binary
114                 Path to the asterisk binary. Default: look for asterisk
115                 in the PATH.
116
117         <coredump> | <pattern>
118                 A list of coredumps or coredump search patterns.  Unless
119                 --append-coredumps was specified, these entries will override
120                 those specified in the config files.
121
122                 Any resulting file that isn't actually a coredump is silently
123                 ignored.  If your patterns contains spaces be sure to only
124                 quote the portion of the pattern that DOESN'T contain wildcard
125                 expressions.  If you quote the whole pattern, it won't be
126                 expanded.
127
128                 If --no-default-search is specified and no files are specified
129                 on the command line, then the only the running asterisk process
130                 will be dumped (if requested).  Otherwise if no files are
131                 specified on the command line the value of COREDUMPS from
132                 ast_debug_tools.conf will be used.  Failing that, the following
133                 patterns will be used:
134                 /tmp/core[-._]asterisk!(*.txt)
135                 /tmp/core[-._]\$(hostname)!(*.txt)
136
137 NOTES
138         You must be root to use $prog.
139
140         $OUTPUTDIR can be read from the current environment or from the
141         ast_debug_tools.conf file described below.  If not specified,
142         work products are placed in the same directory as the core file.
143
144         The script relies on not only bash, but also recent GNU date and
145         gdb with python support.  *BSD operating systems may require
146         installation of the 'coreutils' and 'devel/gdb' packagess and minor
147         tweaking of the ast_debug_tools.conf file.
148
149         Any files output will have ':' characters changed to '-'.  This is
150         to facilitate uploading those files to Jira which doesn't like the
151         colons.
152
153 FILES
154         /etc/asterisk/ast_debug_tools.conf
155         ~/ast_debug_tools.conf
156         ./ast_debug_tools.conf
157
158         #
159         # This file is used by the Asterisk debug tools.
160         # Unlike other Asterisk config files, this one is
161         # "sourced" by bash and must adhere to bash semantics.
162         #
163
164         # A list of coredumps and/or coredump search patterns.
165         # Bash extended globs are enabled and any resulting files
166         # that aren't actually coredumps are silently ignored
167         # so you can be liberal with the globs.
168         #
169         # If your patterns contains spaces be sure to only quote
170         # the portion of the pattern that DOESN'T contain wildcard
171         # expressions.  If you quote the whole pattern, it won't
172         # be expanded and the glob characters will be treated as
173         # literals.
174         #
175         # The exclusion of files ending ".txt" is just for
176         # demonstration purposes as non-coredumps will be ignored
177         # anyway.
178         COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]\$(hostname)!(*.txt))
179
180         # The directory to contain output files and work directories.
181         # For output from existing core files, the default is the
182         # directory that the core file is found in.  For core files
183         # produced from a running process, the default is /tmp.
184         OUTPUTDIR=/some/directory
185
186         # Date command for the "running" coredump and tarballs.
187         # DATEFORMAT will be executed to get the timestamp.
188         # Don't put quotes around the format string or they'll be
189         # treated as literal characters.  Also be aware of colons
190         # in the output as you can't upload files with colons in
191         # the name to Jira.
192         #
193         # Unix timestamp
194         #DATEFORMAT='date +%s.%N'
195         #
196         # *BSD/MacOS doesn't support %N but after installing GNU
197         # coreutils...
198         #DATEFORMAT='gdate +%s.%N'
199         #
200         # Readable GMT
201         #DATEFORMAT='date -u +%FT%H-%M-%S%z'
202         #
203         # Readable Local time
204         DATEFORMAT='date +%FT%H-%M-%S%z'
205
206 EOF
207         exit 1
208 }
209
210 if [ $EUID -ne 0 ] ; then
211         echo "You must be root to use $prog."
212         exit 1
213 fi
214
215 running=false
216 RUNNING=false
217 latest=false
218 tarball_coredumps=false
219 delete_coredumps_after=false
220 tarball_results=false
221 delete_results_after=false
222 append_coredumps=false
223
224 declare -a COREDUMPS
225 declare -a ARGS_COREDUMPS
226
227 # Read config files from least important to most important
228 [ -f /etc/asterisk/ast_debug_tools.conf ] && source /etc/asterisk/ast_debug_tools.conf
229 [ -f ~/ast_debug_tools.conf ] && source ~/ast_debug_tools.conf
230 [ -f ./ast_debug_tools.conf ] && source ./ast_debug_tools.conf
231
232 # For *BSD, the preferred gdb may be in /usr/local/bin so we
233 # need to search for one that supports python.
234 for g in $(which -a gdb) ; do
235         result=$($g --batch --ex "python print('hello')" 2>/dev/null || : )
236         if [[ "$result" =~ ^hello$ ]] ; then
237                 GDB=$g
238                 break
239         fi
240 done
241
242 if [ -z "$GDB" ] ; then
243         echo "No suitable gdb was found in $PATH"
244         exit 1
245 fi
246
247 if [ -n "$OUTPUTDIR" ] ; then
248         if [ ! -d "$OUTPUTDIR" ] ; then
249                 echo "OUTPUTDIR $OUTPUTDIR doesn't exists or is not a directory"
250                 exit 1
251         fi
252 fi
253
254 if [ ${#COREDUMPS[@]} -eq 0 ] ; then
255         COREDUMPS+=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt))
256 fi
257
258 DATEFORMAT=${DATEFORMAT:-'date +%FT%H-%M-%S%z'}
259
260 # Use "$@" (with the quotes) so spaces in patterns or
261 # file names are preserved.
262 # Later on when we have to iterate over COREDUMPS, we always
263 # use the indexes rather than trying to expand the values of COREDUMPS
264 # just in case.
265
266 for a in "$@" ; do
267         case "$a" in
268         --running)
269                 running=true
270                 ;;
271         --RUNNING)
272                 RUNNING=true
273                 ;;
274         --no-default-search)
275                 # Clean out COREDUMPS from config files
276                 COREDUMPS=()
277                 ;;
278         --latest)
279                 latest=true
280                 ;;
281         --tarball-coredumps)
282                 tarball_coredumps=true
283                 ;;
284         --delete-coredumps-after)
285                 delete_coredumps_after=true
286                 ;;
287         --tarball-results)
288                 tarball_results=true
289                 ;;
290         --delete-results-after)
291                 delete_results_after=true
292                 ;;
293         --append-coredumps)
294                 append_coredumps=true
295                 ;;
296         --tarball-uniqueid=*)
297                 tarball_uniqueid=${a#*=}
298                 ;;
299         --asterisk-bin=*)
300                 asterisk_bin=${a#*=}
301                 ;;
302         --help|-*)
303                 print_help
304                 ;;
305         *)
306                 ARGS_COREDUMPS+=("$a")
307                 # If any files are specified on the command line, ignore those
308                 # specified in the config files unless append-coredumps was specified.
309                 if ! $append_coredumps ; then
310                         COREDUMPS=()
311                 fi
312         esac
313 done
314
315 # append coredumps/patterns specified as command line arguments to COREDUMPS.
316 for i in ${!ARGS_COREDUMPS[@]} ; do
317         COREDUMPS+=("${ARGS_COREDUMPS[$i]}")
318 done
319
320 # At this point, all glob entries that match files should be expanded.
321 # Any entries that don't exist are probably globs that didn't match anything
322 # and need to be pruned.  Any non coredumps are also pruned.
323
324 for i in ${!COREDUMPS[@]} ; do
325         if [ ! -f "${COREDUMPS[$i]}" ] ; then
326                 unset COREDUMPS[$i]
327                 continue
328         fi
329         # Some versions of 'file' don't allow only the first n bytes of the
330         # file to be processed so we use dd to grab just the first 32 bytes.
331         mimetype=$(dd if="${COREDUMPS[$i]}" bs=32 count=1 2>/dev/null | file -bi -)
332         if [[ ! "$mimetype" =~ coredump ]] ; then
333                 unset COREDUMPS[$i]
334                 continue
335         fi
336 done
337
338 # Sort and weed out any dups
339 IFS=$'\x0a'
340 readarray -t COREDUMPS < <(echo -n "${COREDUMPS[*]}" | sort -u )
341 unset IFS
342
343 # If --latest, get the last modified timestamp of each file,
344 # sort them, then return the latest.
345 if [ ${#COREDUMPS[@]} -gt 0 ] && $latest ; then
346         lf=$(find "${COREDUMPS[@]}" -printf '%T@ %p\n' | sort -n | tail -1)
347         COREDUMPS=("${lf#* }")
348 fi
349
350 # Timestamp to use for output files
351 df=${tarball_uniqueid:-$(${DATEFORMAT})}
352
353 if [ -z "$asterisk_bin" ]; then
354         asterisk_bin=$(which asterisk)
355 fi
356
357 if $running || $RUNNING ; then
358         # We need to go through some gyrations to find the pid of the running
359         # MAIN asterisk process and not someone or something running asterisk -r.
360         # The pid file may NOT be in /var/run/asterisk so we need to find any
361         # running asterisk process and see if -C was specified on the command
362         # line.  The chances of more than 1 asterisk instance running with
363         # different -C options is so unlikely that we're going to ignore it.
364         #
365         # 'ps axo command' should work on Linux (back to CentOS6) and FreeBSD.
366         # If asterisk was started with -C, get the asterisk.conf file.
367         # If it wasn't, assume /etc/asterisk/asterisk.conf
368         astetcconf=`ps axo command | sed -n -r -e "s/.*asterisk\s+.*-C\s+([^ ]+).*/\1/gp" | tail -1`
369         [ x$astetcconf = x ] && astetcconf=/etc/asterisk/asterisk.conf
370         # Now parse out astrundir and cat asterisk.pid
371         astrundir=$(sed -n -r -e "s/astrundir\s+[=>]+\s+(.*)/\1/gp" $astetcconf)
372         pid=$(cat $astrundir/asterisk.pid 2>/dev/null || : )
373         if [ x$pid = x ] ; then
374                 echo "Asterisk is not running"
375         else
376                 if $RUNNING ; then
377                         answer=Y
378                 else
379                         read -p "WARNING:  Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved.  Do you wish to continue? (y/N) " answer
380                 fi
381                 if [[ "$answer" =~ ^[Yy] ]] ; then
382                         cf="${OUTPUTDIR:-/tmp}/core-asterisk-running-$df"
383                         echo "Dumping running asterisk process to $cf"
384                         ${GDB} ${asterisk_bin} -p $pid -q --batch --ex "gcore $cf" >/dev/null 2>&1
385                         COREDUMPS+=("$cf")
386                 else
387                         echo "Skipping dump of running process"
388                 fi
389         fi
390 fi
391
392 if [ "${#COREDUMPS[@]}" -eq 0 ] ; then
393         echo "No coredumps found"
394         print_help
395 fi
396
397 # Extract the gdb scripts from the end of this script
398 # and save them to /tmp/.gdbinit
399
400 ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
401 tail -n +${ss} $0 >${OUTPUTDIR:-/tmp}/.ast_coredumper.gdbinit
402
403 # Now iterate over the coredumps and dump the debugging info
404 for i in ${!COREDUMPS[@]} ; do
405         cf=${COREDUMPS[$i]}
406         echo "Processing $cf"
407
408         cfdir=`dirname ${cf}`
409         cfname=`basename ${cf}`
410         outputdir=${OUTPUTDIR:-${cfdir}}
411
412         ${GDB} -n --batch -q --ex "source ${OUTPUTDIR:-/tmp}/.ast_coredumper.gdbinit" "$asterisk_bin" "$cf" 2>/dev/null | (
413                 of=/dev/null
414                 while IFS= read line ; do
415                         if [[ "$line" =~ !@!@!@!\ ([^\ ]+)\ !@!@!@! ]] ; then
416                                 of=${outputdir}/${cfname}-${BASH_REMATCH[1]}
417                                 of=${of//:/-}
418                                 rm -f "$of"
419                                 echo "Creating $of"
420                         fi
421                         echo -e $"$line" >> "$of"
422                 done
423         )
424
425         if $tarball_coredumps ; then
426                 cfname=${cfname//:/-}
427                 tf=${outputdir}/${cfname}.tar.gz
428                 echo "Creating ${tf}"
429
430                 dest=${outputdir}/${cfname}.output
431                 rm -rf ${dest} 2>/dev/null || :
432
433                 libdir=usr/lib
434                 [ -d /usr/lib64 ] && libdir+=64
435                 mkdir -p ${dest}/tmp ${dest}/${libdir}/asterisk ${dest}/etc ${dest}/usr/sbin
436
437                 ln -s ${cf} ${dest}/tmp/${cfname}
438                 cp ${outputdir}/${cfname}*.txt ${dest}/tmp/
439                 cp /etc/os-release ${dest}/etc/
440                 if $tarball_config ; then
441                         cp -a /etc/asterisk ${dest}/etc/
442                 fi
443                 cp -a /${libdir}/libasterisk* ${dest}/${libdir}/
444                 cp -a /${libdir}/asterisk/* ${dest}/${libdir}/asterisk/
445                 cp -a /usr/sbin/asterisk ${dest}/usr/sbin
446                 rm -rf ${tf}
447                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
448                 rm -rf ${dest}
449                 echo "Created $tf"
450         elif $tarball_results ; then
451                 cfname=${cfname//:/-}
452                 tf=${outputdir}/${cfname}.tar.gz
453                 echo "Creating ${tf}"
454
455                 dest=${outputdir}/${cfname}.output
456                 rm -rf ${dest} 2>/dev/null || :
457                 mkdir -p ${dest}
458                 cp ${outputdir}/${cfname}*.txt ${dest}/
459                 if $tarball_config ; then
460                         mkdir -p ${dest}/etc
461                         cp -a /etc/asterisk ${dest}/etc/
462                 fi
463                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
464                 rm -rf ${dest}
465                 echo "Created $tf"
466         fi
467
468 if $delete_coredumps_after ; then
469                 rm -rf "${cf}"
470         fi
471
472         if $delete_results_after ; then
473                 rm -rf "${cf//:/-}"-{brief,full,thread1,locks}.txt
474         fi
475 done
476
477 exit
478
479 # Be careful editng the inline scripts.
480 # They're space-indented.
481
482 # We need the python bit because lock_infos isn't
483 # a valid symbol in asterisk unless DEBUG_THREADS was
484 # used during the compile.  Also, interrupt and continue
485 # are only valid for a running program.
486
487 #@@@SCRIPTSTART@@@
488 python
489 class DumpAsteriskCommand(gdb.Command):
490
491     def __init__(self):
492         super(DumpAsteriskCommand, self).__init__ ("dump-asterisk",
493             gdb.COMMAND_OBSCURE, gdb.COMPLETE_COMMAND)
494
495     def invoke(self, arg, from_tty):
496         try:
497             gdb.execute("interrupt", from_tty)
498         except:
499             pass
500         print("!@!@!@! thread1.txt !@!@!@!\n")
501         try:
502             gdb.execute("p $_siginfo", from_tty)
503             gdb.execute("info signal $_siginfo.si_signo")
504         except:
505             pass
506         try:
507             gdb.execute("thread apply 1 bt full", from_tty)
508         except:
509             pass
510         print("!@!@!@! brief.txt !@!@!@!\n")
511         try:
512             gdb.execute("p $_siginfo", from_tty)
513             gdb.execute("info signal $_siginfo.si_signo")
514         except:
515             pass
516         try:
517             gdb.execute("thread apply all bt", from_tty)
518         except:
519             pass
520         print("!@!@!@! full.txt !@!@!@!\n")
521         try:
522             gdb.execute("p $_siginfo", from_tty)
523             gdb.execute("info signal $_siginfo.si_signo")
524         except:
525             pass
526         try:
527             gdb.execute("thread apply all bt full", from_tty)
528         except:
529             pass
530         print("!@!@!@! locks.txt !@!@!@!\n")
531         try:
532             gdb.execute("p $_siginfo", from_tty)
533             gdb.execute("info signal $_siginfo.si_signo")
534         except:
535             pass
536         try:
537             gdb.execute("show_locks", from_tty)
538         except:
539             pass
540         try:
541             gdb.execute("continue", from_tty)
542         except:
543             pass
544
545 DumpAsteriskCommand ()
546 end
547
548 define show_locks
549    set $n = lock_infos.first
550
551    if $argc == 0
552       printf "                                                                                                                    where_held count-|\n"
553       printf "                                                                                                                         suspended-| |\n"
554       printf "                                                                                                        type- |     times locked-| | |\n"
555       printf "thread         status   file                   line function                             lock name            | lock addr        | | |\n"
556    else
557       printf "thread,status,file,line,function,lock_name,lock_type,lock_addr,times_locked,suspended,where_held_count,where_held_file,where_held_line,where_held_function,there_held_thread\n"
558    end
559
560    while $n
561       if $n->num_locks > 0
562       set $i = 0
563       while $i < $n->num_locks
564          if $n->locks[$i]->suspended == 0
565             if ((ast_mutex_t *)$n->locks[$i]->lock_addr)->tracking
566                if $n->locks[$i]->type > 0
567                   set $track = ((ast_rwlock_t *)$n->locks[$i]->lock_addr)->track
568                else
569                   set $track = ((ast_mutex_t *)$n->locks[$i]->lock_addr)->track
570                end
571             end
572             set $reentrancy = $track->reentrancy
573             set $pending = $n->locks[$i]->pending
574             if $argc > 0
575                printf "%p,%d,%s,%d,%s,%s,%d,%p,%d,%d,%d",\
576                   $n->thread_id, $n->locks[$i]->pending, $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
577                   $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
578                   $n->locks[$i]->suspended, $track->reentrancy
579                if $reentrancy
580                   if $pending
581                      printf ",%s,%d,%s,%p", $track->file[0], $track->lineno[0], $track->func[0], $track->thread[0]
582                   end
583                end
584             else
585                if $n->locks[$i]->pending < 0
586                   printf "%p failed   %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
587                      $n->thread_id,\
588                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
589                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
590                      $n->locks[$i]->suspended, $track->reentrancy
591                end
592                if $n->locks[$i]->pending == 0
593                   printf "%p holding  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
594                      $n->thread_id,\
595                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
596                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
597                      $n->locks[$i]->suspended, $track->reentrancy
598                end
599                if $n->locks[$i]->pending > 0
600                   printf "%p waiting  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
601                      $n->thread_id,\
602                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
603                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
604                      $n->locks[$i]->suspended, $track->reentrancy
605                end
606                if $reentrancy
607                   if $pending
608                      printf "\n               held at: %-20s %6d %-36s by 0x%08lx", $track->file[0], $track->lineno[0], $track->func[0], $track->thread_id[0]
609                   end
610                end
611             end
612             printf "\n"
613          end
614          set $i = $i + 1
615       end
616     end
617     set $n = $n->entry->next
618   end
619 end
620
621 dump-asterisk