install_prereq: Update FreeBSD libraries.
[asterisk/asterisk.git] / contrib / scripts / ast_coredumper
1 #!/usr/bin/env bash
2 # Turn on extended globbing
3 shopt -s extglob
4 # Bail on any error
5 set -e
6
7 prog=$(basename $0)
8
9 print_help() {
10 cat <<EOF
11 NAME
12         $prog - Dump and/or format asterisk coredump files
13
14 SYNOPSIS
15         $prog [ --help ] [ --running | --RUNNING ] [ --latest ]
16                 [ --tarball-coredumps ] [ --delete-coredumps-after ]
17                 [ --tarball-results ] [ --delete-results-after ]
18                 [ --tarball-config ] [ --tarball-uniqueid="<uniqueid>" ]
19                 [ --no-default-search ] [ --append-coredumps ]
20                 [ --asterisk-bin="path" ]
21                 [ <coredump> | <pattern> ... ]
22
23 DESCRIPTION
24
25         Extracts backtraces and lock tables from Asterisk coredump files.
26         For each coredump found, 4 new result files are created:
27         - <coredump>.brief.txt: The output of "thread apply all bt".
28
29         - <coredump>.thread1.txt: The output of "thread apply 1 bt full".
30
31         - <coredump>.full.txt: The output of "thread apply all bt full".
32
33         - <coredump>.locks.txt: If asterisk was compiled with
34                 "DEBUG_THREADS", this file will contain a dump of the locks
35                 table similar to doing a "core show locks" from the asterisk
36                 CLI.
37
38         Optional features:
39         - The running asterisk process can be suspended and dumped.
40         - The coredumps can be merged into a tarball.
41         - The coredumps can be deleted after processing.
42         - The results files can be merged into a tarball.
43         - The results files can be deleted after processing.
44
45         Options:
46
47         --help
48                 Print this help.
49
50         --running
51                 Create a coredump from the running asterisk instance and
52                 process it along with any other coredumps found (if any).
53                 WARNING: This WILL interrupt call processing.  You will be
54                 asked to confirm.  The coredump will be written to /tmp if
55                 $OUTPUTDIR is not defined.
56
57         --RUNNING
58                 Same as --running but without the confirmation prompt.
59                 DANGEROUS!!
60
61         --latest
62                 Process only the latest coredump from those specified (based
63                 on last-modified time).  If a dump of the running process was
64                 requested, it is always included in addition to the latest
65                 from the existing coredumps.
66
67         --tarball-coredumps
68                 Creates a gzipped tarball of coredumps processed, their
69                 results txt files and copies of /etc/os-release,
70                 /usr/sbin/asterisk, /usr/lib(64)/libasterisk* and
71                 /usr/lib(64)/asterisk as those files are needed to properly
72                 examine the coredump.  The file will be named
73                 $OUTPUTDIR/asterisk.<timestamp>.coredumps.tar.gz or
74                 $OUTPUTDIR/asterisk-<uniqueid>.coredumps.tar.gz if
75                 --tarball-uniqueid was specified.
76                 WARNING:  This file could 1gb in size!
77                 Mutually exclusive with --tartball-results
78
79         --delete-coredumps-after
80                 Deletes all processed coredumps regardless of whether
81                 a tarball was created.
82
83         --tarball-results
84                 Creates a gzipped tarball of all result files produced.
85                 The tarball name will be:
86                 $OUTPUTDIR/asterisk.<timestamp>.results.tar.gz
87                 Mutually exclusive with --tartball-coredumps
88
89         --delete-results-after
90                 Deletes all processed results regardless of whether
91                 a tarball was created.  It probably doesn't make sense
92                 to use this option unless you have also specified
93                 --tarball-results.
94
95         --tarball-config
96                 Adds the contents of /etc/asterisk to the tarball created
97                 with --tarball-coredumps or --tarball-results.
98
99         --tarball-uniqueid="<uniqueid>"
100                 Normally DATEFORMAT is used to make the tarballs unique
101                 but you can use your own unique id in the tarball names
102                 such as the Jira issue id.
103
104         --no-default-search
105                 Ignore COREDUMPS from the config files and process only
106                 coredumps listed on the command line (if any) and/or
107                 the running asterisk instance (if requested).
108
109         --append-coredumps
110                 Append any coredumps specified on the command line to the
111                 config file specified ones instead of overriding them.
112
113         --asterisk-binary
114                 Path to the asterisk binary. Default: look for asterisk
115                 in the PATH.
116
117         <coredump> | <pattern>
118                 A list of coredumps or coredump search patterns.  Unless
119                 --append-coredumps was specified, these entries will override
120                 those specified in the config files.
121
122                 Any resulting file that isn't actually a coredump is silently
123                 ignored.  If your patterns contains spaces be sure to only
124                 quote the portion of the pattern that DOESN'T contain wildcard
125                 expressions.  If you quote the whole pattern, it won't be
126                 expanded.
127
128                 If --no-default-search is specified and no files are specified
129                 on the command line, then the only the running asterisk process
130                 will be dumped (if requested).  Otherwise if no files are
131                 specified on the command line the value of COREDUMPS from
132                 ast_debug_tools.conf will be used.  Failing that, the following
133                 patterns will be used:
134                 /tmp/core[-._]asterisk!(*.txt)
135                 /tmp/core[-._]\$(hostname)!(*.txt)
136
137 NOTES
138         You must be root to use $prog.
139
140         $OUTPUTDIR can be read from the current environment or from the
141         ast_debug_tools.conf file described below.  If not specified,
142         work products are placed in the same directory as the core file.
143
144         The script relies on not only bash, but also recent GNU date and
145         gdb with python support.  *BSD operating systems may require
146         installation of the 'coreutils' and 'devel/gdb' packagess and minor
147         tweaking of the ast_debug_tools.conf file.
148
149         Any files output will have ':' characters changed to '-'.  This is
150         to facilitate uploading those files to Jira which doesn't like the
151         colons.
152
153 FILES
154         /etc/asterisk/ast_debug_tools.conf
155         ~/ast_debug_tools.conf
156         ./ast_debug_tools.conf
157
158         #
159         # This file is used by the Asterisk debug tools.
160         # Unlike other Asterisk config files, this one is
161         # "sourced" by bash and must adhere to bash semantics.
162         #
163
164         # A list of coredumps and/or coredump search patterns.
165         # Bash extended globs are enabled and any resulting files
166         # that aren't actually coredumps are silently ignored
167         # so you can be liberal with the globs.
168         #
169         # If your patterns contains spaces be sure to only quote
170         # the portion of the pattern that DOESN'T contain wildcard
171         # expressions.  If you quote the whole pattern, it won't
172         # be expanded and the glob characters will be treated as
173         # literals.
174         #
175         # The exclusion of files ending ".txt" is just for
176         # demonstration purposes as non-coredumps will be ignored
177         # anyway.
178         COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]\$(hostname)!(*.txt))
179
180         # The directory to contain output files and work directories.
181         # For output from existing core files, the default is the
182         # directory that the core file is found in.  For core files
183         # produced from a running process, the default is /tmp.
184         OUTPUTDIR=/some/directory
185
186         # Date command for the "running" coredump and tarballs.
187         # DATEFORMAT will be executed to get the timestamp.
188         # Don't put quotes around the format string or they'll be
189         # treated as literal characters.  Also be aware of colons
190         # in the output as you can't upload files with colons in
191         # the name to Jira.
192         #
193         # Unix timestamp
194         #DATEFORMAT='date +%s.%N'
195         #
196         # *BSD/MacOS doesn't support %N but after installing GNU
197         # coreutils...
198         #DATEFORMAT='gdate +%s.%N'
199         #
200         # Readable GMT
201         #DATEFORMAT='date -u +%FT%H-%M-%S%z'
202         #
203         # Readable Local time
204         DATEFORMAT='date +%FT%H-%M-%S%z'
205
206 EOF
207         exit 1
208 }
209
210 if [ $EUID -ne 0 ] ; then
211         echo "You must be root to use $prog."
212         exit 1
213 fi
214
215 running=false
216 RUNNING=false
217 latest=false
218 tarball_coredumps=false
219 tarball_config=false
220 delete_coredumps_after=false
221 tarball_results=false
222 delete_results_after=false
223 append_coredumps=false
224
225 declare -a COREDUMPS
226 declare -a ARGS_COREDUMPS
227
228 # Read config files from least important to most important
229 [ -f /etc/asterisk/ast_debug_tools.conf ] && source /etc/asterisk/ast_debug_tools.conf
230 [ -f ~/ast_debug_tools.conf ] && source ~/ast_debug_tools.conf
231 [ -f ./ast_debug_tools.conf ] && source ./ast_debug_tools.conf
232
233 # For *BSD, the preferred gdb may be in /usr/local/bin so we
234 # need to search for one that supports python.
235 for g in $(which -a gdb) ; do
236         result=$($g --batch --ex "python print('hello')" 2>/dev/null || : )
237         if [[ "$result" =~ ^hello$ ]] ; then
238                 GDB=$g
239                 break
240         fi
241 done
242
243 if [ -z "$GDB" ] ; then
244         echo "No suitable gdb was found in $PATH"
245         exit 1
246 fi
247
248 if [ -n "$OUTPUTDIR" ] ; then
249         if [ ! -d "$OUTPUTDIR" ] ; then
250                 echo "OUTPUTDIR $OUTPUTDIR doesn't exists or is not a directory"
251                 exit 1
252         fi
253 fi
254
255 if [ ${#COREDUMPS[@]} -eq 0 ] ; then
256         COREDUMPS+=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt))
257 fi
258
259 DATEFORMAT=${DATEFORMAT:-'date +%FT%H-%M-%S%z'}
260
261 # Use "$@" (with the quotes) so spaces in patterns or
262 # file names are preserved.
263 # Later on when we have to iterate over COREDUMPS, we always
264 # use the indexes rather than trying to expand the values of COREDUMPS
265 # just in case.
266
267 for a in "$@" ; do
268         case "$a" in
269         --running)
270                 running=true
271                 ;;
272         --RUNNING)
273                 RUNNING=true
274                 ;;
275         --no-default-search)
276                 # Clean out COREDUMPS from config files
277                 COREDUMPS=()
278                 ;;
279         --latest)
280                 latest=true
281                 ;;
282         --tarball-coredumps)
283                 tarball_coredumps=true
284                 ;;
285         --tarball-config)
286                 tarball_config=true
287                 ;;
288         --delete-coredumps-after)
289                 delete_coredumps_after=true
290                 ;;
291         --tarball-results)
292                 tarball_results=true
293                 ;;
294         --delete-results-after)
295                 delete_results_after=true
296                 ;;
297         --append-coredumps)
298                 append_coredumps=true
299                 ;;
300         --tarball-uniqueid=*)
301                 tarball_uniqueid=${a#*=}
302                 ;;
303         --asterisk-bin=*)
304                 asterisk_bin=${a#*=}
305                 ;;
306         --help|-*)
307                 print_help
308                 ;;
309         *)
310                 ARGS_COREDUMPS+=("$a")
311                 # If any files are specified on the command line, ignore those
312                 # specified in the config files unless append-coredumps was specified.
313                 if ! $append_coredumps ; then
314                         COREDUMPS=()
315                 fi
316         esac
317 done
318
319 # append coredumps/patterns specified as command line arguments to COREDUMPS.
320 for i in ${!ARGS_COREDUMPS[@]} ; do
321         COREDUMPS+=("${ARGS_COREDUMPS[$i]}")
322 done
323
324 # At this point, all glob entries that match files should be expanded.
325 # Any entries that don't exist are probably globs that didn't match anything
326 # and need to be pruned.  Any non coredumps are also pruned.
327
328 for i in ${!COREDUMPS[@]} ; do
329         if [ ! -f "${COREDUMPS[$i]}" ] ; then
330                 unset COREDUMPS[$i]
331                 continue
332         fi
333         # Some versions of 'file' don't allow only the first n bytes of the
334         # file to be processed so we use dd to grab just the first 32 bytes.
335         mimetype=$(dd if="${COREDUMPS[$i]}" bs=32 count=1 2>/dev/null | file -bi -)
336         if [[ ! "$mimetype" =~ coredump ]] ; then
337                 unset COREDUMPS[$i]
338                 continue
339         fi
340 done
341
342 # Sort and weed out any dups
343 IFS=$'\x0a'
344 readarray -t COREDUMPS < <(echo -n "${COREDUMPS[*]}" | sort -u )
345 unset IFS
346
347 # If --latest, get the last modified timestamp of each file,
348 # sort them, then return the latest.
349 if [ ${#COREDUMPS[@]} -gt 0 ] && $latest ; then
350         lf=$(find "${COREDUMPS[@]}" -printf '%T@ %p\n' | sort -n | tail -1)
351         COREDUMPS=("${lf#* }")
352 fi
353
354 # Timestamp to use for output files
355 df=${tarball_uniqueid:-$(${DATEFORMAT})}
356
357 if [ -z "$asterisk_bin" ]; then
358         asterisk_bin=$(which asterisk)
359 fi
360
361 if $running || $RUNNING ; then
362         # We need to go through some gyrations to find the pid of the running
363         # MAIN asterisk process and not someone or something running asterisk -r.
364         # The pid file may NOT be in /var/run/asterisk so we need to find any
365         # running asterisk process and see if -C was specified on the command
366         # line.  The chances of more than 1 asterisk instance running with
367         # different -C options is so unlikely that we're going to ignore it.
368         #
369         # 'ps axo command' should work on Linux (back to CentOS6) and FreeBSD.
370         # If asterisk was started with -C, get the asterisk.conf file.
371         # If it wasn't, assume /etc/asterisk/asterisk.conf
372         astetcconf=`ps axo command | sed -n -r -e "s/.*asterisk\s+.*-C\s+([^ ]+).*/\1/gp" | tail -1`
373         [ x$astetcconf = x ] && astetcconf=/etc/asterisk/asterisk.conf
374         # Now parse out astrundir and cat asterisk.pid
375         astrundir=$(sed -n -r -e "s/astrundir\s+[=>]+\s+(.*)/\1/gp" $astetcconf)
376         pid=$(cat $astrundir/asterisk.pid 2>/dev/null || : )
377         if [ x$pid = x ] ; then
378                 echo "Asterisk is not running"
379         else
380                 if $RUNNING ; then
381                         answer=Y
382                 else
383                         read -p "WARNING:  Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved.  Do you wish to continue? (y/N) " answer
384                 fi
385                 if [[ "$answer" =~ ^[Yy] ]] ; then
386                         cf="${OUTPUTDIR:-/tmp}/core-asterisk-running-$df"
387                         echo "Dumping running asterisk process to $cf"
388                         ${GDB} ${asterisk_bin} -p $pid -q --batch --ex "gcore $cf" >/dev/null 2>&1
389                         COREDUMPS+=("$cf")
390                 else
391                         echo "Skipping dump of running process"
392                 fi
393         fi
394 fi
395
396 if [ "${#COREDUMPS[@]}" -eq 0 ] ; then
397         echo "No coredumps found"
398         print_help
399 fi
400
401 # Extract the gdb scripts from the end of this script
402 # and save them to /tmp/.gdbinit
403
404 ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
405 tail -n +${ss} $0 >${OUTPUTDIR:-/tmp}/.ast_coredumper.gdbinit
406
407 # Now iterate over the coredumps and dump the debugging info
408 for i in ${!COREDUMPS[@]} ; do
409         cf=${COREDUMPS[$i]}
410         echo "Processing $cf"
411
412         cfdir=`dirname ${cf}`
413         cfname=`basename ${cf}`
414         outputdir=${OUTPUTDIR:-${cfdir}}
415
416         ${GDB} -n --batch -q --ex "source ${OUTPUTDIR:-/tmp}/.ast_coredumper.gdbinit" "$asterisk_bin" "$cf" 2>/dev/null | (
417                 of=/dev/null
418                 while IFS= read line ; do
419                         if [[ "$line" =~ !@!@!@!\ ([^\ ]+)\ !@!@!@! ]] ; then
420                                 of=${outputdir}/${cfname}-${BASH_REMATCH[1]}
421                                 of=${of//:/-}
422                                 rm -f "$of"
423                                 echo "Creating $of"
424                         fi
425                         echo -e $"$line" >> "$of"
426                 done
427         )
428
429         if $tarball_coredumps ; then
430                 cfname=${cfname//:/-}
431                 tf=${outputdir}/${cfname}.tar.gz
432                 echo "Creating ${tf}"
433
434                 dest=${outputdir}/${cfname}.output
435                 rm -rf ${dest} 2>/dev/null || :
436
437                 libdir=usr/lib
438                 [ -d /usr/lib64 ] && libdir+=64
439                 mkdir -p ${dest}/tmp ${dest}/${libdir}/asterisk ${dest}/etc ${dest}/usr/sbin
440
441                 ln -s ${cf} ${dest}/tmp/${cfname}
442                 cp ${outputdir}/${cfname}*.txt ${dest}/tmp/
443                 [ -f /etc/os-release ] && cp /etc/os-release ${dest}/etc/
444                 if $tarball_config ; then
445                         cp -a /etc/asterisk ${dest}/etc/
446                 fi
447                 cp -a /${libdir}/libasterisk* ${dest}/${libdir}/
448                 cp -a /${libdir}/asterisk/* ${dest}/${libdir}/asterisk/
449                 cp -a /usr/sbin/asterisk ${dest}/usr/sbin
450                 rm -rf ${tf}
451                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
452                 sleep 3
453                 rm -rf ${dest}
454                 echo "Created $tf"
455         elif $tarball_results ; then
456                 cfname=${cfname//:/-}
457                 tf=${outputdir}/${cfname}.tar.gz
458                 echo "Creating ${tf}"
459
460                 dest=${outputdir}/${cfname}.output
461                 rm -rf ${dest} 2>/dev/null || :
462                 mkdir -p ${dest}
463                 cp ${outputdir}/${cfname}*.txt ${dest}/
464                 if $tarball_config ; then
465                         mkdir -p ${dest}/etc
466                         cp -a /etc/asterisk ${dest}/etc/
467                 fi
468                 tar -chzf ${tf} --transform="s/^[.]/${cfname}/" -C ${dest} .
469                 rm -rf ${dest}
470                 echo "Created $tf"
471         fi
472
473 if $delete_coredumps_after ; then
474                 rm -rf "${cf}"
475         fi
476
477         if $delete_results_after ; then
478                 rm -rf "${cf//:/-}"-{brief,full,thread1,locks}.txt
479         fi
480 done
481
482 exit
483
484 # Be careful editng the inline scripts.
485 # They're space-indented.
486
487 # We need the python bit because lock_infos isn't
488 # a valid symbol in asterisk unless DEBUG_THREADS was
489 # used during the compile.  Also, interrupt and continue
490 # are only valid for a running program.
491
492 #@@@SCRIPTSTART@@@
493 python
494 class DumpAsteriskCommand(gdb.Command):
495
496     def __init__(self):
497         super(DumpAsteriskCommand, self).__init__ ("dump-asterisk",
498             gdb.COMMAND_OBSCURE, gdb.COMPLETE_COMMAND)
499
500     def invoke(self, arg, from_tty):
501         try:
502             gdb.execute("interrupt", from_tty)
503         except:
504             pass
505         print("!@!@!@! thread1.txt !@!@!@!\n")
506         try:
507             gdb.execute("p $_siginfo", from_tty)
508             gdb.execute("info signal $_siginfo.si_signo")
509         except:
510             pass
511         try:
512             gdb.execute("thread apply 1 bt full", from_tty)
513         except:
514             pass
515         print("!@!@!@! brief.txt !@!@!@!\n")
516         try:
517             gdb.execute("p $_siginfo", from_tty)
518             gdb.execute("info signal $_siginfo.si_signo")
519         except:
520             pass
521         try:
522             gdb.execute("thread apply all bt", from_tty)
523         except:
524             pass
525         print("!@!@!@! full.txt !@!@!@!\n")
526         try:
527             gdb.execute("p $_siginfo", from_tty)
528             gdb.execute("info signal $_siginfo.si_signo")
529         except:
530             pass
531         try:
532             gdb.execute("thread apply all bt full", from_tty)
533         except:
534             pass
535         print("!@!@!@! locks.txt !@!@!@!\n")
536         try:
537             gdb.execute("p $_siginfo", from_tty)
538             gdb.execute("info signal $_siginfo.si_signo")
539         except:
540             pass
541         try:
542             gdb.execute("show_locks", from_tty)
543         except:
544             pass
545         try:
546             gdb.execute("continue", from_tty)
547         except:
548             pass
549
550 DumpAsteriskCommand ()
551 end
552
553 define show_locks
554    set $n = lock_infos.first
555
556    if $argc == 0
557       printf "                                                                                                                    where_held count-|\n"
558       printf "                                                                                                                         suspended-| |\n"
559       printf "                                                                                                        type- |     times locked-| | |\n"
560       printf "thread         status   file                   line function                             lock name            | lock addr        | | |\n"
561    else
562       printf "thread,status,file,line,function,lock_name,lock_type,lock_addr,times_locked,suspended,where_held_count,where_held_file,where_held_line,where_held_function,there_held_thread\n"
563    end
564
565    while $n
566       if $n->num_locks > 0
567       set $i = 0
568       while $i < $n->num_locks
569          if $n->locks[$i]->suspended == 0
570             if ((ast_mutex_t *)$n->locks[$i]->lock_addr)->tracking
571                if $n->locks[$i]->type > 0
572                   set $track = ((ast_rwlock_t *)$n->locks[$i]->lock_addr)->track
573                else
574                   set $track = ((ast_mutex_t *)$n->locks[$i]->lock_addr)->track
575                end
576             end
577             set $reentrancy = $track->reentrancy
578             set $pending = $n->locks[$i]->pending
579             if $argc > 0
580                printf "%p,%d,%s,%d,%s,%s,%d,%p,%d,%d,%d",\
581                   $n->thread_id, $n->locks[$i]->pending, $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
582                   $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
583                   $n->locks[$i]->suspended, $track->reentrancy
584                if $reentrancy
585                   if $pending
586                      printf ",%s,%d,%s,%p", $track->file[0], $track->lineno[0], $track->func[0], $track->thread[0]
587                   end
588                end
589             else
590                if $n->locks[$i]->pending < 0
591                   printf "%p failed   %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
592                      $n->thread_id,\
593                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
594                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
595                      $n->locks[$i]->suspended, $track->reentrancy
596                end
597                if $n->locks[$i]->pending == 0
598                   printf "%p holding  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
599                      $n->thread_id,\
600                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
601                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
602                      $n->locks[$i]->suspended, $track->reentrancy
603                end
604                if $n->locks[$i]->pending > 0
605                   printf "%p waiting  %-20s %6d %-36s %-20s %d %14p %3d %d %d",\
606                      $n->thread_id,\
607                      $n->locks[$i]->file, $n->locks[$i]->line_num, $n->locks[$i]->func,\
608                      $n->locks[$i]->lock_name, $n->locks[$i]->type, $n->locks[$i]->lock_addr, $n->locks[$i]->times_locked,\
609                      $n->locks[$i]->suspended, $track->reentrancy
610                end
611                if $reentrancy
612                   if $pending
613                      printf "\n               held at: %-20s %6d %-36s by 0x%08lx", $track->file[0], $track->lineno[0], $track->func[0], $track->thread_id[0]
614                   end
615                end
616             end
617             printf "\n"
618          end
619          set $i = $i + 1
620       end
621     end
622     set $n = $n->entry->next
623   end
624 end
625
626 dump-asterisk