debug_utilities: Create ast_loggrabber
authorGeorge Joseph <gjoseph@digium.com>
Fri, 13 Jan 2017 17:03:15 +0000 (10:03 -0700)
committerKevin Harwell <kharwell@digium.com>
Fri, 20 Jan 2017 17:20:22 +0000 (11:20 -0600)
ast_loggrabber gathers log files from customizable search patterns,
optionally converts POSIX timestamps to a readable format and
tarballs the results.

Also a few tweaks were made to ast_coredumper.

Change-Id: I8bfe1468ada24c1344ce4abab7b002a59a659495
(cherry picked from commit c70915287837704090d75f181525765de7a17221)

configs/samples/ast_debug_tools.conf.sample
contrib/Makefile
contrib/scripts/ast_coredumper
contrib/scripts/ast_loggrabber [new file with mode: 0755]

index 90e976f..f26626b 100644 (file)
@@ -38,3 +38,20 @@ COREDUMPS=(/tmp/core[-._]asterisk!(*.txt) /tmp/core[-._]$(hostname)!(*.txt))
 #
 # Readable Local time
 DATEFORMAT='date +%FT%H-%M-%S%z'
+
+# A list of log files and/or log file search patterns using the
+# same syntax as COREDUMPS.
+#
+LOGFILES=(/var/log/asterisk/messages* /var/log/asterisk/queue* \
+       /var/log/asterisk/debug* /var/log/asterisk/security*)
+
+# ast_loggrabber converts POSIX timestamps to readable format
+# using this Python strftime format string.  If not specified
+# or an empty string, no format covnersion is done.
+LOG_DATEFORMAT="%m/%d-%H:%M:%S.%f"
+
+# The timezone to use when converting POSIX timestamps to
+# readable format.  It can be specified in "<continent>/<city>"
+# format or in abbreviation format such as "CST6CDT".  If not
+# specified, the "local" timezone is used.
+# LOG_TIMEZONE=
index a5775cb..a667243 100644 (file)
@@ -20,10 +20,12 @@ clean:
 include $(ASTTOPDIR)/Makefile.rules
 
 install:
-       $(INSTALL) -d "$(DESTDIR)$(ASTDATADIR)/scripts"; \
-       $(INSTALL) -m 755 scripts/refcounter.py "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"; \
+       $(INSTALL) -d "$(DESTDIR)$(ASTDATADIR)/scripts"
+       $(INSTALL) -m 755 scripts/ast_loggrabber "$(DESTDIR)$(ASTDATADIR)/scripts/ast_loggrabber"
        $(INSTALL) -m 755 scripts/ast_coredumper "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper"
+       $(INSTALL) -m 755 scripts/refcounter.py "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"
 
 uninstall:
-       rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"
-       rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper"
+       -rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_loggrabber"
+       -rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/ast_coredumper"
+       -rm -f "$(DESTDIR)$(ASTDATADIR)/scripts/refcounter.py"
index c82732b..81e94e9 100755 (executable)
@@ -15,6 +15,7 @@ SYNOPSIS
        $prog [ --help ] [ --running | --RUNNING ] [ --latest ]
                [ --tarball-coredumps ] [ --delete-coredumps-after ]
                [ --tarball-results ] [ --delete-results-after ]
+               [ --tarball-uniqueid="<uniqueid>" ]
                [ --no-default-search ] [ --append-coredumps ]
                [ <coredump> | <pattern> ... ]
 
@@ -81,6 +82,11 @@ DESCRIPTION
                to use this option unless you have also specified
                --tarball-results.
 
+       --tarball-uniqueid="<uniqueid>"
+               Normally DATEFORMAT is used to make the tarballs unique
+               but you can use your own unique id in the tarball names
+               such as the Jira issue id.
+
        --no-default-search
                Ignore COREDUMPS from the config files and process only
                coredumps listed on the command line (if any) and/or
@@ -111,6 +117,8 @@ DESCRIPTION
                /tmp/core[-._]\$(hostname)!(*.txt)
 
 NOTES
+       You must be root to use $prog.
+
        The script relies on not only bash, but also recent GNU date and
        gdb with python support.  *BSD operating systems may require
        installation of the 'coreutils' and 'devel/gdb' packagess and minor
@@ -171,6 +179,11 @@ EOF
        exit 1
 }
 
+if [ $EUID -ne 0 ] ; then
+       echo "You must be root to use $prog."
+       exit 1
+fi
+
 running=false
 RUNNING=false
 latest=false
@@ -245,6 +258,9 @@ for a in "$@" ; do
        --append-coredumps)
                append_coredumps=true
                ;;
+       --tarball-uniqueid=*)
+               tarball_uniqueid=${a#*=}
+               ;;
        --help|-*)
                print_help
                ;;
@@ -294,7 +310,7 @@ if [ ${#COREDUMPS[@]} -gt 0 ] && $latest ; then
 fi
 
 # Timestamp to use for output files
-df=$(${DATEFORMAT})
+df=${tarball_uniqueid:-$(${DATEFORMAT})}
 
 if $running || $RUNNING ; then
        # We need to go through some gyrations to find the pid of the running
@@ -321,12 +337,9 @@ if $running || $RUNNING ; then
                        read -p "WARNING:  Taking a core dump of the running asterisk instance will suspend call processing while the dump is saved.  Do you wish to continue? (y/N) " answer
                fi
                if [[ "$answer" =~ ^[Yy] ]] ; then
-                       cf="/tmp/core.asterisk.running.$df"
-                       # We want a consistent coredump so stop the process
-                       # and continue it after the dump is complete.
-                       # kill -STOP $pid
+                       cf="/tmp/core-asterisk-running-$df"
+                       echo "Dumping running asterisk process to $cf"
                        ${GDB} -p $pid -q --batch --ex "gcore $cf" >/dev/null 2>&1
-                       # kill -CONT $pid
                        COREDUMPS+=("$cf")
                else
                        echo "Skipping dump of running process"
@@ -343,17 +356,17 @@ fi
 # and save them to /tmp/.gdbinit
 
 ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
-tail -n +${ss} $0 >/tmp/.gdbinit
+tail -n +${ss} $0 >/tmp/.ast_coredumper.gdbinit
 
 # Now iterate over the coredumps and dump the debugging info
 for i in ${!COREDUMPS[@]} ; do
        cf=${COREDUMPS[$i]}
        echo "Processing $cf"
-       ${GDB} -n --batch -q --ex "source /tmp/.gdbinit" $(which asterisk) "$cf" 2>/dev/null | (
+       ${GDB} -n --batch -q --ex "source /tmp/.ast_coredumper.gdbinit" $(which asterisk) "$cf" 2>/dev/null | (
                of=/dev/null
                while IFS= read line ; do
                        if [[ "$line" =~ !@!@!@!\ ([^\ ]+)\ !@!@!@! ]] ; then
-                               of=$cf.${BASH_REMATCH[1]}
+                               of=${cf}-${BASH_REMATCH[1]}
                                of=${of//:/-}
                                rm -f "$of"
                                echo "Creating $of"
@@ -364,7 +377,7 @@ for i in ${!COREDUMPS[@]} ; do
 done
 
 if $tarball_coredumps ; then
-       tf=/tmp/asterisk.$df.coredumps.tar
+       tf=/tmp/asterisk-$df.coredumps.tar
        echo "Creating $tf.gz"
        for i in ${!COREDUMPS[@]} ; do
                tar -uvf $tf "${COREDUMPS[@]}" 2>/dev/null
@@ -379,17 +392,17 @@ if $delete_coredumps_after ; then
 fi
 
 if $tarball_results ; then
-       tf=/tmp/asterisk.$df.results.tar
+       tf=/tmp/asterisk-$df-results.tar
        echo "Creating $tf.gz"
        for i in ${!COREDUMPS[@]} ; do
-               tar -uvf $tf "${COREDUMPS[$i]//:/-}".{brief,full,thread1,locks}.txt 2>/dev/null
+               tar -uvf $tf "${COREDUMPS[$i]//:/-}"-{brief,full,thread1,locks}.txt 2>/dev/null
        done
        gzip $tf
 fi
 
 if $delete_results_after ; then
        for i in ${!COREDUMPS[@]} ; do
-               rm -rf "${COREDUMPS[$i]//:/-}".{brief,full,thread1,locks}.txt
+               rm -rf "${COREDUMPS[$i]//:/-}"-{brief,full,thread1,locks}.txt
        done
 fi
 
diff --git a/contrib/scripts/ast_loggrabber b/contrib/scripts/ast_loggrabber
new file mode 100755 (executable)
index 0000000..2036d54
--- /dev/null
@@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+# Turn on extended globbing
+shopt -s extglob
+# Bail on any error
+set -e
+
+prog=$(basename $0)
+
+print_help() {
+cat <<EOF
+NAME
+$prog - Gather asterisk log files
+
+SYNOPSIS
+       $prog [ --help ] [ --dateformat="<dateformat>" ]
+               [ --timezone="<timezone>" ] [ --append-logfiles ]
+               [ --tarball-uniqueid="<uniqueid>" ]
+               [ <logfiles> | <pattern> ... ]
+
+DESCRIPTION
+
+       Gathers log files, optionally converts POSIX timestamps
+       to readable format. and creates a tarball.
+
+       Options:
+
+       --help
+               Print this help.
+
+       --dateformat="<dateformat>"
+               A Python strftime format string to be used when converting
+               POSIX timestamps in log files to readable format.  If not
+               specified as an argument or in the config file, no conversion
+               is done.
+
+       --timezone="<timezone>"
+               The timezone to use when converting POSIX timestamps to
+               readable format.  It can be specified in "<continent>/<city>"
+               format or in abbreviation format such as "CST6CDT".  If not
+               specified as an argument or in the config file, the "local"
+               timezone is used.
+
+       --append-logfiles
+               Append any log files specified on the command line to the
+               config file specified ones instead of overriding them.
+
+       --tarball-uniqueid="<uniqueid>"
+               Normally DATEFORMAT is used to make the tarballs unique
+               but you can use your own unique id in the tarball names
+               such as a Jira issue id.
+
+       <logfiles> | <pattern>
+               A list of log files or log file search patterns.  Unless
+               --append-logfiles was specified, these entries will override
+               those specified in the config files.
+
+               If no files are specified on the command line the, value of
+               LOGFILES from ast_debug_tools.conf will be used.  Failing
+               that, the following patterns will be used:
+               /var/log/asterisk/messages*
+               /var/log/asterisk/queue*
+               /var/log/asterisk/debug*
+               /var/log/asterisk/security*
+
+NOTES
+       Any files output will have ':' characters changed to '-'.  This is
+       to facilitate uploading those files to Jira which doesn't like the
+       colons.
+
+FILES
+       /etc/asterisk/ast_debug_tools.conf
+       ~/ast_debug_tools.conf
+       ./ast_debug_tools.conf
+
+       # Readable Local time for the tarball names
+       DATEFORMAT='date +%FT%H-%M-%S%z'
+
+       # A list of log files and/or log file search patterns using the
+       # same syntax as COREDUMPS.
+       #
+       LOGFILES=(/var/log/asterisk/messages* /var/log/asterisk/queue* \\
+               /var/log/asterisk/debug* /var/log/asterisk/security*)
+
+       # $prog converts POSIX timestamps to readable format
+       # using this Python strftime format string.  If not specified
+       # or an empty string, no format covnersion is done.
+       LOG_DATEFORMAT="%m/%d %H:%M:%S.%f"
+
+       # The timezone to use when converting POSIX timestamps to
+       # readable format.  It can be specified in "<continent>/<city>"
+       # format or in abbreviation format such as "CST6CDT".  If not
+       # specified, the "local" timezone is used.
+       # LOG_TIMEZONE=
+
+EOF
+       exit 1
+}
+
+append_logfiles=false
+
+declare -a LOGFILES
+declare -a ARGS_LOGFILES
+
+# Read config files from least important to most important
+[ -f /etc/asterisk/ast_debug_tools.conf ] && source /etc/asterisk/ast_debug_tools.conf
+[ -f ~/ast_debug_tools.conf ] && source ~/ast_debug_tools.conf
+[ -f ./ast_debug_tools.conf ] && source ./ast_debug_tools.conf
+
+if [ ${#LOGFILES[@]} -eq 0 ] ; then
+       LOGFILES+=(/var/log/asterisk/messages* /var/log/asterisk/queue* \
+       /var/log/asterisk/debug* /var/log/asterisk/security*)
+fi
+
+DATEFORMAT=${DATEFORMAT:-'date +%FT%H-%M-%S%z'}
+
+# Use "$@" (with the quotes) so spaces in patterns or
+# file names are preserved.
+# Later on when we have to iterate over LOGFILES, we always
+# use the indexes rather than trying to expand the values of LOGFILES
+# just in case.
+
+for a in "$@" ; do
+       case "$a" in
+       --dateformat=*)
+               LOG_DATEFORMAT=${a#*=}
+               ;;
+       --timezone=*)
+               LOG_TIMEZONE=${a#*=}
+               ;;
+       --append-logfiles)
+               append_logfiles=true
+               ;;
+       --tarball-uniqueid=*)
+               tarball_uniqueid=${a#*=}
+               ;;
+       --help|-*)
+               print_help
+               ;;
+       *)
+               ARGS_LOGFILES+=("$a")
+               # If any files are specified on the command line, ignore those
+               # specified in the config files unless append-logfiles was specified.
+               if ! $append_logfiles ; then
+                       LOGFILES=()
+               fi
+       esac
+done
+
+# append logfiles/patterns specified as command line arguments to LOGFILES.
+for i in ${!ARGS_LOGFILES[@]} ; do
+       LOGFILES+=("${ARGS_LOGFILES[$i]}")
+done
+
+# At this point, all glob entries that match files should be expanded.
+# Any entries that don't exist are probably globs that didn't match anything
+# and need to be pruned.
+
+for i in ${!LOGFILES[@]} ; do
+       if [ ! -f "${LOGFILES[$i]}" ] ; then
+               unset LOGFILES[$i]
+               continue
+       fi
+done
+
+# Sort and weed out any dups
+IFS=$'\x0a'
+readarray -t LOGFILES < <(echo -n "${LOGFILES[*]}" | sort -u )
+unset IFS
+
+if [ "${#LOGFILES[@]}" -eq 0 ] ; then
+       echo "No log files found"
+       print_help
+fi
+
+# Timestamp to use for output files
+df=${tarball_uniqueid:-$(${DATEFORMAT})}
+
+# Extract the Python timestamp conver script from the end of this
+# script and save it to /tmp/.ast_tsconvert.py
+
+ss=`egrep -n "^#@@@SCRIPTSTART@@@" $0 |cut -f1 -d:`
+tail -n +${ss} $0 >/tmp/.ast_tsconvert.py
+
+tmpdir=$(mktemp -d)
+if [ -z "$tmpdir" ] ; then
+       echo "${prog}: Unable to create temporary directory."
+       exit 1
+fi
+trap "rm -rf $tmpdir" EXIT
+tardir=asterisk-${df}.logfiles
+
+# Now iterate over the logfiles
+for i in ${!LOGFILES[@]} ; do
+       lf=${LOGFILES[$i]}
+       destdir="$tmpdir/$tardir/$(dirname $lf)"
+       destfile="$tmpdir/$tardir/$lf"
+       mkdir -p "$destdir" 2>/dev/null || :
+       if [ -n "$LOG_DATEFORMAT" ] ; then
+               echo "Converting $lf"
+               cat "$lf" | python /tmp/.ast_tsconvert.py --format="$LOG_DATEFORMAT" --timezone="$LOG_TIMEZONE" > "${destfile}"
+       else
+               echo "Copying $lf"
+               cp "$lf" "${destfile}"
+       fi
+done
+
+echo "Creating /tmp/$tardir.tar.gz"
+tar -czvf /tmp/$tardir.tar.gz -C $tmpdir $tardir 2>/dev/null
+
+exit
+
+# Be careful editng the inline scripts.
+# They're space-indented.
+
+# We need the python bit because lock_infos isn't
+# a valid symbol in asterisk unless DEBUG_THREADS was
+# used during the compile.  Also, interrupt and continue
+# are only valid for a running program.
+
+#@@@SCRIPTSTART@@@
+import argparse
+import datetime as dt
+import dateutil.tz as tz
+import re
+import sys
+import time
+
+parser = argparse.ArgumentParser(description="Make POSIX timestamps readable")
+parser.add_argument('--format', action='store', required=True)
+parser.add_argument('--timezone', action='store', required=False)
+args=parser.parse_args()
+
+# We only convert timestamps that are at the beginning of a line
+# or are preceeded by a whilespace character or a '['
+rets = re.compile(r'(^|(?<=\s|\[))\d+(\.\d+)?', flags=re.M)
+if args.timezone and len(args.timezone) > 0:
+   tzf = tz.tzfile('/usr/share/zoneinfo/' + args.timezone)
+else:
+   tzf = tz.tzfile('/etc/localtime')
+
+now = time.time()
+a_year_ago = now - (86400.0 * 365)
+
+def convert(match):
+   ts = float(match.group(0))
+   if ts <= now and ts > a_year_ago and len(args.format) > 0:
+      return dt.datetime.fromtimestamp(ts, tzf).strftime(args.format)
+   else:
+      return match.group(0)
+
+while 1:
+   line = sys.stdin.readline()
+   if not line:
+      break
+   print(rets.sub(convert, line))