CI: Add support for coverage processing.

Enable coverage with `./tests/CI/buildAsterisk.sh --coverage`.  This
will cause Asterisk to be compiled with coverage support.  It also
initializes 'before' coverage data for all sources.  Accept
--tested-only to disable modules which are not run by any test.
Enabling coverage also sets tested-only true by default.  To build
everything with coverage enabled use `--coverage --tested-only=0`.

./tests/CI/processCoverage.sh is used to process the coverage and
generate HTML reports.

Fix utils/check_expr2 which failed to compiled with coverage enabled.

Add status output 5 times per stage of astobj2_test_perf to ensure
remote CLI does not timeout when compiled with coverage.  Remote CLI
disconnects if no output is received for 60 seconds.  When coverage is
enabled it takes about 70 seconds for my laptop to run the stages of
this test, so with the change a message is printed every 14 seconds.

Change-Id: I890f7d5665087426ad7d3e363187691b9afc2222

diff --git a/tests/CI/buildAsterisk.sh b/tests/CI/buildAsterisk.sh
index e9d3013..60f0917 100755 (executable)
--- a/tests/CI/buildAsterisk.sh
+++ b/tests/CI/buildAsterisk.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 CIDIR=$(dirname $(readlink -fn $0))
+COVERAGE=0
 REF_DEBUG=0
 source $CIDIR/ci.functions
 
@@ -28,6 +29,15 @@ gen_mods() {
 
 [ x"$OUTPUT_DIR" != x ] && mkdir -p "$OUTPUT_DIR" 2> /dev/null
 
+if [ -z $TESTED_ONLY ]; then
+       # Skip building untested modules by default if coverage is enabled.
+       TESTED_ONLY=$COVERAGE
+fi
+
+if [ -z $LCOV_DIR ]; then
+       LCOV_DIR="${OUTPUT_DIR:+${OUTPUT_DIR}/}lcov"
+fi
+
 if [ x"$CACHE_DIR" != x ] ; then
        mkdir -p $CACHE_DIR/sounds $CACHE_DIR/externals 2> /dev/null
 fi
@@ -65,6 +75,9 @@ common_config_args="--prefix=/usr ${_libdir:+--libdir=${_libdir}} --sysconfdir=/
 $PKGCONFIG 'jansson >= 2.11' || common_config_args+=" --with-jansson-bundled"
 common_config_args+=" ${CACHE_DIR:+--with-sounds-cache=${CACHE_DIR}/sounds --with-externals-cache=${CACHE_DIR}/externals}"
 common_config_args+=" --enable-dev-mode"
+if [ $COVERAGE -eq 1 ] ; then
+       common_config_args+=" --enable-coverage"
+fi
 export WGET_EXTRA_ARGS="--quiet"
 
 runner ./configure ${common_config_args} > ${OUTPUT_DIR:+${OUTPUT_DIR}/}configure.txt
@@ -83,6 +96,25 @@ cat_enables+=" MENUSELECT_PBX MENUSELECT_RES MENUSELECT_UTILS MENUSELECT_TESTS"
 runner menuselect/menuselect `gen_cats enable $cat_enables` menuselect.makeopts
 
 mod_disables="res_digium_phone chan_vpb"
+if [ $TESTED_ONLY -eq 1 ] ; then
+       # These modules are not tested at all.  They are loaded but nothing is ever done
+       # with them, no testsuite tests depend on them.
+       mod_disables+=" app_adsiprog app_alarmreceiver app_celgenuserevent app_db app_dictate"
+       mod_disables+=" app_dumpchan app_externalivr app_festival app_getcpeid app_ices app_image"
+       mod_disables+=" app_jack app_milliwatt app_minivm app_morsecode app_mp3 app_nbscat app_privacy"
+       mod_disables+=" app_readexten app_sms app_speech_utils app_test app_url app_waitforring"
+       mod_disables+=" app_waitforsilence app_waituntil app_zapateller"
+       mod_disables+=" cdr_adaptive_odbc cdr_custom cdr_manager cdr_odbc cdr_pgsql cdr_radius"
+       mod_disables+=" cdr_syslog cdr_tds"
+       mod_disables+=" cel_odbc cel_pgsql cel_radius cel_sqlite3_custom cel_tds"
+       mod_disables+=" chan_alsa chan_console chan_mgcp chan_motif chan_oss chan_rtp chan_skinny chan_unistim"
+       mod_disables+=" func_frame_trace func_pitchshift func_speex func_volume func_dialgroup"
+       mod_disables+=" func_periodic_hook func_sprintf func_enum func_extstate func_sysinfo func_iconv"
+       mod_disables+=" func_callcompletion func_version func_rand func_sha1 func_module func_md5"
+       mod_disables+=" pbx_dundi pbx_loopback"
+       mod_disables+=" res_ael_share res_calendar res_config_ldap res_config_pgsql res_corosync"
+       mod_disables+=" res_http_post res_pktccops res_rtp_multicast res_snmp res_xmpp"
+fi
 [ "$BRANCH_NAME" == "master" ] && mod_disables+=" codec_opus codec_silk codec_g729a codec_siren7 codec_siren14"
 runner menuselect/menuselect `gen_mods disable $mod_disables` menuselect.makeopts
 
@@ -93,6 +125,21 @@ runner menuselect/menuselect `gen_mods enable $mod_enables` menuselect.makeopts
 
 runner ${MAKE} -j8 || runner ${MAKE} -j1 NOISY_BUILD=yes
 
+runner rm -f ${LCOV_DIR}/*.info
+if [ $COVERAGE -eq 1 ] ; then
+       runner mkdir -p ${LCOV_DIR}
+
+       # Zero counter data
+       runner lcov --quiet --directory . --zerocounters
+
+       # Branch coverage is not supported by --initial.  Disable to suppresses a notice
+       # printed if it was enabled in lcovrc.
+       # This initial capture ensures any module which was built but never loaded is
+       # reported with 0% coverage for all sources.
+       runner lcov --quiet --directory . --no-external --capture --initial --rc lcov_branch_coverage=0 \
+               --output-file ${LCOV_DIR}/initial.info
+fi
+
 ALEMBIC=$(which alembic 2>/dev/null || : )
 if [ x"$ALEMBIC" = x ] ; then
        echo "Alembic not installed"

diff --git a/tests/CI/processCoverage.sh b/tests/CI/processCoverage.sh
new file mode 100755 (executable)
index 0000000..b34ebee
--- /dev/null
+++ b/tests/CI/processCoverage.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+
+if [ ! -r main/asterisk.gcno ]; then
+       # Coverage is not enabled.
+       exit 0
+fi
+
+if [ -z $LCOV_DIR ]; then
+       LCOV_DIR="${OUTPUT_DIR:+${OUTPUT_DIR}/}lcov"
+fi
+
+if [ -z $COVERAGE_DIR ]; then
+       COVERAGE_DIR="${OUTPUT_DIR:+${OUTPUT_DIR}/}coverage"
+fi
+
+if [ -z $ASTERISK_VERSION ]; then
+       ASTERISK_VERSION=$(./build_tools/make_version .)
+fi
+
+set -x
+# Capture counter data from testing
+lcov --no-external --capture --directory . --output-file ${LCOV_DIR}/tested.info > /dev/null
+
+# Combine initial and tested data.
+lcov \
+       --add-tracefile ${LCOV_DIR}/initial.info \
+       --add-tracefile ${LCOV_DIR}/tested.info \
+       --output-file ${LCOV_DIR}/combined.info > /dev/null
+
+# We don't care about coverage reporting for tests, utils or third-party.
+lcov --remove ${LCOV_DIR}/combined.info \
+               "${PWD}/main/dns_test.*" \
+               "${PWD}/main/test.*" \
+               "${PWD}/tests/*" \
+               "${PWD}/utils/*" \
+               "${PWD}/third-party/*" \
+       --output-file ${LCOV_DIR}/filtered.info > /dev/null
+
+# Generate HTML coverage report.
+mkdir -p ${COVERAGE_DIR}
+genhtml --prefix ${PWD} --ignore-errors source ${LCOV_DIR}/filtered.info \
+       --legend --title "Asterisk ${ASTERISK_VERSION}" --output-directory=${COVERAGE_DIR} > /dev/null

diff --git a/tests/test_astobj2.c b/tests/test_astobj2.c
index 827ebb5..3158a66 100644 (file)
--- a/tests/test_astobj2.c
+++ b/tests/test_astobj2.c
@@ -1994,10 +1994,15 @@ static enum ast_test_result_state testloop(struct ast_test *test,
 {
        int res = AST_TEST_PASS;
        int i;
+       int reportcount = iterations / 5;
        struct timeval start;
 
        start = ast_tvnow();
        for (i = 1 ; i <= iterations && res == AST_TEST_PASS ; i++) {
+               if (i % reportcount == 0 && i != iterations) {
+                       ast_test_status_update(test, "%5.2fK traversals, %9s\n",
+                               i / 1000.0, test_container2str(type));
+               }
                res = test_performance(test, type, copt);
        }
        ast_test_status_update(test, "%5.2fK traversals, %9s : %5lu ms\n",

diff --git a/utils/Makefile b/utils/Makefile
index 6bd33da..2ec1b91 100644 (file)
--- a/utils/Makefile
+++ b/utils/Makefile
@@ -183,7 +183,7 @@ check_expr2: $(ASTTOPDIR)/main/ast_expr2f.c $(ASTTOPDIR)/main/ast_expr2.c $(ASTT
        $(ECHO_PREFIX) echo "   [CC] ast_expr2.c -> ast_expr2z.o"
        $(CC) -g -c -I$(ASTTOPDIR)/include -DSTANDALONE2 $(ASTTOPDIR)/main/ast_expr2.c -o ast_expr2z.o
        $(ECHO_PREFIX) echo "   [LD] ast_expr2fz.o ast_expr2z.o  -> check_expr2"
-       $(CC) -g -o check_expr2 ast_expr2fz.o ast_expr2z.o astmm.o -lm
+       $(CC) -g -o check_expr2 ast_expr2fz.o ast_expr2z.o astmm.o -lm $(_ASTLDFLAGS)
        $(ECHO_PREFIX) echo "   [RM] ast_expr2fz.o ast_expr2z.o"
        rm ast_expr2z.o ast_expr2fz.o
        ./check_expr2 expr2.testinput