2 * Asterisk -- An open source telephony toolkit.
4 * Copyright (C) 2009-2010, Digium, Inc.
6 * David Vossel <dvossel@digium.com>
7 * Russell Bryant <russell@digium.com>
9 * See http://www.asterisk.org for more information about
10 * the Asterisk project. Please do not directly contact
11 * any of the maintainers of this project for assistance;
12 * the project provides a web site, mailing lists and IRC
13 * channels for your use.
15 * This program is free software, distributed under the terms of
16 * the GNU General Public License Version 2. See the LICENSE file
17 * at the top of the source tree.
22 * \brief Unit Test Framework
24 * \author David Vossel <dvossel@digium.com>
25 * \author Russell Bryant <russell@digium.com>
30 ASTERISK_FILE_VERSION(__FILE__, "$Revision$");
32 #include "asterisk/_private.h"
35 #include "asterisk/test.h"
36 #include "asterisk/logger.h"
37 #include "asterisk/linkedlists.h"
38 #include "asterisk/utils.h"
39 #include "asterisk/cli.h"
40 #include "asterisk/term.h"
41 #include "asterisk/version.h"
42 #include "asterisk/paths.h"
43 #include "asterisk/time.h"
45 /*! This array corresponds to the values defined in the ast_test_state enum */
46 static const char * const test_result2str[] = {
47 [AST_TEST_NOT_RUN] = "NOT RUN",
48 [AST_TEST_PASS] = "PASS",
49 [AST_TEST_FAIL] = "FAIL",
52 /*! holds all the information pertaining to a single defined test */
54 struct ast_test_info info; /*!< holds test callback information */
56 * \brief Test defined status output from last execution
58 struct ast_str *status_str;
60 * \brief CLI arguments, if tests being run from the CLI
62 * If this is set, status updates from the tests will be sent to the
63 * CLI in addition to being saved off in status_str.
65 struct ast_cli_args *cli;
66 enum ast_test_result_state state; /*!< current test state */
67 unsigned int time; /*!< time in ms test took */
68 ast_test_cb_t *cb; /*!< test callback function */
69 AST_LIST_ENTRY(ast_test) entry;
72 /*! global structure containing both total and last test execution results */
73 static struct ast_test_execute_results {
74 unsigned int total_tests; /*!< total number of tests, regardless if they have been executed or not */
75 unsigned int total_passed; /*!< total number of executed tests passed */
76 unsigned int total_failed; /*!< total number of executed tests failed */
77 unsigned int total_time; /*!< total time of all executed tests */
78 unsigned int last_passed; /*!< number of passed tests during last execution */
79 unsigned int last_failed; /*!< number of failed tests during last execution */
80 unsigned int last_time; /*!< total time of the last test execution */
86 TEST_NAME_CATEGORY = 2,
89 /*! List of registered test definitions */
90 static AST_LIST_HEAD_STATIC(tests, ast_test);
92 static struct ast_test *test_alloc(ast_test_cb_t *cb);
93 static struct ast_test *test_free(struct ast_test *test);
94 static int test_insert(struct ast_test *test);
95 static struct ast_test *test_remove(ast_test_cb_t *cb);
96 static int test_cat_cmp(const char *cat1, const char *cat2);
98 int __ast_test_status_update(const char *file, const char *func, int line,
99 struct ast_test *test, const char *fmt, ...)
101 struct ast_str *buf = NULL;
104 if (!(buf = ast_str_create(128))) {
109 ast_str_set_va(&buf, 0, fmt, ap);
113 ast_cli(test->cli->fd, "[%s:%s:%d]: %s",
114 file, func, line, ast_str_buffer(buf));
117 ast_str_append(&test->status_str, 0, "[%s:%s:%d]: %s",
118 file, func, line, ast_str_buffer(buf));
125 int ast_test_register(ast_test_cb_t *cb)
127 struct ast_test *test;
130 ast_log(LOG_WARNING, "Attempted to register test without all required information\n");
134 if (!(test = test_alloc(cb))) {
138 if (test_insert(test)) {
146 int ast_test_unregister(ast_test_cb_t *cb)
148 struct ast_test *test;
150 if (!(test = test_remove(cb))) {
151 return -1; /* not found */
161 * \brief executes a single test, storing the results in the test->result structure.
163 * \note The last_results structure which contains global statistics about test execution
164 * must be updated when using this function. See use in test_execute_multiple().
166 static void test_execute(struct ast_test *test)
168 struct timeval begin;
170 ast_str_reset(test->status_str);
173 test->state = test->cb(&test->info, TEST_EXECUTE, test);
174 test->time = ast_tvdiff_ms(ast_tvnow(), begin);
177 static void test_xml_entry(struct ast_test *test, FILE *f)
179 if (!f || !test || test->state == AST_TEST_NOT_RUN) {
183 fprintf(f, "\t<testcase time=\"%d.%d\" name=\"%s%s\"%s>\n",
184 test->time / 1000, test->time % 1000,
185 test->info.category, test->info.name,
186 test->state == AST_TEST_PASS ? "/" : "");
188 if (test->state == AST_TEST_FAIL) {
189 fprintf(f, "\t\t<failure>%s</failure>\n",
190 S_OR(ast_str_buffer(test->status_str), "NA"));
191 fprintf(f, "\t</testcase>\n");
196 static void test_txt_entry(struct ast_test *test, FILE *f)
202 fprintf(f, "\nName: %s\n", test->info.name);
203 fprintf(f, "Category: %s\n", test->info.category);
204 fprintf(f, "Summary: %s\n", test->info.summary);
205 fprintf(f, "Description: %s\n", test->info.description);
206 fprintf(f, "Result: %s\n", test_result2str[test->state]);
207 if (test->state != AST_TEST_NOT_RUN) {
208 fprintf(f, "Time: %d\n", test->time);
210 if (test->state == AST_TEST_FAIL) {
211 fprintf(f, "Error Description: %s\n\n", S_OR(ast_str_buffer(test->status_str), "NA"));
217 * \brief Executes registered unit tests
219 * \param name of test to run (optional)
220 * \param test category to run (optional)
221 * \param cli args for cli test updates (optional)
223 * \return number of tests executed.
225 * \note This function has three modes of operation
226 * -# When given a name and category, a matching individual test will execute if found.
227 * -# When given only a category all matching tests within that category will execute.
228 * -# If given no name or category all registered tests will execute.
230 static int test_execute_multiple(const char *name, const char *category, struct ast_cli_args *cli)
232 char result_buf[32] = { 0 };
233 struct ast_test *test = NULL;
234 enum test_mode mode = TEST_ALL; /* 3 modes, 0 = run all, 1 = only by category, 2 = only by name and category */
238 if (!ast_strlen_zero(category)) {
239 if (!ast_strlen_zero(name)) {
240 mode = TEST_NAME_CATEGORY;
242 mode = TEST_CATEGORY;
246 AST_LIST_LOCK(&tests);
247 /* clear previous execution results */
248 memset(&last_results, 0, sizeof(last_results));
249 AST_LIST_TRAVERSE(&tests, test, entry) {
254 if (!test_cat_cmp(test->info.category, category)) {
258 case TEST_NAME_CATEGORY:
259 if (!(test_cat_cmp(test->info.category, category)) && !(strcmp(test->info.name, name))) {
269 ast_cli(cli->fd, "START %s - %s \n", test->info.category, test->info.name);
272 /* set the test status update argument. it is ok if cli is NULL */
275 /* execute the test and save results */
280 /* update execution specific counts here */
281 last_results.last_time += test->time;
282 if (test->state == AST_TEST_PASS) {
283 last_results.last_passed++;
284 } else if (test->state == AST_TEST_FAIL) {
285 last_results.last_failed++;
289 term_color(result_buf,
290 test_result2str[test->state],
291 (test->state == AST_TEST_FAIL) ? COLOR_RED : COLOR_GREEN,
294 ast_cli(cli->fd, "END %s - %s Time: %s%dms Result: %s\n",
297 test->time ? "" : "<",
298 test->time ? test->time : 1,
303 /* update total counts as well during this iteration
304 * even if the current test did not execute this time */
305 last_results.total_time += test->time;
306 last_results.total_tests++;
307 if (test->state != AST_TEST_NOT_RUN) {
308 if (test->state == AST_TEST_PASS) {
309 last_results.total_passed++;
311 last_results.total_failed++;
315 res = last_results.last_passed + last_results.last_failed;
316 AST_LIST_UNLOCK(&tests);
323 * \brief Generate test results.
325 * \param name of test result to generate (optional)
326 * \param test category to generate (optional)
327 * \param path to xml file to generate. (optional)
328 * \param path to txt file to generate, (optional)
333 * \note This function has three modes of operation.
334 * -# When given both a name and category, results will be generated for that single test.
335 * -# When given only a category, results for every test within the category will be generated.
336 * -# When given no name or category, results for every registered test will be generated.
338 * In order for the results to be generated, an xml and or txt file path must be provided.
340 static int test_generate_results(const char *name, const char *category, const char *xml_path, const char *txt_path)
342 enum test_mode mode = TEST_ALL; /* 0 generate all, 1 generate by category only, 2 generate by name and category */
343 FILE *f_xml = NULL, *f_txt = NULL;
345 struct ast_test *test = NULL;
347 /* verify at least one output file was given */
348 if (ast_strlen_zero(xml_path) && ast_strlen_zero(txt_path)) {
352 /* define what mode is to be used */
353 if (!ast_strlen_zero(category)) {
354 if (!ast_strlen_zero(name)) {
355 mode = TEST_NAME_CATEGORY;
357 mode = TEST_CATEGORY;
360 /* open files for writing */
361 if (!ast_strlen_zero(xml_path)) {
362 if (!(f_xml = fopen(xml_path, "w"))) {
363 ast_log(LOG_WARNING, "Could not open file %s for xml test results\n", xml_path);
368 if (!ast_strlen_zero(txt_path)) {
369 if (!(f_txt = fopen(txt_path, "w"))) {
370 ast_log(LOG_WARNING, "Could not open file %s for text output of test results\n", txt_path);
376 AST_LIST_LOCK(&tests);
377 /* xml header information */
380 * http://confluence.atlassian.com/display/BAMBOO/JUnit+parsing+in+Bamboo
382 fprintf(f_xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
383 fprintf(f_xml, "<testsuite errors=\"0\" time=\"%d.%d\" tests=\"%d\" "
384 "name=\"AsteriskUnitTests\">\n",
385 last_results.total_time / 1000, last_results.total_time % 1000,
386 last_results.total_tests);
387 fprintf(f_xml, "\t<properties>\n");
388 fprintf(f_xml, "\t\t<property name=\"version\" value=\"%s\"/>\n", ASTERISK_VERSION);
389 fprintf(f_xml, "\t</properties>\n");
392 /* txt header information */
394 fprintf(f_txt, "Asterisk Version: %s\n", ASTERISK_VERSION);
395 fprintf(f_txt, "Asterisk Version Number: %d\n", ASTERISK_VERSION_NUM);
396 fprintf(f_txt, "Number of Tests: %d\n", last_results.total_tests);
397 fprintf(f_txt, "Number of Tests Executed: %d\n", (last_results.total_passed + last_results.total_failed));
398 fprintf(f_txt, "Passed Tests: %d\n", last_results.total_passed);
399 fprintf(f_txt, "Failed Tests: %d\n", last_results.total_failed);
400 fprintf(f_txt, "Total Execution Time: %d\n", last_results.total_time);
403 /* export each individual test */
404 AST_LIST_TRAVERSE(&tests, test, entry) {
407 if (!test_cat_cmp(test->info.category, category)) {
408 test_xml_entry(test, f_xml);
409 test_txt_entry(test, f_txt);
412 case TEST_NAME_CATEGORY:
413 if (!(strcmp(test->info.category, category)) && !(strcmp(test->info.name, name))) {
414 test_xml_entry(test, f_xml);
415 test_txt_entry(test, f_txt);
419 test_xml_entry(test, f_xml);
420 test_txt_entry(test, f_txt);
423 AST_LIST_UNLOCK(&tests);
427 fprintf(f_xml, "</testsuite>\n");
439 * \brief adds test to container sorted first by category then by name
444 static int test_insert(struct ast_test *test)
446 /* This is a slow operation that may need to be optimized in the future
447 * as the test framework expands. At the moment we are doing string
448 * comparisons on every item within the list to insert in sorted order. */
450 AST_LIST_LOCK(&tests);
451 AST_LIST_INSERT_SORTALPHA(&tests, test, entry, info.category);
452 AST_LIST_UNLOCK(&tests);
459 * \brief removes test from container
461 * \return ast_test removed from list on success, or NULL on failure
463 static struct ast_test *test_remove(ast_test_cb_t *cb)
465 struct ast_test *cur = NULL;
467 AST_LIST_LOCK(&tests);
468 AST_LIST_TRAVERSE_SAFE_BEGIN(&tests, cur, entry) {
470 AST_LIST_REMOVE_CURRENT(entry);
474 AST_LIST_TRAVERSE_SAFE_END;
475 AST_LIST_UNLOCK(&tests);
481 * \brief compares two test categories to determine if cat1 resides in cat2
485 * \retval non-zero false
488 static int test_cat_cmp(const char *cat1, const char *cat2)
493 if (!cat1 || !cat2) {
504 return strncmp(cat1, cat2, len2) ? 1 : 0;
509 * \brief free an ast_test object and all it's data members
511 static struct ast_test *test_free(struct ast_test *test)
517 ast_free(test->status_str);
525 * \brief allocate an ast_test object.
527 static struct ast_test *test_alloc(ast_test_cb_t *cb)
529 struct ast_test *test;
531 if (!cb || !(test = ast_calloc(1, sizeof(*test)))) {
537 test->cb(&test->info, TEST_INIT, test);
539 if (ast_strlen_zero(test->info.name)) {
540 ast_log(LOG_WARNING, "Test has no name, test registration refused.\n");
541 return test_free(test);
544 if (ast_strlen_zero(test->info.category)) {
545 ast_log(LOG_WARNING, "Test %s has no category, test registration refused.\n",
547 return test_free(test);
550 if (ast_strlen_zero(test->info.summary)) {
551 ast_log(LOG_WARNING, "Test %s/%s has no summary, test registration refused.\n",
552 test->info.category, test->info.name);
553 return test_free(test);
556 if (ast_strlen_zero(test->info.description)) {
557 ast_log(LOG_WARNING, "Test %s/%s has no description, test registration refused.\n",
558 test->info.category, test->info.name);
559 return test_free(test);
562 if (!(test->status_str = ast_str_create(128))) {
563 return test_free(test);
569 static char *complete_test_category(const char *line, const char *word, int pos, int state)
572 int wordlen = strlen(word);
574 struct ast_test *test;
576 AST_LIST_LOCK(&tests);
577 AST_LIST_TRAVERSE(&tests, test, entry) {
578 if (!strncasecmp(word, test->info.category, wordlen) && ++which > state) {
579 ret = ast_strdup(test->info.category);
583 AST_LIST_UNLOCK(&tests);
587 static char *complete_test_name(const char *line, const char *word, int pos, int state, const char *category)
590 int wordlen = strlen(word);
592 struct ast_test *test;
594 AST_LIST_LOCK(&tests);
595 AST_LIST_TRAVERSE(&tests, test, entry) {
596 if (!test_cat_cmp(test->info.category, category) && (!strncasecmp(word, test->info.name, wordlen) && ++which > state)) {
597 ret = ast_strdup(test->info.name);
601 AST_LIST_UNLOCK(&tests);
606 static char *test_cli_show_registered(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
608 #define FORMAT "%-25.25s %-30.30s %-40.40s %-13.13s\n"
609 static const char * const option1[] = { "all", "category", NULL };
610 static const char * const option2[] = { "name", NULL };
611 struct ast_test *test = NULL;
615 e->command = "test show registered";
618 "Usage: 'test show registered' can be used in three ways.\n"
619 " 1. 'test show registered all' shows all registered tests\n"
620 " 2. 'test show registered category [test category]' shows all tests in the given\n"
622 " 3. 'test show registered category [test category] name [test name]' shows all\n"
623 " tests in a given category matching a given name\n";
627 return ast_cli_complete(a->word, option1, a->n);
630 return complete_test_category(a->line, a->word, a->pos, a->n);
633 return ast_cli_complete(a->word, option2, a->n);
636 return complete_test_name(a->line, a->word, a->pos, a->n, a->argv[3]);
640 if ((a->argc < 4) || (a->argc == 6) || (a->argc > 7) ||
641 ((a->argc == 4) && strcmp(a->argv[3], "all")) ||
642 ((a->argc == 7) && strcmp(a->argv[5], "name"))) {
643 return CLI_SHOWUSAGE;
645 ast_cli(a->fd, FORMAT, "Category", "Name", "Summary", "Test Result");
646 ast_cli(a->fd, FORMAT, "--------", "----", "-------", "-----------");
647 AST_LIST_LOCK(&tests);
648 AST_LIST_TRAVERSE(&tests, test, entry) {
649 if ((a->argc == 4) ||
650 ((a->argc == 5) && !test_cat_cmp(test->info.category, a->argv[4])) ||
651 ((a->argc == 7) && !strcmp(test->info.category, a->argv[4]) && !strcmp(test->info.name, a->argv[6]))) {
653 ast_cli(a->fd, FORMAT, test->info.category, test->info.name,
654 test->info.summary, test_result2str[test->state]);
658 AST_LIST_UNLOCK(&tests);
659 ast_cli(a->fd, FORMAT, "--------", "----", "-------", "-----------");
660 ast_cli(a->fd, "\n%d Registered Tests Matched\n", count);
668 static char *test_cli_execute_registered(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
670 static const char * const option1[] = { "all", "category", NULL };
671 static const char * const option2[] = { "name", NULL };
675 e->command = "test execute";
677 "Usage: test execute can be used in three ways.\n"
678 " 1. 'test execute all' runs all registered tests\n"
679 " 2. 'test execute category [test category]' runs all tests in the given\n"
681 " 3. 'test execute category [test category] name [test name]' runs all\n"
682 " tests in a given category matching a given name\n";
686 return ast_cli_complete(a->word, option1, a->n);
689 return complete_test_category(a->line, a->word, a->pos, a->n);
692 return ast_cli_complete(a->word, option2, a->n);
695 return complete_test_name(a->line, a->word, a->pos, a->n, a->argv[3]);
700 if (a->argc < 3|| a->argc > 6) {
701 return CLI_SHOWUSAGE;
704 if ((a->argc == 3) && !strcmp(a->argv[2], "all")) { /* run all registered tests */
705 ast_cli(a->fd, "Running all available tests...\n\n");
706 test_execute_multiple(NULL, NULL, a);
707 } else if (a->argc == 4) { /* run only tests within a category */
708 ast_cli(a->fd, "Running all available tests matching category %s\n\n", a->argv[3]);
709 test_execute_multiple(NULL, a->argv[3], a);
710 } else if (a->argc == 6) { /* run only a single test matching the category and name */
711 ast_cli(a->fd, "Running all available tests matching category %s and name %s\n\n", a->argv[3], a->argv[5]);
712 test_execute_multiple(a->argv[5], a->argv[3], a);
714 return CLI_SHOWUSAGE;
717 AST_LIST_LOCK(&tests);
718 if (!(last_results.last_passed + last_results.last_failed)) {
719 ast_cli(a->fd, "--- No Tests Found! ---\n");
721 ast_cli(a->fd, "\n%d Test(s) Executed %d Passed %d Failed\n",
722 (last_results.last_passed + last_results.last_failed),
723 last_results.last_passed,
724 last_results.last_failed);
725 AST_LIST_UNLOCK(&tests);
733 static char *test_cli_show_results(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
735 #define FORMAT_RES_ALL1 "%s%s %-30.30s %-25.25s %-10.10s\n"
736 #define FORMAT_RES_ALL2 "%s%s %-30.30s %-25.25s %s%ums\n"
737 static const char * const option1[] = { "all", "failed", "passed", NULL };
738 char result_buf[32] = { 0 };
739 struct ast_test *test = NULL;
742 int mode; /* 0 for show all, 1 for show fail, 2 for show passed */
746 e->command = "test show results";
748 "Usage: test show results can be used in three ways\n"
749 " 1. 'test show results all' Displays results for all executed tests.\n"
750 " 2. 'test show results passed' Displays results for all passed tests.\n"
751 " 3. 'test show results failed' Displays results for all failed tests.\n";
755 return ast_cli_complete(a->word, option1, a->n);
762 return CLI_SHOWUSAGE;
763 } else if (!strcmp(a->argv[3], "passed")) {
765 } else if (!strcmp(a->argv[3], "failed")) {
767 } else if (!strcmp(a->argv[3], "all")) {
770 return CLI_SHOWUSAGE;
773 ast_cli(a->fd, FORMAT_RES_ALL1, "Result", "", "Name", "Category", "Time");
774 AST_LIST_LOCK(&tests);
775 AST_LIST_TRAVERSE(&tests, test, entry) {
776 if (test->state == AST_TEST_NOT_RUN) {
779 test->state == AST_TEST_FAIL ? failed++ : passed++;
780 if (!mode || ((mode == 1) && (test->state == AST_TEST_FAIL)) || ((mode == 2) && (test->state == AST_TEST_PASS))) {
781 /* give our results pretty colors */
782 term_color(result_buf, test_result2str[test->state],
783 (test->state == AST_TEST_FAIL) ? COLOR_RED : COLOR_GREEN,
784 0, sizeof(result_buf));
786 ast_cli(a->fd, FORMAT_RES_ALL2,
791 test->time ? " " : "<",
792 test->time ? test->time : 1);
795 AST_LIST_UNLOCK(&tests);
797 ast_cli(a->fd, "%d Test(s) Executed %d Passed %d Failed\n", (failed + passed), passed, failed);
804 static char *test_cli_generate_results(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
806 static const char * const option[] = { "xml", "txt", NULL };
807 const char *file = NULL;
808 const char *type = "";
811 struct ast_str *buf = NULL;
812 struct timeval time = ast_tvnow();
816 e->command = "test generate results";
818 "Usage: 'test generate results'\n"
819 " Generates test results in either xml or txt format. An optional \n"
820 " file path may be provided to specify the location of the xml or\n"
822 " \nExample usage:\n"
823 " 'test generate results xml' this writes to a default file\n"
824 " 'test generate results xml /path/to/file.xml' writes to specified file\n";
828 return ast_cli_complete(a->word, option, a->n);
834 if (a->argc < 4 || a->argc > 5) {
835 return CLI_SHOWUSAGE;
836 } else if (!strcmp(a->argv[3], "xml")) {
839 } else if (!strcmp(a->argv[3], "txt")) {
842 return CLI_SHOWUSAGE;
848 if (!(buf = ast_str_create(256))) {
851 ast_str_set(&buf, 0, "%s/asterisk_test_results-%ld.%s", ast_config_AST_LOG_DIR, (long) time.tv_sec, type);
853 file = ast_str_buffer(buf);
857 res = test_generate_results(NULL, NULL, file, NULL);
859 res = test_generate_results(NULL, NULL, NULL, file);
863 ast_cli(a->fd, "Results Generated Successfully: %s\n", S_OR(file, ""));
865 ast_cli(a->fd, "Results Could Not Be Generated: %s\n", S_OR(file, ""));
876 static struct ast_cli_entry test_cli[] = {
877 AST_CLI_DEFINE(test_cli_show_registered, "show registered tests"),
878 AST_CLI_DEFINE(test_cli_execute_registered, "execute registered tests"),
879 AST_CLI_DEFINE(test_cli_show_results, "show last test results"),
880 AST_CLI_DEFINE(test_cli_generate_results, "generate test results to file"),
882 #endif /* TEST_FRAMEWORK */
886 #ifdef TEST_FRAMEWORK
887 /* Register cli commands */
888 ast_cli_register_multiple(test_cli, ARRAY_LEN(test_cli));