res_prometheus: Add Asterisk channel metrics
authorMatt Jordan <mjordan@digium.com>
Fri, 3 May 2019 00:45:27 +0000 (19:45 -0500)
committerMatt Jordan <mjordan@digium.com>
Tue, 21 May 2019 16:03:13 +0000 (11:03 -0500)
This patch adds basic Asterisk channel statistics to the res_prometheus
module. This includes:

* asterisk_calls_sum: A running sum of the total number of
  processed calls

* asterisk_calls_count: The current number of calls

* asterisk_channels_count: The current number of channels

* asterisk_channels_state: The state of any particular channel

* asterisk_channels_duration_seconds: How long a channel has existed,
  in seconds

In all cases, enough information is provided with each channel metric
to determine a unique instance of Asterisk that provided the data, as
well as the name, type, unique ID, and - if present - linked ID of each
channel.

ASTERISK-28403

Change-Id: I0db306ec94205d4f58d1e7fbabfe04b185869f59

configs/samples/prometheus.conf.sample
include/asterisk/res_prometheus.h
res/Makefile
res/prometheus/channels.c [new file with mode: 0644]
res/prometheus/prometheus_internal.h [new file with mode: 0644]
res/res_prometheus.c
tests/test_res_prometheus.c

index 63e9bd6..3ee9290 100644 (file)
@@ -3,12 +3,9 @@
 ;
 
 ;
-; Note that this configuration file is consumed by res_prometheus, which
-; provides core functionality for serving up Asterisk statistics to a
-; Prometheus server. By default, this only includes basic information about
-; the Asterisk instance that is running. Additional modules can be loaded to
-; provide specific statistics. In all cases, configuration of said statistics
-; is done through this configuration file.
+; This configuration file is consumed by res_prometheus, which
+; provides the functionality for serving up Asterisk statistics to a
+; Prometheus server.
 ;
 ; Because Prometheus scrapes statistics from HTTP servers, this module requires
 ; Asterisk's built-in HTTP server to be enabled and configured properly.
index cf62b7b..4b6ce16 100644 (file)
@@ -84,6 +84,36 @@ struct prometheus_general_config {
 };
 
 /*!
+ * \brief A function table for a metrics provider
+ *
+ * \details
+ * It's generally nice to separate out things that provide metrics
+ * from the core of this module. For those that want to be notified
+ * when things happen in the core module, they can provide an instance
+ * of this function table using \c prometheus_metrics_provider_register
+ * and be notified when module affecting changes occur.
+ */
+struct prometheus_metrics_provider {
+       /*!
+        * \brief Handy name of the provider for debugging purposes
+        */
+       const char *name;
+       /*!
+        * \brief Reload callback
+        *
+        * \param config The reloaded config
+        *
+        * \retval 0 success
+        * \retval -1 error
+        */
+       int (* const reload_cb)(struct prometheus_general_config *config);
+       /*!
+        * \brief Unload callback.
+        */
+       void (* const unload_cb)(void);
+};
+
+/*!
  * \brief Prometheus metric type
  *
  * \note
@@ -438,6 +468,13 @@ int prometheus_callback_register(struct prometheus_callback *callback);
 void prometheus_callback_unregister(struct prometheus_callback *callback);
 
 /*!
+ * \brief Register a metrics provider
+ *
+ * \param provider The provider function table to register
+ */
+void prometheus_metrics_provider_register(const struct prometheus_metrics_provider *provider);
+
+/*!
  * \brief Retrieve the current configuration of the module
  *
  * \note
index 5ff38ad..78410ad 100644 (file)
@@ -66,6 +66,7 @@ $(call MOD_ADD_C,res_stasis,$(wildcard stasis/*.c))
 $(call MOD_ADD_C,res_snmp,snmp/agent.c)
 $(call MOD_ADD_C,res_parking,$(wildcard parking/*.c))
 $(call MOD_ADD_C,res_pjsip,$(wildcard res_pjsip/*.c))
+$(call MOD_ADD_C,res_prometheus,$(wildcard prometheus/*.c))
 $(call MOD_ADD_C,res_ari,ari/cli.c ari/config.c ari/ari_websockets.c)
 $(call MOD_ADD_C,res_ari_model,ari/ari_model_validators.c)
 $(call MOD_ADD_C,res_stasis_recording,stasis_recording/stored.c)
diff --git a/res/prometheus/channels.c b/res/prometheus/channels.c
new file mode 100644 (file)
index 0000000..97b7519
--- /dev/null
@@ -0,0 +1,236 @@
+
+/*
+ * Asterisk -- An open source telephony toolkit.
+ *
+ * Copyright (C) 2019 Sangoma, Inc.
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+/*!
+ * \file
+ * \brief Prometheus Channel Metrics
+ *
+ * \author Matt Jordan <mjordan@digium.com>
+ *
+ */
+
+#include "asterisk.h"
+#include "asterisk/res_prometheus.h"
+#include "asterisk/stasis_channels.h"
+#include "asterisk/pbx.h"
+#include "prometheus_internal.h"
+
+#define CHANNELS_STATE_HELP "Individual channel states. 0=down; 1=reserved; 2=offhook; 3=dialing; 4=ring; 5=ringing; 6=up; 7=busy; 8=dialing_offhook; 9=prering."
+
+#define CHANNELS_DURATION_HELP "Individual channel durations (in seconds)."
+
+/*!
+ * \internal
+ * \brief Callback function to get a channel's current state
+ *
+ * \param metric The metric to populate
+ * \snapshot Channel snapshot
+ */
+static void get_channel_state(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot)
+{
+       snprintf(metric->value, sizeof(metric->value), "%d", snapshot->state);
+}
+
+/*!
+ * \internal
+ * \brief Callback function to get a channel's current duration
+ *
+ * \param metric The metric to populate
+ * \param snapshot Channel snapshot
+ */
+static void get_channel_duration(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot)
+{
+       struct timeval now = ast_tvnow();
+       int64_t duration = ast_tvdiff_sec(now, snapshot->base->creationtime);
+
+       snprintf(metric->value, sizeof(metric->value), "%" PRIu64, duration);
+}
+
+/*!
+ * \internal
+ * \brief Helper struct for generating individual channel stats
+ */
+struct channel_metric_defs {
+       /*!
+        * \brief Help text to display
+        */
+       const char *help;
+       /*!
+        * \brief Name of the metric
+        */
+       const char *name;
+       /*!
+        * \brief Callback function to generate a metric value for a given channel
+        */
+       void (* const get_value)(struct prometheus_metric *metric, struct ast_channel_snapshot *snapshot);
+} channel_metric_defs[] = {
+       {
+               .help = CHANNELS_STATE_HELP,
+               .name = "asterisk_channels_state",
+               .get_value = get_channel_state,
+       },
+       {
+               .help = CHANNELS_DURATION_HELP,
+               .name = "asterisk_channels_duration_seconds",
+               .get_value = get_channel_duration,
+       },
+};
+
+static void get_total_call_count(struct prometheus_metric *metric)
+{
+       snprintf(metric->value, sizeof(metric->value), "%d", ast_processed_calls());
+}
+
+static void get_current_call_count(struct prometheus_metric *metric)
+{
+       snprintf(metric->value, sizeof(metric->value), "%d", ast_active_calls());
+}
+
+/*!
+ * \internal
+ * \brief Channel based metrics that are always available
+ */
+static struct prometheus_metric global_channel_metrics[] = {
+       PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+               PROMETHEUS_METRIC_COUNTER,
+               "asterisk_calls_sum",
+               "Total call count.",
+               &get_total_call_count
+       ),
+       PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+               PROMETHEUS_METRIC_GAUGE,
+               "asterisk_calls_count",
+               "Current call count.",
+               &get_current_call_count
+       ),
+};
+
+/*!
+ * \internal
+ * \brief Callback invoked when Prometheus scrapes the server
+ *
+ * \param response The response to populate with formatted metrics
+ */
+static void channels_scrape_cb(struct ast_str **response)
+{
+       struct ao2_container *channels;
+       struct ao2_iterator it_chans;
+       struct ast_channel_snapshot *snapshot;
+       struct prometheus_metric *channel_metrics;
+       char eid_str[32];
+       int num_channels;
+       int i, j;
+       struct prometheus_metric channel_count = PROMETHEUS_METRIC_STATIC_INITIALIZATION(
+               PROMETHEUS_METRIC_GAUGE,
+               "asterisk_channels_count",
+               "Current channel count.",
+               NULL
+       );
+
+       ast_eid_to_str(eid_str, sizeof(eid_str), &ast_eid_default);
+
+       channels = ast_channel_cache_all();
+       num_channels = ao2_container_count(channels);
+
+       /* Channel count */
+       PROMETHEUS_METRIC_SET_LABEL(&channel_count, 0, "eid", eid_str);
+       snprintf(channel_count.value, sizeof(channel_count.value), "%d", num_channels);
+       prometheus_metric_to_string(&channel_count, response);
+
+       /* Global call values */
+       for (i = 0; i < ARRAY_LEN(global_channel_metrics); i++) {
+               PROMETHEUS_METRIC_SET_LABEL(&global_channel_metrics[i], 0, "eid", eid_str);
+               global_channel_metrics[i].get_metric_value(&global_channel_metrics[i]);
+               prometheus_metric_to_string(&global_channel_metrics[i], response);
+       }
+
+       if (num_channels == 0) {
+               ao2_ref(channels, -1);
+               return;
+       }
+
+       /* Channel dependent values */
+       channel_metrics = ast_calloc(ARRAY_LEN(channel_metric_defs) * num_channels, sizeof(*channel_metrics));
+       if (!channel_metrics) {
+               ao2_ref(channels, -1);
+               return;
+       }
+
+       it_chans = ao2_iterator_init(channels, 0);
+       for (i = 0; (snapshot = ao2_iterator_next(&it_chans)); ao2_ref(snapshot, -1), i++) {
+               for (j = 0; j < ARRAY_LEN(channel_metric_defs); j++) {
+                       int index = i * ARRAY_LEN(channel_metric_defs) + j;
+
+                       channel_metrics[index].type = PROMETHEUS_METRIC_GAUGE;
+                       ast_copy_string(channel_metrics[index].name, channel_metric_defs[j].name, sizeof(channel_metrics[index].name));
+                       channel_metrics[index].help = channel_metric_defs[j].help;
+                       PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 0, "eid", eid_str);
+                       PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 1, "name", (snapshot->base->name));
+                       PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 2, "id", (snapshot->base->uniqueid));
+                       PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 3, "type", (snapshot->base->type));
+                       if (snapshot->peer) {
+                               PROMETHEUS_METRIC_SET_LABEL(&channel_metrics[index], 4, "linkedid", (snapshot->peer->linkedid));
+                       }
+                       channel_metric_defs[j].get_value(&channel_metrics[index], snapshot);
+
+                       if (i > 0) {
+                               AST_LIST_INSERT_TAIL(&channel_metrics[j].children, &channel_metrics[index], entry);
+                       }
+               }
+       }
+       ao2_iterator_destroy(&it_chans);
+
+       for (j = 0; j < ARRAY_LEN(channel_metric_defs); j++) {
+               prometheus_metric_to_string(&channel_metrics[j], response);
+       }
+
+       ast_free(channel_metrics);
+       ao2_ref(channels, -1);
+}
+
+struct prometheus_callback channels_callback = {
+       .name = "Channels callback",
+       .callback_fn = channels_scrape_cb,
+};
+
+/*!
+ * \internal
+ * \brief Callback invoked when the core module is unloaded
+ */
+static void channel_metrics_unload_cb(void)
+{
+       prometheus_callback_unregister(&channels_callback);
+}
+
+/*!
+ * \internal
+ * \brief Metrics provider definition
+ */
+static struct prometheus_metrics_provider provider = {
+       .name = "channels",
+       .unload_cb = channel_metrics_unload_cb,
+};
+
+int channel_metrics_init(void)
+{
+       prometheus_metrics_provider_register(&provider);
+       prometheus_callback_register(&channels_callback);
+
+       return 0;
+}
\ No newline at end of file
diff --git a/res/prometheus/prometheus_internal.h b/res/prometheus/prometheus_internal.h
new file mode 100644 (file)
index 0000000..06cff98
--- /dev/null
@@ -0,0 +1,41 @@
+/*
+ * Prometheus Internal API
+ *
+ * Copyright (C) 2019 Sangoma, Inc.
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+#ifndef PROMETHEUS_INTERNAL_H__
+#define PROMETHEUS_INTERNAL_H__
+
+/*!
+ * \file prometheus_internal
+ *
+ * \brief Prometheus Metric Internal API
+ *
+ * This module provides internal APIs for \file res_prometheus.
+ * It should not be used outsize of that module, and should
+ * typically only provide intialization functions for units that
+ * want to register metrics / handlers with the core API.
+ */
+
+/*!
+ * \brief Initialize channel metrics
+ *
+ * \retval 0 success
+ * \retval -1 error
+ */
+int channel_metrics_init(void);
+
+#endif /* #define PROMETHEUS_INTERNAL_H__ */
index 1f4e635..4533e1b 100644 (file)
 #include "asterisk/buildinfo.h"
 #include "asterisk/res_prometheus.h"
 
+#include "prometheus/prometheus_internal.h"
+
 /*! \brief Lock that protects data structures during an HTTP scrape */
 AST_MUTEX_DEFINE_STATIC(scrape_lock);
 
@@ -135,6 +137,8 @@ AST_VECTOR(, struct prometheus_metric *) metrics;
 
 AST_VECTOR(, struct prometheus_callback *) callbacks;
 
+AST_VECTOR(, const struct prometheus_metrics_provider *) providers;
+
 /*! \brief The actual module config */
 struct module_config {
        /*! \brief General settings */
@@ -812,6 +816,11 @@ static void prometheus_config_post_apply(void)
        }
 }
 
+void prometheus_metrics_provider_register(const struct prometheus_metrics_provider *provider)
+{
+       AST_VECTOR_APPEND(&providers, provider);
+}
+
 static int unload_module(void)
 {
        SCOPED_MUTEX(lock, &scrape_lock);
@@ -819,6 +828,16 @@ static int unload_module(void)
 
        ast_http_uri_unlink(&prometheus_uri);
 
+       for (i = 0; i < AST_VECTOR_SIZE(&providers); i++) {
+               const struct prometheus_metrics_provider *provider = AST_VECTOR_GET(&providers, i);
+
+               if (!provider->unload_cb) {
+                       continue;
+               }
+
+               provider->unload_cb();
+       }
+
        for (i = 0; i < AST_VECTOR_SIZE(&metrics); i++) {
                struct prometheus_metric *metric = AST_VECTOR_GET(&metrics, i);
 
@@ -827,7 +846,7 @@ static int unload_module(void)
        AST_VECTOR_FREE(&metrics);
 
        AST_VECTOR_FREE(&callbacks);
-
+       AST_VECTOR_FREE(&providers);
        aco_info_destroy(&cfg_info);
        ao2_global_obj_release(global_config);
 
@@ -836,11 +855,31 @@ static int unload_module(void)
 
 static int reload_module(void) {
        SCOPED_MUTEX(lock, &scrape_lock);
+       int i;
+       struct prometheus_general_config *general_config;
 
        ast_http_uri_unlink(&prometheus_uri);
        if (aco_process_config(&cfg_info, 1) == ACO_PROCESS_ERROR) {
                return -1;
        }
+
+       /* Our config should be all reloaded now */
+       general_config = prometheus_general_config_get();
+       for (i = 0; i < AST_VECTOR_SIZE(&providers); i++) {
+               const struct prometheus_metrics_provider *provider = AST_VECTOR_GET(&providers, i);
+
+               if (!provider->reload_cb) {
+                       continue;
+               }
+
+               if (provider->reload_cb(general_config)) {
+                       ast_log(AST_LOG_WARNING, "Failed to reload metrics provider %s\n", provider->name);
+                       ao2_ref(general_config, -1);
+                       return -1;
+               }
+       }
+       ao2_ref(general_config, -1);
+
        if (ast_http_uri_link(&prometheus_uri)) {
                ast_log(AST_LOG_WARNING, "Failed to re-register Prometheus Metrics URI during reload\n");
                return -1;
@@ -861,6 +900,10 @@ static int load_module(void)
                goto cleanup;
        }
 
+       if (AST_VECTOR_INIT(&providers, 8)) {
+               goto cleanup;
+       }
+
        if (aco_info_init(&cfg_info)) {
                goto cleanup;
        }
@@ -874,6 +917,10 @@ static int load_module(void)
                goto cleanup;
        }
 
+       if (channel_metrics_init()) {
+               goto cleanup;
+       }
+
        if (ast_http_uri_link(&prometheus_uri)) {
                goto cleanup;
        }
@@ -885,6 +932,7 @@ cleanup:
        aco_info_destroy(&cfg_info);
        AST_VECTOR_FREE(&metrics);
        AST_VECTOR_FREE(&callbacks);
+       AST_VECTOR_FREE(&providers);
 
        return AST_MODULE_LOAD_DECLINE;
 }
index 01279be..2719267 100644 (file)
@@ -173,13 +173,13 @@ AST_TEST_DEFINE(metric_values)
        }
 
        ast_test_status_update(test, " -> Retrieved: %s\n", ast_str_buffer(buffer));
-       ast_test_validate_cleanup(test, strcmp(ast_str_buffer(buffer),
+       ast_test_validate_cleanup(test, strstr(ast_str_buffer(buffer),
                "# HELP test_counter_one A test counter\n"
                "# TYPE test_counter_one counter\n"
                "test_counter_one 1\n"
                "# HELP test_counter_two A test counter\n"
                "# TYPE test_counter_two counter\n"
-               "test_counter_two 2\n") == 0, result, metric_values_cleanup);
+               "test_counter_two 2\n") != NULL, result, metric_values_cleanup);
 
 metric_values_cleanup:
        prometheus_metric_unregister(&test_counter_one);
@@ -247,10 +247,10 @@ AST_TEST_DEFINE(metric_callback_register)
        }
 
        ast_test_status_update(test, " -> Retrieved: %s\n", ast_str_buffer(buffer));
-       ast_test_validate(test, strcmp(ast_str_buffer(buffer),
+       ast_test_validate(test, strstr(ast_str_buffer(buffer),
                "# HELP test_counter A test counter\n"
                "# TYPE test_counter counter\n"
-               "test_counter 0\n") == 0);
+               "test_counter 0\n") != NULL);
 
        prometheus_callback_unregister(&callback);