HTTP: Stop accepting requests on final system shutdown.
authorRichard Mudgett <rmudgett@digium.com>
Wed, 11 Feb 2015 17:39:13 +0000 (17:39 +0000)
committerRichard Mudgett <rmudgett@digium.com>
Wed, 11 Feb 2015 17:39:13 +0000 (17:39 +0000)
There are three CLI commands to stop and restart Asterisk each.

1) core stop/restart now - Hangup all calls and stop or restart Asterisk.
New channels are prevented while the shutdown request is pending.

2) core stop/restart gracefully - Stop or restart Asterisk when there are
no calls remaining in the system.  New channels are prevented while the
shutdown request is pending.

3) core stop/restart when convenient - Stop or restart Asterisk when there
are no calls in the system.  New calls are not prevented while the
shutdown request is pending.

ARI has made stopping/restarting Asterisk more problematic.  While a
shutdown request is pending it is desirable to continue to process ARI
HTTP requests for current calls.  To handle the current calls while a
shutdown request is pending, a new committed to shutdown phase is needed
so ARI applications can deal with the calls until the system is fully
committed to shutdown.

* Added a new shutdown committed phase so ARI applications can deal with
calls until the final committed to shutdown phase is reached.

* Made refuse new HTTP requests when the system has reached the final
system shutdown phase.  Starting anything while the system is actively
releasing resources and unloading modules is not a good thing.

* Split the bridging framework shutdown to not cleanup the global bridging
containers when shutting down in a hurry.  This is similar to how other
modules prevent crashes on rapid system shutdown.

* Moved ast_begin_shutdown(), ast_cancel_shutdown(), and
ast_shutting_down().  You should not have to include channel.h just to
access these system functions.

ASTERISK-24752 #close
Reported by: Matthew Jordan

Review: https://reviewboard.asterisk.org/r/4399/
........

Merged revisions 431692 from http://svn.asterisk.org/svn/asterisk/branches/13

git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@431694 65c4cc65-6c06-0410-ace0-fbb531ad65f3

apps/app_confbridge.c
channels/chan_sip.c
include/asterisk.h
include/asterisk/channel.h
main/asterisk.c
main/bridge.c
main/channel.c
main/http.c
res/res_pjsip/pjsip_options.c
res/res_pjsip_pubsub.c

index ed4f8d5..0f1b977 100644 (file)
@@ -1757,6 +1757,12 @@ static int confbridge_exec(struct ast_channel *chan, const char *data)
 
        /* if we're shutting down, don't attempt to do further processing */
        if (ast_shutting_down()) {
+               /*
+                * Not taking any new calls at this time.  We cannot create
+                * the announcer channel if this is the first channel into
+                * the conference and we certainly cannot create any
+                * recording channel.
+                */
                leave_conference(&user);
                conference = NULL;
                goto confbridge_cleanup;
index a41a8ae..43dc6af 100644 (file)
@@ -24854,6 +24854,10 @@ static int handle_request_options(struct sip_pvt *p, struct sip_request *req, st
                ast_string_field_set(p, context, sip_cfg.default_context);
 
        if (ast_shutting_down()) {
+               /*
+                * Not taking any new calls at this time.
+                * Likely a server availability OPTIONS poll.
+                */
                msg = "503 Unavailable";
        } else {
                msg = "404 Not Found";
index ee1a9c3..edb100b 100644 (file)
@@ -111,6 +111,42 @@ int ast_register_cleanup(void (*func)(void));
  */
 void ast_unregister_atexit(void (*func)(void));
 
+/*!
+ * \brief Cancel an existing shutdown and return to normal operation.
+ *
+ * \note Shutdown can be cancelled while the server is waiting for
+ * any existing channels to be destroyed before shutdown becomes
+ * irreversible.
+ *
+ * \return non-zero if shutdown cancelled.
+ */
+int ast_cancel_shutdown(void);
+
+/*!
+ * \details
+ * The server is preventing new channel creation in preparation for
+ * shutdown and may actively be releasing resources.  The shutdown
+ * process may be canceled by ast_cancel_shutdown() if it is not too
+ * late.
+ *
+ * \note The preparation to shutdown phase can be quite lengthy
+ * if we are gracefully shutting down.  How long existing calls will
+ * last is not up to us.
+ *
+ * \return non-zero if the server is preparing to or actively shutting down.
+ */
+int ast_shutting_down(void);
+
+/*!
+ * \return non-zero if the server is actively shutting down.
+ * \since 13.3.0
+ *
+ * \details
+ * The server is releasing resources and unloading modules.
+ * It won't be long now.
+ */
+int ast_shutdown_final(void);
+
 #if !defined(LOW_MEMORY)
 /*!
  * \brief Register the version of a source code file with the core.
index 595c91d..b6755e2 100644 (file)
@@ -1511,6 +1511,14 @@ const struct ast_channel_tech *ast_get_channel_tech(const char *name);
 void ast_hangup(struct ast_channel *chan);
 
 /*!
+ * \brief Soft hangup all active channels.
+ * \since 13.3.0
+ *
+ * \return Nothing
+ */
+void ast_softhangup_all(void);
+
+/*!
  * \brief Softly hangup up a channel
  *
  * \param chan channel to be soft-hung-up
@@ -2203,23 +2211,12 @@ int ast_channel_defer_dtmf(struct ast_channel *chan);
 /*! Undo defer.  ast_read will return any DTMF characters that were queued */
 void ast_channel_undefer_dtmf(struct ast_channel *chan);
 
-/*! Initiate system shutdown -- prevents new channels from being allocated.
- * \param hangup  If "hangup" is non-zero, all existing channels will receive soft
- *  hangups */
-void ast_begin_shutdown(int hangup);
-
-/*! Cancels an existing shutdown and returns to normal operation */
-void ast_cancel_shutdown(void);
-
 /*! \return number of channels available for lookup */
 int ast_active_channels(void);
 
 /*! \return the number of channels not yet destroyed */
 int ast_undestroyed_channels(void);
 
-/*! \return non-zero if Asterisk is being shut down */
-int ast_shutting_down(void);
-
 /*! Activate a given generator */
 int ast_activate_generator(struct ast_channel *chan, struct ast_generator *gen, void *params);
 
index 1e58296..1817eef 100644 (file)
@@ -432,16 +432,34 @@ static char ast_config_AST_CTL[PATH_MAX] = "asterisk.ctl";
 extern unsigned int ast_FD_SETSIZE;
 
 static char *_argv[256];
+
 typedef enum {
-       NOT_SHUTTING_DOWN = -2,
-       SHUTTING_DOWN = -1,
-       /* Valid values for quit_handler niceness below: */
+       /*! Normal operation */
+       NOT_SHUTTING_DOWN,
+       /*! Committed to shutting down.  Final phase */
+       SHUTTING_DOWN_FINAL,
+       /*! Committed to shutting down.  Initial phase */
+       SHUTTING_DOWN,
+       /*!
+        * Valid values for quit_handler() niceness below.
+        * These shutdown/restart levels can be cancelled.
+        *
+        * Remote console exit right now
+        */
        SHUTDOWN_FAST,
+       /*! core stop/restart now */
        SHUTDOWN_NORMAL,
+       /*! core stop/restart gracefully */
        SHUTDOWN_NICE,
+       /*! core stop/restart when convenient */
        SHUTDOWN_REALLY_NICE
 } shutdown_nice_t;
+
 static shutdown_nice_t shuttingdown = NOT_SHUTTING_DOWN;
+
+/*! Prevent new channel allocation for shutdown. */
+static int shutdown_pending;
+
 static int restartnow;
 static pthread_t consolethread = AST_PTHREADT_NULL;
 static pthread_t mon_sig_flags;
@@ -1867,6 +1885,43 @@ int ast_set_priority(int pri)
        return 0;
 }
 
+int ast_shutdown_final(void)
+{
+       return shuttingdown == SHUTTING_DOWN_FINAL;
+}
+
+int ast_shutting_down(void)
+{
+       return shutdown_pending;
+}
+
+int ast_cancel_shutdown(void)
+{
+       int shutdown_aborted = 0;
+
+       ast_mutex_lock(&safe_system_lock);
+       if (shuttingdown >= SHUTDOWN_FAST) {
+               shuttingdown = NOT_SHUTTING_DOWN;
+               shutdown_pending = 0;
+               shutdown_aborted = 1;
+       }
+       ast_mutex_unlock(&safe_system_lock);
+       return shutdown_aborted;
+}
+
+/*!
+ * \internal
+ * \brief Initiate system shutdown -- prevents new channels from being allocated.
+ */
+static void ast_begin_shutdown(void)
+{
+       ast_mutex_lock(&safe_system_lock);
+       if (shuttingdown != NOT_SHUTTING_DOWN) {
+               shutdown_pending = 1;
+       }
+       ast_mutex_unlock(&safe_system_lock);
+}
+
 static int can_safely_quit(shutdown_nice_t niceness, int restart);
 static void really_quit(int num, shutdown_nice_t niceness, int restart);
 
@@ -1879,8 +1934,53 @@ static void quit_handler(int num, shutdown_nice_t niceness, int restart)
        /* It wasn't our time. */
 }
 
+#define SHUTDOWN_TIMEOUT       15      /* Seconds */
+
+/*!
+ * \internal
+ * \brief Wait for all channels to die, a timeout, or shutdown cancelled.
+ * \since 13.3.0
+ *
+ * \param niceness Shutdown niceness in effect
+ * \param seconds Number of seconds to wait or less than zero if indefinitely.
+ *
+ * \retval zero if waiting wasn't necessary.  We were idle.
+ * \retval non-zero if we had to wait.
+ */
+static int wait_for_channels_to_die(shutdown_nice_t niceness, int seconds)
+{
+       time_t start;
+       time_t now;
+       int waited = 0;
+
+       time(&start);
+       for (;;) {
+               if (!ast_undestroyed_channels() || shuttingdown != niceness) {
+                       break;
+               }
+               if (seconds < 0) {
+                       /* No timeout so just poll every second */
+                       sleep(1);
+               } else {
+                       time(&now);
+
+                       /* Wait up to the given seconds for all channels to go away */
+                       if (seconds < (now - start)) {
+                               break;
+                       }
+
+                       /* Sleep 1/10 of a second */
+                       usleep(100000);
+               }
+               waited = 1;
+       }
+       return waited;
+}
+
 static int can_safely_quit(shutdown_nice_t niceness, int restart)
 {
+       int waited = 0;
+
        /* Check if someone else isn't already doing this. */
        ast_mutex_lock(&safe_system_lock);
        if (shuttingdown != NOT_SHUTTING_DOWN && niceness >= shuttingdown) {
@@ -1897,40 +1997,30 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart)
         * the atexit handlers, otherwise this would be a bit early. */
        ast_cdr_engine_term();
 
-       /* Shutdown the message queue for the technology agnostic message channel.
-        * This has to occur before we pause shutdown pending ast_undestroyed_channels. */
+       /*
+        * Shutdown the message queue for the technology agnostic message channel.
+        * This has to occur before we pause shutdown pending ast_undestroyed_channels.
+        *
+        * XXX This is not reversed on shutdown cancel.
+        */
        ast_msg_shutdown();
 
        if (niceness == SHUTDOWN_NORMAL) {
-               time_t s, e;
                /* Begin shutdown routine, hanging up active channels */
-               ast_begin_shutdown(1);
+               ast_begin_shutdown();
                if (ast_opt_console) {
                        ast_verb(0, "Beginning asterisk %s....\n", restart ? "restart" : "shutdown");
                }
-               time(&s);
-               for (;;) {
-                       time(&e);
-                       /* Wait up to 15 seconds for all channels to go away */
-                       if ((e - s) > 15 || !ast_undestroyed_channels() || shuttingdown != niceness) {
-                               break;
-                       }
-                       /* Sleep 1/10 of a second */
-                       usleep(100000);
-               }
+               ast_softhangup_all();
+               waited |= wait_for_channels_to_die(niceness, SHUTDOWN_TIMEOUT);
        } else if (niceness >= SHUTDOWN_NICE) {
                if (niceness != SHUTDOWN_REALLY_NICE) {
-                       ast_begin_shutdown(0);
+                       ast_begin_shutdown();
                }
                if (ast_opt_console) {
                        ast_verb(0, "Waiting for inactivity to perform %s...\n", restart ? "restart" : "halt");
                }
-               for (;;) {
-                       if (!ast_undestroyed_channels() || shuttingdown != niceness) {
-                               break;
-                       }
-                       sleep(1);
-               }
+               waited |= wait_for_channels_to_die(niceness, -1);
        }
 
        /* Re-acquire lock and check if someone changed the niceness, in which
@@ -1944,9 +2034,28 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart)
                ast_mutex_unlock(&safe_system_lock);
                return 0;
        }
-       shuttingdown = SHUTTING_DOWN;
+
+       if (niceness >= SHUTDOWN_REALLY_NICE) {
+               shuttingdown = SHUTTING_DOWN;
+               ast_mutex_unlock(&safe_system_lock);
+
+               /* No more Mr. Nice guy.  We are committed to shutting down now. */
+               ast_begin_shutdown();
+               ast_softhangup_all();
+               waited |= wait_for_channels_to_die(SHUTTING_DOWN, SHUTDOWN_TIMEOUT);
+
+               ast_mutex_lock(&safe_system_lock);
+       }
+       shuttingdown = SHUTTING_DOWN_FINAL;
        ast_mutex_unlock(&safe_system_lock);
 
+       if (niceness >= SHUTDOWN_NORMAL && waited) {
+               /*
+                * We were not idle.  Give things in progress a chance to
+                * recognize the final shutdown phase.
+                */
+               sleep(1);
+       }
        return 1;
 }
 
@@ -2454,8 +2563,6 @@ static char *handle_restart_when_convenient(struct ast_cli_entry *e, int cmd, st
 
 static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-       int aborting_shutdown = 0;
-
        switch (cmd) {
        case CLI_INIT:
                e->command = "core abort shutdown";
@@ -2471,16 +2578,8 @@ static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_
        if (a->argc != e->args)
                return CLI_SHOWUSAGE;
 
-       ast_mutex_lock(&safe_system_lock);
-       if (shuttingdown >= SHUTDOWN_FAST) {
-               aborting_shutdown = 1;
-               shuttingdown = NOT_SHUTTING_DOWN;
-       }
-       ast_mutex_unlock(&safe_system_lock);
+       ast_cancel_shutdown();
 
-       if (aborting_shutdown) {
-               ast_cancel_shutdown();
-       }
        return CLI_SUCCESS;
 }
 
index c1fc145..6737fa6 100644 (file)
@@ -5322,7 +5322,7 @@ static void bridge_prnt_obj(void *v_obj, void *where, ao2_prnt_fn *prnt)
 
 /*!
  * \internal
- * \brief Shutdown the bridging system.
+ * \brief Shutdown the bridging system.  Stuff to always do.
  * \since 12.0.0
  *
  * \return Nothing
@@ -5334,6 +5334,17 @@ static void bridge_shutdown(void)
        ast_manager_unregister("BridgeTechnologyUnsuspend");
        ast_cli_unregister_multiple(bridge_cli, ARRAY_LEN(bridge_cli));
        ao2_container_unregister("bridges");
+}
+
+/*!
+ * \internal
+ * \brief Shutdown the bridging system.  More stuff to do on graceful shutdown.
+ * \since 13.3.0
+ *
+ * \return Nothing
+ */
+static void bridge_cleanup(void)
+{
        ao2_cleanup(bridges);
        bridges = NULL;
        ao2_cleanup(bridge_manager);
@@ -5342,6 +5353,7 @@ static void bridge_shutdown(void)
 
 int ast_bridging_init(void)
 {
+       ast_register_cleanup(bridge_cleanup);
        ast_register_atexit(bridge_shutdown);
 
        if (ast_stasis_bridging_init()) {
index aab8e22..f8ae442 100644 (file)
@@ -100,9 +100,6 @@ struct ast_epoll_data {
 #define MONITOR_DELAY  150 * 8         /*!< 150 ms of MONITORING DELAY */
 #endif
 
-/*! \brief Prevent new channel allocation if shutting down. */
-static int shutting_down;
-
 static int chancount;
 
 unsigned long global_fin, global_fout;
@@ -507,13 +504,9 @@ static int ast_channel_softhangup_cb(void *obj, void *arg, int flags)
        return 0;
 }
 
-void ast_begin_shutdown(int hangup)
+void ast_softhangup_all(void)
 {
-       shutting_down = 1;
-
-       if (hangup) {
-               ao2_callback(channels, OBJ_NODATA | OBJ_MULTIPLE, ast_channel_softhangup_cb, NULL);
-       }
+       ao2_callback(channels, OBJ_NODATA | OBJ_MULTIPLE, ast_channel_softhangup_cb, NULL);
 }
 
 /*! \brief returns number of active/allocated channels */
@@ -527,18 +520,6 @@ int ast_undestroyed_channels(void)
        return ast_atomic_fetchadd_int(&chancount, 0);
 }
 
-/*! \brief Cancel a shutdown in progress */
-void ast_cancel_shutdown(void)
-{
-       shutting_down = 0;
-}
-
-/*! \brief Returns non-zero if Asterisk is being shut down */
-int ast_shutting_down(void)
-{
-       return shutting_down;
-}
-
 /*! \brief Set when to hangup channel */
 void ast_channel_setwhentohangup_tv(struct ast_channel *chan, struct timeval offset)
 {
index ef3b4b2..3346cad 100644 (file)
@@ -1873,6 +1873,11 @@ static int httpd_process_request(struct ast_tcptls_session_instance *ser)
                return -1;
        }
 
+       if (ast_shutdown_final()) {
+               ast_http_error(ser, 503, "Service Unavailable", "Shutdown in progress");
+               return -1;
+       }
+
        /* process "Request Headers" lines */
        if (http_request_headers_get(ser, &headers)) {
                return -1;
index cf55f4d..44f33f3 100644 (file)
@@ -655,6 +655,10 @@ static pj_bool_t options_on_rx_request(pjsip_rx_data *rdata)
        ast_copy_pj_str(exten, &sip_ruri->user, sizeof(exten));
 
        if (ast_shutting_down()) {
+               /*
+                * Not taking any new calls at this time.
+                * Likely a server availability OPTIONS poll.
+                */
                send_options_response(rdata, 503);
        } else if (!ast_strlen_zero(exten) && !ast_exists_extension(NULL, endpoint->context, exten, 1, NULL)) {
                send_options_response(rdata, 404);
index 5047184..4d53091 100644 (file)
@@ -2059,7 +2059,7 @@ static int send_notify(struct sip_subscription_tree *sub_tree, unsigned int forc
        pjsip_evsub *evsub = sub_tree->evsub;
        pjsip_tx_data *tdata;
 
-       if (ast_shutting_down()
+       if (ast_shutdown_final()
                && sub_tree->root->subscription_state == PJSIP_EVSUB_STATE_TERMINATED
                && sub_tree->persistence) {
                return 0;