diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c index e33d525fd41..36ba1c03e29 100644 --- a/daemons/attrd/attrd_cib.c +++ b/daemons/attrd/attrd_cib.c @@ -33,15 +33,14 @@ attrd_cib_destroy_cb(void *user_data) cib->cmds->signoff(cib); - if (attrd_shutting_down()) { + if (attrd.shutting_down) { pcmk__info("Disconnected from the CIB manager"); - - } else { - // @TODO This should trigger a reconnect, not a shutdown - pcmk__crit("Lost connection to the CIB manager, shutting down"); - attrd_exit_status = CRM_EX_DISCONNECT; - attrd_shutdown(0); + return; } + + // @TODO This should trigger a reconnect, not a shutdown + pcmk__crit("Lost connection to the CIB manager, shutting down"); + pcmk__daemon_quit(&attrd, CRM_EX_DISCONNECT); } static void @@ -56,7 +55,7 @@ attrd_cib_updated_cb(const char *event, xmlNode *msg) } if (pcmk__cib_element_in_patchset(patchset, PCMK_XE_ALERTS)) { - if (attrd_shutting_down()) { + if (attrd.shutting_down) { pcmk__debug("Ignoring alerts change in CIB during shutdown"); } else { mainloop_set_trigger(attrd_config_read); @@ -81,7 +80,7 @@ attrd_cib_updated_cb(const char *event, xmlNode *msg) if (status_changed || pcmk__cib_element_in_patchset(patchset, PCMK_XE_NODES)) { - if (attrd_shutting_down()) { + if (attrd.shutting_down) { pcmk__debug("Ignoring node change in CIB during shutdown"); return; } @@ -154,7 +153,10 @@ attrd_cib_connect(int max_retry) void attrd_cib_disconnect(void) { - CRM_CHECK(the_cib != NULL, return); + if (the_cib == NULL) { + return; + } + the_cib->cmds->del_notify_callback(the_cib, PCMK__VALUE_CIB_DIFF_NOTIFY, attrd_cib_updated_cb); cib__clean_up_connection(&the_cib); @@ -518,7 +520,7 @@ write_attribute(attribute_t *a, bool ignore_delay) } // Private attributes (or any in standalone mode) are not written to the CIB - if (attrd_stand_alone() || pcmk__is_set(a->flags, attrd_attr_is_private)) { + if (attrd.stand_alone || pcmk__is_set(a->flags, attrd_attr_is_private)) { should_write = false; } diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c index 20519e0911d..2cb2fb849e3 100644 --- a/daemons/attrd/attrd_corosync.c +++ b/daemons/attrd/attrd_corosync.c @@ -82,7 +82,7 @@ attrd_peer_message(pcmk__node_status_t *peer, xmlNode *xml) return; } - if (attrd_shutting_down()) { + if (attrd.shutting_down) { /* If we're shutting down, we want to continue responding to election * ops as long as we're a cluster member (because our vote may be * needed). Ignore all other messages. @@ -179,14 +179,13 @@ attrd_cpg_dispatch(cpg_handle_t handle, const struct cpg_name *group_name, static void attrd_cpg_destroy(void *unused) { - if (attrd_shutting_down()) { + if (attrd.shutting_down) { pcmk__info("Disconnected from Corosync process group"); - - } else { - pcmk__crit("Lost connection to Corosync process group, shutting down"); - attrd_exit_status = CRM_EX_DISCONNECT; - attrd_shutdown(0); + return; } + + pcmk__crit("Lost connection to Corosync process group, shutting down"); + pcmk__daemon_quit(&attrd, CRM_EX_DISCONNECT); } #endif // SUPPORT_COROSYNC @@ -511,7 +510,10 @@ attrd_cluster_connect(void) pcmk__cluster_set_status_callback(&attrd_peer_change_cb); rc = pcmk_cluster_connect(attrd_cluster); - if (rc != pcmk_rc_ok) { + + if (rc == pcmk_rc_ok) { + pcmk__info("Cluster connection active"); + } else { pcmk__err("Cluster connection failed"); } diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c index c48c808fda7..8a0c7d4915b 100644 --- a/daemons/attrd/attrd_elections.c +++ b/daemons/attrd/attrd_elections.c @@ -46,7 +46,7 @@ attrd_start_election_if_needed(void) { if ((peer_writer == NULL) && (election_state(attrd_cluster) != election_in_progress) - && !attrd_shutting_down()) { + && !attrd.shutting_down) { pcmk__info("Starting an election to determine the writer"); election_vote(attrd_cluster); @@ -68,7 +68,7 @@ attrd_handle_election_op(const pcmk__node_status_t *peer, xmlNode *xml) pcmk__xe_set(xml, PCMK__XA_SRC, peer->name); // Don't become writer if we're shutting down - rc = election_count_vote(attrd_cluster, xml, !attrd_shutting_down()); + rc = election_count_vote(attrd_cluster, xml, !attrd.shutting_down); switch(rc) { case election_start: diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c index 3b0113ccac1..1544f157099 100644 --- a/daemons/attrd/attrd_ipc.c +++ b/daemons/attrd/attrd_ipc.c @@ -492,7 +492,7 @@ static int32_t attrd_ipc_accept(qb_ipcs_connection_t *c, uid_t uid, gid_t gid) { pcmk__trace("New client connection %p", c); - if (attrd_shutting_down()) { + if (attrd.shutting_down) { pcmk__info("Ignoring new connection from pid %d during shutdown", pcmk__client_pid(c)); return -ECONNREFUSED; @@ -649,8 +649,9 @@ attrd_ipc_cleanup(void) * \internal * \brief Set up attrd IPC communication */ -void +bool attrd_ipc_init(void) { pcmk__serve_attrd_ipc(&ipcs, &ipc_callbacks); + return ipcs != NULL; } diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c index 3025f197e76..de214d62db2 100644 --- a/daemons/attrd/attrd_utils.c +++ b/daemons/attrd/attrd_utils.c @@ -24,83 +24,11 @@ cib_t *the_cib = NULL; -static bool shutting_down = false; -static GMainLoop *mloop = NULL; - /* A hash table storing information on the protocol version of each peer attrd. * The key is the peer's uname, and the value is the protocol version number. */ GHashTable *peer_protocol_vers = NULL; -/*! - * \internal - * \brief Check whether local attribute manager is shutting down - * - * \return \c true if local attribute manager has begun shutdown sequence, - * otherwise \c false - */ -bool -attrd_shutting_down(void) -{ - return shutting_down; -} - -/*! - * \internal - * \brief Exit (using mainloop or not, as appropriate) - * - * \param[in] nsig Ignored - */ -void -attrd_shutdown(int nsig) -{ - // Tell various functions not to do anthing - shutting_down = true; - - // Don't respond to signals while shutting down - mainloop_destroy_signal(SIGTERM); - mainloop_destroy_signal(SIGCHLD); - mainloop_destroy_signal(SIGPIPE); - mainloop_destroy_signal(SIGUSR1); - mainloop_destroy_signal(SIGUSR2); - mainloop_destroy_signal(SIGTRAP); - - attrd_free_waitlist(); - attrd_free_confirmations(); - - g_clear_pointer(&peer_protocol_vers, g_hash_table_destroy); - - if ((mloop == NULL) || !g_main_loop_is_running(mloop)) { - /* If there's no main loop active, just exit. This should be possible - * only if we get SIGTERM in brief windows at start-up and shutdown. - */ - crm_exit(CRM_EX_OK); - } else { - g_main_loop_quit(mloop); - g_main_loop_unref(mloop); - } -} - -/*! - * \internal - * \brief Create a main loop for attrd - */ -void -attrd_init_mainloop(void) -{ - mloop = g_main_loop_new(NULL, FALSE); -} - -/*! - * \internal - * \brief Run attrd main loop - */ -void -attrd_run_mainloop(void) -{ - g_main_loop_run(mloop); -} - /* strlen("value") */ #define plus_plus_len (5) diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c index 1682ba8ba56..f63f704b4ce 100644 --- a/daemons/attrd/pacemaker-attrd.c +++ b/daemons/attrd/pacemaker-attrd.c @@ -30,11 +30,23 @@ #define SUMMARY "daemon for managing Pacemaker node attributes" -static gboolean stand_alone = false; -gchar **log_files = NULL; +static pcmk__daemon_ipc_fns_t ipc_fns = { + .already_running = pcmk__daemon_ipc_running, +}; + +pcmk__daemon_t attrd = { + .type = pcmk_ipc_attrd, + .ec = CRM_EX_OK, + .ipc_fns = &ipc_fns, +}; + +static gchar **log_files = NULL; +static gchar **processed_args = NULL; +static GOptionContext *context = NULL; static GOptionEntry entries[] = { - { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone, + { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, + &attrd.stand_alone, "(Advanced use only) Run in stand-alone mode", NULL }, { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY, @@ -54,44 +66,6 @@ static pcmk__supported_format_t formats[] = { lrmd_t *the_lrmd = NULL; crm_trigger_t *attrd_config_read = NULL; -crm_exit_t attrd_exit_status = CRM_EX_OK; - -/*! - * \internal - * \brief Check whether local attribute manager is running in stand-alone mode - * - * \return \c true if local attribute manager is in stand-alone mode, or - * \c false otherwise - */ -bool -attrd_stand_alone(void) -{ - return stand_alone; -} - -static bool -ipc_already_running(void) -{ - pcmk_ipc_api_t *old_instance = NULL; - int rc = pcmk_rc_ok; - - rc = pcmk_new_ipc_api(&old_instance, pcmk_ipc_attrd); - if (rc != pcmk_rc_ok) { - return false; - } - - rc = pcmk__connect_ipc(old_instance, pcmk_ipc_dispatch_sync, 2); - if (rc != pcmk_rc_ok) { - pcmk__debug("No existing %s instance found: %s", - pcmk_ipc_name(old_instance, true), pcmk_rc_str(rc)); - pcmk_free_ipc_api(old_instance); - return false; - } - - pcmk_disconnect_ipc(old_instance); - pcmk_free_ipc_api(old_instance); - return true; -} static GOptionContext * build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { @@ -102,33 +76,73 @@ build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { return context; } +static void +attrd_cleanup_cmdline(void) +{ + g_clear_pointer(&processed_args, g_strfreev); + g_clear_pointer(&context, g_option_context_free); + g_clear_pointer(&log_files, g_strfreev); +} + +static void +attrd_cleanup(void) +{ + attrd_ipc_cleanup(); + attrd_lrmd_disconnect(); + attrd_unregister_handlers(); + attrd_cib_disconnect(); + attrd_cluster_disconnect(); + + attrd_free_removed_peers(); + attrd_free_waitlist(); + attrd_free_confirmations(); + attrd_cleanup_xml_ids(); + + g_clear_pointer(&attributes, g_hash_table_destroy); + g_clear_pointer(&peer_protocol_vers, g_hash_table_destroy); +} + +/*! + * \internal + * \brief Quit the main loop and set the exit code to \c CRM_EX_OK + * + * \param[in] nsig Ignored + * + * \note This is a main loop signal handler function. + */ +static void +attrd_shutdown(int nsig) +{ + pcmk__daemon_quit(&attrd, CRM_EX_OK); +} + int main(int argc, char **argv) { int rc = pcmk_rc_ok; GError *error = NULL; - bool initialized = false; - GOptionGroup *output_group = NULL; - pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); - gchar **processed_args = pcmk__cmdline_preproc(argv, NULL); - GOptionContext *context = build_arg_context(args, &output_group); + pcmk__common_args_t *args = NULL; + + atexit(attrd_cleanup_cmdline); + + args = pcmk__new_common_args(SUMMARY); + processed_args = pcmk__cmdline_preproc(argv, NULL); + context = build_arg_context(args, &output_group); - attrd_init_mainloop(); crm_log_preinit(NULL, argc, argv); - mainloop_add_signal(SIGTERM, attrd_shutdown); pcmk__register_formats(output_group, formats); if (!g_option_context_parse_strv(context, &processed_args, &error)) { - attrd_exit_status = CRM_EX_USAGE; + attrd.ec = CRM_EX_USAGE; goto done; } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if ((rc != pcmk_rc_ok) || (out == NULL)) { - attrd_exit_status = CRM_EX_ERROR; - g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, + attrd.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, attrd.ec, "Error creating output format %s: %s", args->output_ty, pcmk_rc_str(rc)); goto done; @@ -143,19 +157,18 @@ main(int argc, char **argv) pcmk__add_logfiles(log_files, out); crm_log_init(PCMK__VALUE_ATTRD, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); - pcmk__notice("Starting Pacemaker node attribute manager%s", - (attrd_stand_alone() ? " in standalone mode" : "")); - if (ipc_already_running()) { - attrd_exit_status = CRM_EX_OK; - g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, + if (attrd.ipc_fns->already_running(&attrd)) { + attrd.ec = CRM_EX_OK; + g_set_error(&error, PCMK__EXITC_ERROR, attrd.ec, "Aborting start-up because an attribute manager " "instance is already active"); pcmk__crit("%s", error->message); goto done; } - initialized = true; + pcmk__notice("Starting Pacemaker node attribute manager%s", + (attrd.stand_alone ? " in standalone mode" : "")); attributes = pcmk__strkey_table(NULL, attrd_free_attribute); @@ -163,10 +176,10 @@ main(int argc, char **argv) * This allows us to assume the CIB is connected whenever we process a * cluster or IPC message (which also avoids start-up race conditions). */ - if (!attrd_stand_alone()) { + if (!attrd.stand_alone) { if (attrd_cib_connect(30) != pcmk_ok) { - attrd_exit_status = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, + attrd.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, attrd.ec, "Could not connect to the CIB"); goto done; } @@ -174,18 +187,16 @@ main(int argc, char **argv) } if (attrd_cluster_connect() != pcmk_rc_ok) { - attrd_exit_status = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, + attrd.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, attrd.ec, "Could not connect to the cluster"); goto done; } - pcmk__info("Cluster connection active"); - // Initialization that requires the cluster to be connected attrd_election_init(); - if (!attrd_stand_alone()) { + if (!attrd.stand_alone) { attrd_cib_init(); } @@ -196,42 +207,40 @@ main(int argc, char **argv) */ attrd_send_protocol(NULL); - attrd_ipc_init(); - pcmk__notice("Pacemaker node attribute manager successfully started and " - "accepting connections"); - attrd_run_mainloop(); - - done: - if (initialized) { - pcmk__info("Shutting down attribute manager"); - - attrd_ipc_cleanup(); - attrd_lrmd_disconnect(); - - if (!attrd_stand_alone()) { - attrd_cib_disconnect(); - } + if (!attrd_ipc_init()) { + attrd.ec = CRM_EX_FATAL; + goto done; + } - attrd_free_removed_peers(); - attrd_free_waitlist(); - attrd_cluster_disconnect(); - attrd_unregister_handlers(); - g_hash_table_destroy(attributes); + rc = pcmk__daemon_init(&attrd); + if (rc != pcmk_rc_ok) { + attrd.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, attrd.ec, + "Error initializing daemon object: %s", + pcmk_rc_str(rc)); + goto done; } - attrd_cleanup_xml_ids(); + mainloop_add_signal(SIGTERM, attrd_shutdown); - g_strfreev(processed_args); - pcmk__free_arg_context(context); + pcmk__daemon_run(&attrd); + + done: + /* If we got here through any of the "goto done" calls instead of by the + * main loop quitting on SIGTERM, shutting_down will still be false. Set + * it here so attrd_cleanup -> attrd_cib_disconnect -> attrd_cib_destroy_cb + * doesn't call pcmk__daemon_quit with no main loop. + */ + attrd.shutting_down = true; - g_strfreev(log_files); + attrd_cleanup(); pcmk__output_and_clear_error(&error, out); if (out != NULL) { - out->finish(out, attrd_exit_status, true, NULL); + out->finish(out, attrd.ec, true, NULL); pcmk__output_free(out); } pcmk__unregister_formats(); - crm_exit(attrd_exit_status); + crm_exit(attrd.ec); } diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h index 8d9b50ad408..8f111312028 100644 --- a/daemons/attrd/pacemaker-attrd.h +++ b/daemons/attrd/pacemaker-attrd.h @@ -57,14 +57,8 @@ pcmk__ipc_send_ack((client), (id), (flags), ATTRD_PROTOCOL_VERSION, \ CRM_EX_INDETERMINATE) -void attrd_init_mainloop(void); -void attrd_run_mainloop(void); - void attrd_free_waitlist(void); -void attrd_shutdown(int nsig); -bool attrd_shutting_down(void); -bool attrd_stand_alone(void); -void attrd_ipc_init(void); +bool attrd_ipc_init(void); void attrd_ipc_cleanup(void); int attrd_cib_connect(int max_retry); @@ -93,7 +87,6 @@ int attrd_failure_regex(regex_t *regex, const char *rsc, const char *op, unsigned int interval_ms); extern cib_t *the_cib; -extern crm_exit_t attrd_exit_status; /* Alerts */ @@ -188,6 +181,7 @@ typedef struct { extern pcmk_cluster_t *attrd_cluster; extern GHashTable *attributes; extern GHashTable *peer_protocol_vers; +extern pcmk__daemon_t attrd; #define CIB_OP_TIMEOUT_S 120 diff --git a/daemons/based/based_corosync.c b/daemons/based/based_corosync.c index cf8fc547d6c..b4fe16eafd2 100644 --- a/daemons/based/based_corosync.c +++ b/daemons/based/based_corosync.c @@ -115,7 +115,10 @@ based_cluster_connect(void) #endif // SUPPORT_COROSYNC rc = pcmk_cluster_connect(cluster); - if (rc != pcmk_rc_ok) { + + if (rc == pcmk_rc_ok) { + pcmk__info("Cluster connection active"); + } else { pcmk__err("Cluster connection failed"); } diff --git a/daemons/based/based_ipc.c b/daemons/based/based_ipc.c index 9e32dfdbd5f..56629b50d3b 100644 --- a/daemons/based/based_ipc.c +++ b/daemons/based/based_ipc.c @@ -306,6 +306,10 @@ based_ipc_init(void) { pcmk__serve_based_ipc(&ipcs_ro, &ipcs_rw, &ipc_ro_callbacks, &ipc_rw_callbacks); + + if ((ipcs_ro == NULL) || (ipcs_rw == NULL)) { + crm_exit(CRM_EX_FATAL); + } } /*! diff --git a/daemons/based/pacemaker-based.c b/daemons/based/pacemaker-based.c index 46206af7ffc..d8d9a145b64 100644 --- a/daemons/based/pacemaker-based.c +++ b/daemons/based/pacemaker-based.c @@ -405,8 +405,6 @@ main(int argc, char **argv) "Could not connect to the cluster"); goto done; } - - pcmk__info("Cluster connection active"); } // Run the main loop diff --git a/daemons/execd/execd_alerts.c b/daemons/execd/execd_alerts.c index 5d1f9e3e7fa..e835e0c0621 100644 --- a/daemons/execd/execd_alerts.c +++ b/daemons/execd/execd_alerts.c @@ -168,28 +168,40 @@ execd_process_alert_exec(pcmk__client_t *client, xmlNode *request) static bool drain_check(unsigned int remaining_timeout_ms) { - if (inflight_alerts != NULL) { - unsigned int count = g_hash_table_size(inflight_alerts); + unsigned int count = 0; - if (count > 0) { - pcmk__trace("%d alerts pending (%.3fs timeout remaining)", - count, (remaining_timeout_ms / 1000.0)); - return TRUE; - } + if (inflight_alerts == NULL) { + return false; } - return FALSE; + + count = g_hash_table_size(inflight_alerts); + if (count > 0) { + pcmk__trace("%d alerts pending (%.3fs timeout remaining)", + count, (remaining_timeout_ms / 1000.0)); + return true; + } + + return false; } void lrmd_drain_alerts(GMainLoop *mloop) { - if (inflight_alerts != NULL) { - unsigned int timer_ms = max_inflight_timeout() + 5000; + unsigned int timer_ms = 0; + + if (mloop == NULL) { + return; + } - pcmk__trace("Draining in-flight alerts (timeout %.3fs)", - (timer_ms / 1000.0)); - draining_alerts = TRUE; - pcmk_drain_main_loop(mloop, timer_ms, drain_check); - g_clear_pointer(&inflight_alerts, g_hash_table_destroy); + if (inflight_alerts == NULL) { + return; } + + timer_ms = max_inflight_timeout() + 5000; + + pcmk__trace("Draining in-flight alerts (timeout %.3fs)", + (timer_ms / 1000.0)); + draining_alerts = TRUE; + pcmk_drain_main_loop(mloop, timer_ms, drain_check); + g_clear_pointer(&inflight_alerts, g_hash_table_destroy); } diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c index 89d77d6ec08..67a98a172e5 100644 --- a/daemons/execd/execd_commands.c +++ b/daemons/execd/execd_commands.c @@ -101,7 +101,6 @@ typedef struct { GHashTable *params; } lrmd_cmd_t; -static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc); static gboolean execute_resource_action(void *user_data); static void cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id); @@ -1580,7 +1579,7 @@ execd_process_signon(pcmk__client_t *client, xmlNode *request, int call_id, pcmk__xe_set(*reply, PCMK__XA_LRMD_OP, CRM_OP_REGISTER); pcmk__xe_set(*reply, PCMK__XA_LRMD_CLIENTID, client->id); pcmk__xe_set(*reply, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION); - pcmk__xe_set_time(*reply, PCMK__XA_UPTIME, now - start_time); + pcmk__xe_set_time(*reply, PCMK__XA_UPTIME, now - execd.start_time); if (start_state) { pcmk__xe_set(*reply, PCMK__XA_NODE_START_STATE, start_state); diff --git a/daemons/execd/execd_ipc.c b/daemons/execd/execd_ipc.c index c8821f51c9e..47f87a99ec4 100644 --- a/daemons/execd/execd_ipc.c +++ b/daemons/execd/execd_ipc.c @@ -218,8 +218,9 @@ execd_ipc_cleanup(void) * \internal * \brief Set up executor IPC communication */ -void +bool execd_ipc_init(void) { pcmk__serve_execd_ipc(&ipcs, &ipc_callbacks); + return ipcs != NULL; } diff --git a/daemons/execd/execd_messages.c b/daemons/execd/execd_messages.c index 95335f366a4..a88daa0a11f 100644 --- a/daemons/execd/execd_messages.c +++ b/daemons/execd/execd_messages.c @@ -50,9 +50,17 @@ handle_ipc_fwd_request(pcmk__request_t *request) rc = ipc_proxy_forward_client(request->ipc_client, request->xml); - if (rc == pcmk_rc_ok) { + if ((rc == pcmk_rc_ok) || (rc == ESHUTDOWN)) { pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL); + if (rc == ESHUTDOWN) { + /* We're shutting down so return NULL for the reply, but + * execd_handle_request will still want to process a result which + * is why we set one above. + */ + return NULL; + } + } else { pcmk__set_result(&request->result, pcmk_rc2exitc(rc), PCMK_EXEC_ERROR, pcmk_rc_str(rc)); diff --git a/daemons/execd/pacemaker-execd.c b/daemons/execd/pacemaker-execd.c index 26b780e8e9a..4320e56dc36 100644 --- a/daemons/execd/pacemaker-execd.c +++ b/daemons/execd/pacemaker-execd.c @@ -40,9 +40,22 @@ # define SUMMARY "resource agent executor daemon for Pacemaker cluster nodes" #endif -static GMainLoop *mainloop = NULL; +static bool execd_quit(pcmk__daemon_t *d); + +static pcmk__daemon_fns_t fns = { + .quit = execd_quit, +}; + +pcmk__daemon_t execd = { + .type = pcmk_ipc_execd, + .ec = CRM_EX_OK, + .fns = &fns, +}; + static stonith_t *fencer_api = NULL; -time_t start_time; + +static gchar **processed_args = NULL; +static GOptionContext *context = NULL; static struct { gchar **log_files; @@ -51,13 +64,6 @@ static struct { #endif // PCMK__COMPILE_REMOTE } options; -#ifdef PCMK__COMPILE_REMOTE -/* whether shutdown request has been sent */ -static gboolean shutting_down = FALSE; -#endif - -static void exit_executor(void); - static void fencer_connection_destroy_cb(stonith_t *st, stonith_event_t *e) { @@ -112,11 +118,19 @@ lrmd_client_destroy(pcmk__client_t *client) pcmk__free_client(client); #ifdef PCMK__COMPILE_REMOTE - /* If we were waiting to shut down, we can now safely do so - * if there are no more proxied IPC providers + /* If we were waiting to shut down, we can now safely do so if there are + * no more proxied IPC providers + * + * This kills the main loop and cleans everything up. Control flow will + * eventually make it back up to lrmd_remote_client_destroy or + * execd_ipc_closed, which were both called directly from the main loop. + * After that, we'll return to the done label in main() and finish + * shutting down. */ - if (shutting_down && (ipc_proxy_get_provider() == NULL)) { - exit_executor(); + if (execd.shutting_down && (ipc_proxy_get_provider() == NULL)) { + // Unset shutting_down so pcmk__daemon_quit does something + execd.shutting_down = false; + pcmk__daemon_quit(&execd, CRM_EX_OK); } #endif } @@ -170,12 +184,8 @@ lrmd_server_send_notify(pcmk__client_t *client, xmlNode *msg) return ENOTCONN; } -/*! - * \internal - * \brief Clean up and exit immediately - */ static void -exit_executor(void) +execd_cleanup(void) { const unsigned int nclients = pcmk__ipc_client_count(); @@ -189,64 +199,8 @@ exit_executor(void) ipc_proxy_cleanup(); #endif - if (mainloop) { - lrmd_drain_alerts(mainloop); - } - execd_unregister_handlers(); - g_hash_table_destroy(rsc_list); - - // @TODO End mainloop instead so all cleanup is done - crm_exit(CRM_EX_OK); -} - -/*! - * \internal - * \brief Request cluster shutdown if appropriate, otherwise exit immediately - * - * \param[in] nsig Signal that caused invocation (ignored) - */ -static void -lrmd_shutdown(int nsig) -{ -#ifdef PCMK__COMPILE_REMOTE - pcmk__client_t *ipc_proxy = ipc_proxy_get_provider(); - - /* If there are active proxied IPC providers, then we may be running - * resources, so notify the cluster that we wish to shut down. - */ - if (ipc_proxy) { - if (shutting_down) { - pcmk__notice("Waiting for cluster to stop resources before " - "exiting"); - return; - } - - pcmk__info("Sending shutdown request to cluster"); - if (ipc_proxy_shutdown_req(ipc_proxy) < 0) { - pcmk__crit("Shutdown request failed, exiting immediately"); - - } else { - /* We requested a shutdown. Now, we need to wait for an - * acknowledgement from the proxy host, then wait for all proxy - * hosts to disconnect (which ensures that all resources have been - * stopped). - */ - shutting_down = TRUE; - - /* Stop accepting new proxy connections */ - execd_stop_tls_server(); - - /* Currently, we let the OS kill us if the clients don't disconnect - * in a reasonable time. We could instead set a long timer here - * (shorter than what the OS is likely to use) and exit immediately - * if it pops. - */ - return; - } - } -#endif - exit_executor(); + g_clear_pointer(&rsc_list, g_hash_table_destroy); } /*! @@ -257,7 +211,7 @@ void handle_shutdown_ack(void) { #ifdef PCMK__COMPILE_REMOTE - if (shutting_down) { + if (execd.shutting_down) { pcmk__info("IPC proxy provider acknowledged shutdown request"); return; } @@ -266,23 +220,45 @@ handle_shutdown_ack(void) "provider"); } +#ifdef PCMK__COMPILE_REMOTE +static bool +execd_quit_on_nack(pcmk__daemon_t *srv) +{ + lrmd_drain_alerts(execd.mainloop); + return true; +} +#endif + /*! * \internal * \brief Handle rejection of shutdown request + * + * \return Standard Pacemaker return code */ -void +int handle_shutdown_nack(void) { #ifdef PCMK__COMPILE_REMOTE - if (shutting_down) { + if (execd.shutting_down) { pcmk__info("Exiting immediately after IPC proxy provider indicated no " "resources will be stopped"); - exit_executor(); - return; + + /* Avoid calling the original quit function because that can potentially + * just lead us right back to this point. However, we still want to do + * everything in pcmk__daemon_quit (most importantly, kill the main loop) + * as well as drain alerts. + */ + execd.shutting_down = false; + execd.fns->quit = execd_quit_on_nack; + pcmk__daemon_quit(&execd, CRM_EX_OK); + + return ESHUTDOWN; } #endif + pcmk__debug("Ignoring unexpected shutdown rejection from IPC proxy " "provider"); + return pcmk_rc_ok; } static GOptionEntry entries[] = { @@ -315,11 +291,81 @@ build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) return context; } +static void +execd_cleanup_cmdline(void) +{ + g_clear_pointer(&processed_args, g_strfreev); + g_clear_pointer(&context, g_option_context_free); + g_clear_pointer(&options.log_files, g_strfreev); +#ifdef PCMK__COMPILE_REMOTE + g_clear_pointer(&options.port, g_free); +#endif +} + +static bool +execd_quit(pcmk__daemon_t *d) +{ +#ifdef PCMK__COMPILE_REMOTE + pcmk__client_t *ipc_proxy = ipc_proxy_get_provider(); + + if (ipc_proxy == NULL) { + goto done; + } + + /* If there are active proxied IPC providers, then we may be running + * resources, so notify the cluster that we wish to shut down. + */ + if (execd.shutting_down) { + pcmk__notice("Waiting for cluster to stop resources before exiting"); + return false; + } + + pcmk__info("Sending shutdown request to cluster"); + if (ipc_proxy_shutdown_req(ipc_proxy) < 0) { + pcmk__crit("Shutdown request failed, exiting immediately"); + goto done; + } + + /* We requested a shutdown. Now, we need to wait for an acknowledgement + * from the proxy host, then wait for all proxy hosts to disconnect (which + * ensures that all resources have been stopped). + */ + execd.shutting_down = true; + + /* Stop accepting new proxy connections */ + execd_stop_tls_server(); + + /* Currently, we let the OS kill us if the clients don't disconnect in a + * reasonable time. We could instead set a long timer here (shorter than + * what the OS is likely to use) and exit immediately if it pops. + */ + return false; + +done: +#endif + + lrmd_drain_alerts(execd.mainloop); + return true; +} + +/*! + * \internal + * \brief Quit the main loop and set the exit code to \c CRM_EX_OK + * + * \param[in] nsig Ignored + * + * \note This is a main loop signal handler function. + */ +static void +execd_shutdown(int nsig) +{ + pcmk__daemon_quit(&execd, CRM_EX_OK); +} + int main(int argc, char **argv) { int rc = pcmk_rc_ok; - crm_exit_t exit_code = CRM_EX_OK; const char *option = NULL; @@ -329,8 +375,8 @@ main(int argc, char **argv) GOptionGroup *output_group = NULL; pcmk__common_args_t *args = NULL; - gchar **processed_args = NULL; - GOptionContext *context = NULL; + + atexit(execd_cleanup_cmdline); #ifdef PCMK__COMPILE_REMOTE // If necessary, create PID 1 now before any file descriptors are opened @@ -349,14 +395,14 @@ main(int argc, char **argv) pcmk__register_formats(output_group, formats); if (!g_option_context_parse_strv(context, &processed_args, &error)) { - exit_code = CRM_EX_USAGE; + execd.ec = CRM_EX_USAGE; goto done; } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if (rc != pcmk_rc_ok) { - exit_code = CRM_EX_ERROR; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + execd.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, execd.ec, "Error creating output format %s: %s", args->output_ty, pcmk_rc_str(rc)); goto done; @@ -398,8 +444,6 @@ main(int argc, char **argv) } #endif // PCMK__COMPILE_REMOTE - start_time = time(NULL); - pcmk__notice("Starting Pacemaker " EXECD_TYPE " executor"); /* The presence of this variable allegedly controls whether child @@ -421,43 +465,46 @@ main(int argc, char **argv) rsc_list = pcmk__strkey_table(NULL, execd_free_rsc); - execd_ipc_init(); + if (!execd_ipc_init()) { + execd.ec = CRM_EX_FATAL; + goto done; + } #ifdef PCMK__COMPILE_REMOTE if (lrmd_init_remote_tls_server() < 0) { pcmk__err("Failed to create TLS listener: shutting down and staying " "down"); - exit_code = CRM_EX_FATAL; + execd.ec = CRM_EX_FATAL; + goto done; + } + + if (!ipc_proxy_init()) { goto done; } - ipc_proxy_init(); #endif - mainloop_add_signal(SIGTERM, lrmd_shutdown); - mainloop = g_main_loop_new(NULL, FALSE); - pcmk__notice("Pacemaker " EXECD_TYPE " executor successfully started and " - "accepting connections"); - pcmk__notice("OCF resource agent search path is %s", PCMK__OCF_RA_PATH); - g_main_loop_run(mainloop); + rc = pcmk__daemon_init(&execd); + if (rc != pcmk_rc_ok) { + execd.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, execd.ec, + "Error initializing daemon object: %s", + pcmk_rc_str(rc)); + goto done; + } - /* should never get here */ - exit_executor(); + mainloop_add_signal(SIGTERM, execd_shutdown); -done: - g_strfreev(options.log_files); -#ifdef PCMK__COMPILE_REMOTE - g_free(options.port); -#endif // PCMK__COMPILE_REMOTE + pcmk__daemon_run(&execd); - g_strfreev(processed_args); - pcmk__free_arg_context(context); +done: + execd_cleanup(); pcmk__output_and_clear_error(&error, out); if (out != NULL) { - out->finish(out, exit_code, true, NULL); + out->finish(out, execd.ec, true, NULL); pcmk__output_free(out); } pcmk__unregister_formats(); - crm_exit(exit_code); + crm_exit(execd.ec); } diff --git a/daemons/execd/pacemaker-execd.h b/daemons/execd/pacemaker-execd.h index ffc98c3b7e5..2403ad1e8dd 100644 --- a/daemons/execd/pacemaker-execd.h +++ b/daemons/execd/pacemaker-execd.h @@ -12,7 +12,6 @@ #include // bool #include // uint32_t -#include // time_t #include // GList, GHashTable, GMainLoop #include // xmlNode @@ -22,7 +21,7 @@ #include // stonith_t extern GHashTable *rsc_list; -extern time_t start_time; +extern pcmk__daemon_t execd; typedef struct { char *rsc_id; @@ -70,7 +69,7 @@ void execd_free_rsc(void *data); void handle_shutdown_ack(void); -void handle_shutdown_nack(void); +int handle_shutdown_nack(void); void lrmd_client_destroy(pcmk__client_t *client); @@ -86,7 +85,7 @@ stonith_t *execd_get_fencer_connection(void); void execd_fencer_connection_failed(void); #ifdef PCMK__COMPILE_REMOTE -void ipc_proxy_init(void); +bool ipc_proxy_init(void); void ipc_proxy_cleanup(void); void ipc_proxy_add_provider(pcmk__client_t *client); void ipc_proxy_remove_provider(pcmk__client_t *client); @@ -104,7 +103,7 @@ void lrmd_drain_alerts(GMainLoop *mloop); bool execd_invalid_msg(xmlNode *msg); void execd_handle_request(pcmk__request_t *request); -void execd_ipc_init(void); +bool execd_ipc_init(void); void execd_ipc_cleanup(void); xmlNode *execd_create_reply_as(const char *origin, int rc, int call_id); diff --git a/daemons/execd/remoted_proxy.c b/daemons/execd/remoted_proxy.c index bc3c9a98838..8be6223baac 100644 --- a/daemons/execd/remoted_proxy.c +++ b/daemons/execd/remoted_proxy.c @@ -176,8 +176,7 @@ ipc_proxy_forward_client(pcmk__client_t *ipc_proxy, xmlNode *xml) } if (pcmk__str_eq(msg_type, LRMD_IPC_OP_SHUTDOWN_NACK, pcmk__str_casei)) { - handle_shutdown_nack(); - return rc; + return handle_shutdown_nack(); } ipc_client = pcmk__find_client_by_id(session); @@ -514,23 +513,43 @@ ipc_proxy_remove_provider(pcmk__client_t *ipc_proxy) g_list_free(remove_these); } -void +bool ipc_proxy_init(void) { ipc_clients = pcmk__strkey_table(NULL, NULL); pcmk__serve_based_ipc(&cib_ro, &cib_rw, &cib_proxy_callbacks_ro, &cib_proxy_callbacks_rw); + if ((cib_ro == NULL) || (cib_rw == NULL)) { + execd.ec = CRM_EX_FATAL; + return false; + } + pcmk__serve_attrd_ipc(&attrd_ipcs, &attrd_proxy_callbacks); + if (attrd_ipcs == NULL) { + execd.ec = CRM_EX_FATAL; + return false; + } pcmk__serve_controld_ipc(&controld_ipcs, &crmd_proxy_callbacks); if (controld_ipcs == NULL) { - // Error already logged - crm_exit(CRM_EX_FATAL); + execd.ec = CRM_EX_FATAL; + return false; } pcmk__serve_fenced_ipc(&fencer_ipcs, &fencer_proxy_callbacks); + if (fencer_ipcs == NULL) { + execd.ec = CRM_EX_FATAL; + return false; + } + pcmk__serve_pacemakerd_ipc(&pacemakerd_ipcs, &pacemakerd_proxy_callbacks); + if (pacemakerd_ipcs == NULL) { + execd.ec = CRM_EX_OSERR; + return false; + } + + return true; } void diff --git a/daemons/fenced/fenced_cib.c b/daemons/fenced/fenced_cib.c index 803cad11e33..b4683027187 100644 --- a/daemons/fenced/fenced_cib.c +++ b/daemons/fenced/fenced_cib.c @@ -367,10 +367,9 @@ watchdog_device_update(void) rc = fenced_device_register(xml, true); pcmk__xml_free(xml); if (rc != pcmk_rc_ok) { - exit_code = CRM_EX_FATAL; pcmk__crit("Cannot register watchdog pseudo fence agent: %s", pcmk_rc_str(rc)); - stonith_shutdown(0); + pcmk__daemon_quit(&fenced, CRM_EX_FATAL); } } @@ -586,16 +585,18 @@ init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *us static void cib_connection_destroy(void *user_data) { - if (stonith_shutdown_flag) { + if (fenced.shutting_down) { pcmk__info("Connection to the CIB manager closed"); return; - } else { - pcmk__crit("Lost connection to the CIB manager, shutting down"); } + + pcmk__crit("Lost connection to the CIB manager, shutting down"); + if (cib_api) { cib_api->cmds->signoff(cib_api); } - stonith_shutdown(0); + + pcmk__daemon_quit(&fenced, CRM_EX_DISCONNECT); } /*! diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c index 233d8a9acc5..79ac34a4e2a 100644 --- a/daemons/fenced/fenced_commands.c +++ b/daemons/fenced/fenced_commands.c @@ -69,7 +69,6 @@ struct device_search_s { uint32_t support_action_only; }; -static gboolean stonith_device_dispatch(void *user_data); static void st_child_done(int pid, const pcmk__action_result_t *result, void *user_data); diff --git a/daemons/fenced/fenced_corosync.c b/daemons/fenced/fenced_corosync.c index 70fe9cd1fef..a24721e4876 100644 --- a/daemons/fenced/fenced_corosync.c +++ b/daemons/fenced/fenced_corosync.c @@ -170,7 +170,7 @@ static void fenced_cpg_destroy(void *unused) { pcmk__crit("Lost connection to cluster layer, shutting down"); - stonith_shutdown(0); + pcmk__daemon_quit(&fenced, CRM_EX_DISCONNECT); } #endif // SUPPORT_COROSYNC @@ -198,7 +198,10 @@ fenced_cluster_connect(void) pcmk__cluster_set_status_callback(&fenced_peer_change_cb); rc = pcmk_cluster_connect(fenced_cluster); - if (rc != pcmk_rc_ok) { + + if (rc == pcmk_rc_ok) { + pcmk__info("Cluster connection active"); + } else { pcmk__err("Cluster connection failed"); } diff --git a/daemons/fenced/fenced_ipc.c b/daemons/fenced/fenced_ipc.c index 34be2dd2b45..cd346c8c5c1 100644 --- a/daemons/fenced/fenced_ipc.c +++ b/daemons/fenced/fenced_ipc.c @@ -65,7 +65,7 @@ static int32_t fenced_ipc_accept(qb_ipcs_connection_t *c, uid_t uid, gid_t gid) { pcmk__trace("New client connection %p", c); - if (stonith_shutdown_flag) { + if (fenced.shutting_down) { pcmk__info("Ignoring new connection from pid %d during shutdown", pcmk__client_pid(c)); return -ECONNREFUSED; @@ -276,8 +276,9 @@ fenced_ipc_cleanup(void) * \internal * \brief Set up fenced IPC communication */ -void +bool fenced_ipc_init(void) { pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks); + return ipcs != NULL; } diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c index 531d0738c10..8af9bbb4ece 100644 --- a/daemons/fenced/pacemaker-fenced.c +++ b/daemons/fenced/pacemaker-fenced.c @@ -37,16 +37,25 @@ #define SUMMARY "daemon for executing fencing devices in a Pacemaker cluster" +static pcmk__daemon_ipc_fns_t ipc_fns = { + .already_running = pcmk__generic_ipc_running, +}; + +pcmk__daemon_t fenced = { + .type = pcmk_ipc_fenced, + .ec = CRM_EX_OK, + .ipc_fns = &ipc_fns, +}; + // @TODO This should be unsigned int long long fencing_watchdog_timeout_ms = 0; GList *stonith_watchdog_targets = NULL; -static GMainLoop *mainloop = NULL; - -gboolean stonith_shutdown_flag = FALSE; - static pcmk__output_t *out = NULL; +static gchar **processed_args = NULL; +static GOptionContext *context = NULL; +static gchar **log_files = NULL; pcmk__supported_format_t formats[] = { PCMK__SUPPORTED_FORMAT_NONE, @@ -55,15 +64,6 @@ pcmk__supported_format_t formats[] = { { NULL, NULL, NULL } }; -static struct { - gboolean stand_alone; - gchar **log_files; -} options; - -crm_exit_t exit_code = CRM_EX_OK; - -static void stonith_cleanup(void); - void do_local_reply(const xmlNode *notify_src, pcmk__client_t *client, int call_options) @@ -260,27 +260,6 @@ node_does_watchdog_fencing(const char *node) pcmk__str_in_list(node, stonith_watchdog_targets, pcmk__str_casei)); } -void -stonith_shutdown(int nsig) -{ - pcmk__info("Terminating with %d clients", pcmk__ipc_client_count()); - stonith_shutdown_flag = TRUE; - if (mainloop != NULL && g_main_loop_is_running(mainloop)) { - g_main_loop_quit(mainloop); - } -} - -static void -stonith_cleanup(void) -{ - fenced_cib_cleanup(); - fenced_ipc_cleanup(); - free_stonith_remote_op_list(); - free_topology_list(); - fenced_free_device_table(); - free_metadata_cache(); -} - /* @COMPAT Deprecated since 2.1.8. Use pcmk_list_fence_attrs() or * crm_resource --list-options=fencing instead of querying daemon metadata. * @@ -302,11 +281,11 @@ fencer_metadata(void) static GOptionEntry entries[] = { { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, - &options.stand_alone, N_("Intended for use in regression testing only"), + &fenced.stand_alone, N_("Intended for use in regression testing only"), NULL }, { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY, - &options.log_files, N_("Send logs to the additional named logfile"), NULL }, + &log_files, N_("Send logs to the additional named logfile"), NULL }, { NULL } }; @@ -321,31 +300,42 @@ build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) return context; } -static bool -ipc_already_running(void) +static void +fenced_cleanup_cmdline(void) { - crm_ipc_t *old_instance = NULL; - int rc = pcmk_rc_ok; + g_clear_pointer(&processed_args, g_strfreev); + g_clear_pointer(&context, g_option_context_free); + g_clear_pointer(&log_files, g_strfreev); +} - old_instance = crm_ipc_new("stonith-ng", 0); - if (old_instance == NULL) { - /* This is an error - memory allocation failed, etc. - but crm_ipc_new - * will have already logged an error message. - */ - return false; - } +/*! + * \internal + * \brief Quit the main loop and set the exit code to \c CRM_EX_OK + * + * \param[in] nsig Ignored + * + * \note This is a main loop signal handler function. + */ +static void +fenced_shutdown(int nsig) +{ + pcmk__info("Terminating with %d clients", pcmk__ipc_client_count()); + pcmk__daemon_quit(&fenced, CRM_EX_OK); +} - rc = pcmk__connect_generic_ipc(old_instance); - if (rc != pcmk_rc_ok) { - pcmk__debug("No existing stonith-ng instance found: %s", - pcmk_rc_str(rc)); - crm_ipc_destroy(old_instance); - return false; - } +static void +fenced_cleanup(void) +{ + fenced_cib_cleanup(); + fenced_ipc_cleanup(); + fenced_unregister_handlers(); + fenced_cluster_disconnect(); + fenced_scheduler_cleanup(); - crm_ipc_close(old_instance); - crm_ipc_destroy(old_instance); - return true; + free_stonith_remote_op_list(); + free_topology_list(); + fenced_free_device_table(); + free_metadata_cache(); } int @@ -356,22 +346,26 @@ main(int argc, char **argv) GError *error = NULL; GOptionGroup *output_group = NULL; - pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); - gchar **processed_args = pcmk__cmdline_preproc(argv, "l"); - GOptionContext *context = build_arg_context(args, &output_group); + pcmk__common_args_t *args = NULL; + + atexit(fenced_cleanup_cmdline); + + args = pcmk__new_common_args(SUMMARY); + processed_args = pcmk__cmdline_preproc(argv, "l"); + context = build_arg_context(args, &output_group); crm_log_preinit(NULL, argc, argv); pcmk__register_formats(output_group, formats); if (!g_option_context_parse_strv(context, &processed_args, &error)) { - exit_code = CRM_EX_USAGE; + fenced.ec = CRM_EX_USAGE; goto done; } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if ((rc != pcmk_rc_ok) || (out == NULL)) { - exit_code = CRM_EX_ERROR; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + fenced.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, fenced.ec, "Error creating output format %s: %s", args->output_ty, pcmk_rc_str(rc)); goto done; @@ -387,84 +381,89 @@ main(int argc, char **argv) rc = fencer_metadata(); if (rc != pcmk_rc_ok) { - exit_code = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + fenced.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, fenced.ec, "Unable to display metadata: %s", pcmk_rc_str(rc)); } goto done; } // Open additional log files - pcmk__add_logfiles(options.log_files, out); + pcmk__add_logfiles(log_files, out); crm_log_init(NULL, LOG_INFO + args->verbosity, TRUE, (args->verbosity > 0), argc, argv, FALSE); - pcmk__notice("Starting Pacemaker fencer"); - - if (ipc_already_running()) { - exit_code = CRM_EX_OK; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + if (fenced.ipc_fns->already_running(&fenced)) { + g_set_error(&error, PCMK__EXITC_ERROR, fenced.ec, "Aborting start-up because a fencer instance is already active"); pcmk__crit("%s", error->message); goto done; } - mainloop_add_signal(SIGTERM, stonith_shutdown); + pcmk__notice("Starting Pacemaker fencer"); pcmk__cluster_init_node_caches(); rc = fenced_scheduler_init(); if (rc != pcmk_rc_ok) { - exit_code = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + fenced.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, fenced.ec, "Error initializing scheduler data: %s", pcmk_rc_str(rc)); goto done; } if (fenced_cluster_connect() != pcmk_rc_ok) { - exit_code = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + fenced.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, fenced.ec, "Could not connect to the cluster"); goto done; } - pcmk__info("Cluster connection active"); - fenced_set_local_node(fenced_cluster->priv->node_name); - if (!options.stand_alone) { + if (!fenced.stand_alone) { setup_cib(); } fenced_init_device_table(); init_topology_list(); - fenced_ipc_init(); - // Create the mainloop and run it... - mainloop = g_main_loop_new(NULL, FALSE); - pcmk__notice("Pacemaker fencer successfully started and accepting " - "connections"); - g_main_loop_run(mainloop); + if (!fenced_ipc_init()) { + fenced.ec = CRM_EX_FATAL; + goto done; + } -done: - g_strfreev(processed_args); - pcmk__free_arg_context(context); + rc = pcmk__daemon_init(&fenced); + if (rc != pcmk_rc_ok) { + fenced.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, fenced.ec, + "Error initializing daemon object: %s", + pcmk_rc_str(rc)); + goto done; + } - g_strfreev(options.log_files); + mainloop_add_signal(SIGTERM, fenced_shutdown); - stonith_cleanup(); - fenced_cluster_disconnect(); - fenced_unregister_handlers(); - fenced_scheduler_cleanup(); + pcmk__daemon_run(&fenced); + +done: + /* If we got here through any of the "goto done" calls instead of by the + * main loop quitting on SIGTERM, shutting_down will still be false. Set + * it here so fenced_cleanup -> fenced_cib_cleanup -> cib_connection_destroy + * doesn't call pcmk__daemon_quit with no main loop. + */ + fenced.shutting_down = true; + + fenced_cleanup(); pcmk__output_and_clear_error(&error, out); if (out != NULL) { - out->finish(out, exit_code, true, NULL); + out->finish(out, fenced.ec, true, NULL); pcmk__output_free(out); } pcmk__unregister_formats(); - crm_exit(exit_code); + crm_exit(fenced.ec); } diff --git a/daemons/fenced/pacemaker-fenced.h b/daemons/fenced/pacemaker-fenced.h index 6002c2f3601..567cb239b71 100644 --- a/daemons/fenced/pacemaker-fenced.h +++ b/daemons/fenced/pacemaker-fenced.h @@ -295,8 +295,6 @@ typedef struct { } stonith_topology_t; -void stonith_shutdown(int nsig); - void fenced_init_device_table(void); void fenced_free_device_table(void); bool fenced_has_watchdog_device(void); @@ -375,7 +373,7 @@ const char *fenced_get_local_node(void); void fenced_scheduler_cleanup(void); void fenced_scheduler_run(xmlNode *cib); -void fenced_ipc_init(void); +bool fenced_ipc_init(void); void fenced_ipc_cleanup(void); int fenced_cluster_connect(void); @@ -405,6 +403,5 @@ extern GHashTable *topology; extern long long fencing_watchdog_timeout_ms; extern GList *stonith_watchdog_targets; extern GHashTable *stonith_remote_op_list; -extern crm_exit_t exit_code; -extern gboolean stonith_shutdown_flag; extern pcmk_cluster_t *fenced_cluster; +extern pcmk__daemon_t fenced; diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c index 286ff374ae4..b542c34df8f 100644 --- a/daemons/pacemakerd/pacemakerd.c +++ b/daemons/pacemakerd/pacemakerd.c @@ -39,6 +39,11 @@ #define SUMMARY "pacemakerd - primary Pacemaker daemon that launches and monitors all subsidiary Pacemaker daemons" +pcmk__daemon_t pacemakerd = { + .type = pcmk_ipc_pacemakerd, + .ec = CRM_EX_OK, +}; + struct { gboolean features; gboolean foreground; @@ -47,6 +52,8 @@ struct { } options; static pcmk__output_t *out = NULL; +static gchar **processed_args = NULL; +static GOptionContext *context = NULL; static pcmk__supported_format_t formats[] = { PCMK__SUPPORTED_FORMAT_NONE, @@ -153,7 +160,7 @@ pacemakerd_chown(const char *path, uid_t uid, gid_t gid) } } -static void +static int create_pcmk_dirs(void) { uid_t pcmk_uid = 0; @@ -172,7 +179,7 @@ create_pcmk_dirs(void) if (pcmk__daemon_user(&pcmk_uid, &pcmk_gid) != pcmk_rc_ok) { pcmk__err("Cluster user " CRM_DAEMON_USER " does not exist, aborting " "Pacemaker startup"); - crm_exit(CRM_EX_NOUSER); + return EINVAL; } // Used by some resource agents @@ -193,6 +200,8 @@ create_pcmk_dirs(void) pacemakerd_chown(dirs[i], pcmk_uid, pcmk_gid); } } + + return pcmk_rc_ok; } static void @@ -243,12 +252,8 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api, { pcmk_pacemakerd_api_reply_t *reply = event_data; - switch (event_type) { - case pcmk_ipc_event_reply: - break; - - default: - return; + if (event_type != pcmk_ipc_event_reply) { + return; } if (status != CRM_EX_OK) { @@ -339,37 +344,57 @@ handle_old_instance(gboolean shutdown) return rc; } +static void +pacemakerd_cleanup_cmdline(void) +{ + g_clear_pointer(&processed_args, g_strfreev); + g_clear_pointer(&context, g_option_context_free); +} + +static void +pacemakerd_cleanup(void) +{ + pacemakerd_ipc_cleanup(); + pacemakerd_unregister_handlers(); + +#if SUPPORT_COROSYNC + cluster_disconnect_cfg(); +#endif +} + int main(int argc, char **argv) { int rc = pcmk_rc_ok; - crm_exit_t exit_code = CRM_EX_OK; GError *error = NULL; GOptionGroup *output_group = NULL; - pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); - gchar **processed_args = pcmk__cmdline_preproc(argv, "p"); - GOptionContext *context = build_arg_context(args, &output_group); + pcmk__common_args_t * args = NULL; + + atexit(pacemakerd_cleanup_cmdline); + + args = pcmk__new_common_args(SUMMARY); + processed_args = pcmk__cmdline_preproc(argv, "p"); + context = build_arg_context(args, &output_group); subdaemon_check_progress = time(NULL); setenv("LC_ALL", "C", 1); // Ensure logs are in a common language crm_log_preinit(NULL, argc, argv); - mainloop_add_signal(SIGHUP, pcmk_ignore); - mainloop_add_signal(SIGQUIT, pcmk_sigquit); pcmk__register_formats(output_group, formats); if (!g_option_context_parse_strv(context, &processed_args, &error)) { - exit_code = CRM_EX_USAGE; + pacemakerd.ec = CRM_EX_USAGE; goto done; } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if ((rc != pcmk_rc_ok) || (out == NULL)) { - exit_code = CRM_EX_ERROR; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s", + pacemakerd.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, pacemakerd.ec, + "Error creating output format %s: %s", args->output_ty, pcmk_rc_str(rc)); goto done; } @@ -378,7 +403,7 @@ main(int argc, char **argv) if (options.features) { out->message(out, "features"); - exit_code = CRM_EX_OK; + pacemakerd.ec = CRM_EX_OK; goto done; } @@ -397,22 +422,23 @@ main(int argc, char **argv) if ((rc == pcmk_rc_ok) && options.shutdown) { goto done; } else if (rc == pcmk_rc_already) { - exit_code = CRM_EX_FATAL; + pacemakerd.ec = CRM_EX_FATAL; goto done; } else if (rc != pcmk_rc_ok) { - exit_code = pcmk_rc2exitc(rc); + pacemakerd.ec = pcmk_rc2exitc(rc); goto done; } /* Don't allow any accidental output after this point. */ if (out != NULL) { - out->finish(out, exit_code, true, NULL); + out->finish(out, pacemakerd.ec, true, NULL); g_clear_pointer(&out, pcmk__output_free); } #if SUPPORT_COROSYNC - if (pacemakerd_read_config() == FALSE) { - crm_exit(CRM_EX_UNAVAILABLE); + if (!pcmkd_read_config()) { + pacemakerd.ec = CRM_EX_UNAVAILABLE; + goto done; } #endif @@ -428,16 +454,22 @@ main(int argc, char **argv) pcmk__notice("Starting Pacemaker " PACEMAKER_VERSION " " QB_XS " build=" BUILD_VERSION " features:" CRM_FEATURES); - mainloop = g_main_loop_new(NULL, FALSE); remove_core_file_limit(); - create_pcmk_dirs(); - pacemakerd_ipc_init(); + if (create_pcmk_dirs() != pcmk_rc_ok) { + pacemakerd.ec = CRM_EX_NOUSER; + goto done; + } + + if (!pacemakerd_ipc_init()) { + pacemakerd.ec = CRM_EX_OSERR; + goto done; + } #if SUPPORT_COROSYNC /* Allows us to block shutdown */ if (!cluster_connect_cfg()) { - exit_code = CRM_EX_PROTOCOL; + pacemakerd.ec = CRM_EX_PROTOCOL; goto done; } #endif @@ -450,16 +482,13 @@ main(int argc, char **argv) case pcmk_rc_ok: break; case pcmk_rc_ipc_unauthorized: - exit_code = CRM_EX_CANTCREAT; + pacemakerd.ec = CRM_EX_CANTCREAT; goto done; default: - exit_code = CRM_EX_FATAL; + pacemakerd.ec = CRM_EX_FATAL; goto done; }; - mainloop_add_signal(SIGTERM, pcmk_shutdown); - mainloop_add_signal(SIGINT, pcmk_shutdown); - if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) { pcmk__notice("Waiting for startup-trigger from SBD"); pacemakerd_state = PCMK__VALUE_WAIT_FOR_PING; @@ -474,27 +503,31 @@ main(int argc, char **argv) init_children_processes(NULL); } - pcmk__notice("Pacemaker daemon successfully started and accepting " - "connections"); - g_main_loop_run(mainloop); - pacemakerd_ipc_cleanup(); - pacemakerd_unregister_handlers(); + rc = pcmk__daemon_init(&pacemakerd); + if (rc != pcmk_rc_ok) { + pacemakerd.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, pacemakerd.ec, + "Error initializing daemon object: %s", + pcmk_rc_str(rc)); + goto done; + } - g_main_loop_unref(mainloop); -#if SUPPORT_COROSYNC - cluster_disconnect_cfg(); -#endif + mainloop_add_signal(SIGHUP, pcmk_ignore); + mainloop_add_signal(SIGINT, pcmk_shutdown); + mainloop_add_signal(SIGQUIT, pcmk_sigquit); + mainloop_add_signal(SIGTERM, pcmk_shutdown); + + pcmk__daemon_run(&pacemakerd); done: - g_strfreev(processed_args); - pcmk__free_arg_context(context); + pacemakerd_cleanup(); pcmk__output_and_clear_error(&error, out); if (out != NULL) { - out->finish(out, exit_code, true, NULL); + out->finish(out, pacemakerd.ec, true, NULL); pcmk__output_free(out); } pcmk__unregister_formats(); - crm_exit(exit_code); + crm_exit(pacemakerd.ec); } diff --git a/daemons/pacemakerd/pacemakerd.h b/daemons/pacemakerd/pacemakerd.h index a2d91618bee..658a683bf75 100644 --- a/daemons/pacemakerd/pacemakerd.h +++ b/daemons/pacemakerd/pacemakerd.h @@ -1,5 +1,5 @@ /* - * Copyright 2010-2025 the Pacemaker project contributors + * Copyright 2010-2026 the Pacemaker project contributors * * The version control history for this file may have further details. * @@ -16,7 +16,6 @@ #define MAX_RESPAWN 100 -extern GMainLoop *mainloop; extern const char *pacemakerd_state; extern bool running_with_sbd; extern bool shutdown_complete_state_reported_client_closed; @@ -24,13 +23,14 @@ extern unsigned int shutdown_complete_state_reported_to; extern crm_trigger_t *shutdown_trigger; extern crm_trigger_t *startup_trigger; extern time_t subdaemon_check_progress; +extern pcmk__daemon_t pacemakerd; int find_and_track_existing_processes(void); gboolean init_children_processes(void *user_data); void pcmk_shutdown(int nsig); void restart_cluster_subdaemons(void); -void pacemakerd_ipc_init(void); +bool pacemakerd_ipc_init(void); void pacemakerd_ipc_cleanup(void); void pacemakerd_unregister_handlers(void); void pacemakerd_handle_request(pcmk__request_t *request); diff --git a/daemons/pacemakerd/pcmkd_corosync.c b/daemons/pacemakerd/pcmkd_corosync.c index e085789d0cc..79f75866d97 100644 --- a/daemons/pacemakerd/pcmkd_corosync.c +++ b/daemons/pacemakerd/pcmkd_corosync.c @@ -98,13 +98,14 @@ cluster_reconnect_cb(void *data) if (cluster_connect_cfg()) { g_clear_pointer(&reconnect_timer, mainloop_timer_del); pcmk__notice("Cluster reconnect succeeded"); - pacemakerd_read_config(); + pcmkd_read_config(); restart_cluster_subdaemons(); return G_SOURCE_REMOVE; - } else { - pcmk__info("Cluster reconnect failed (connection will be reattempted " - "once per second)"); } + + pcmk__info("Cluster reconnect failed (connection will be reattempted " + "once per second)"); + /* * In theory this will continue forever. In practice the CIB connection from * attrd will timeout and shut down Pacemaker when it gets bored. @@ -275,10 +276,10 @@ get_config_opt(uint64_t unused, cmap_handle_t object_handle, const char *key, ch return rc; } -gboolean -pacemakerd_read_config(void) +bool +pcmkd_read_config(void) { - cs_error_t rc = CS_OK; + cs_error_t cs_rc = CS_OK; int retries = 0; cmap_handle_t local_handle; uint64_t config = 0; @@ -287,53 +288,52 @@ pacemakerd_read_config(void) gid_t found_gid = 0; pid_t found_pid = 0; int rv; + bool success = false; enum pcmk_cluster_layer cluster_layer = pcmk_cluster_layer_unknown; const char *cluster_layer_s = NULL; // There can be only one possibility do { - rc = pcmk__init_cmap(&local_handle); - if (rc != CS_OK) { - retries++; - pcmk__info("Could not connect to Corosync CMAP: %s " - "(retrying in %ds) " QB_XS " rc=%d", - pcmk_rc_str(pcmk__corosync2rc(rc)), retries, rc); - sleep(retries); - - } else { + cs_rc = pcmk__init_cmap(&local_handle); + if (cs_rc == CS_OK) { break; } + retries++; + pcmk__info("Could not connect to Corosync CMAP: %s " + "(retrying in %ds) " QB_XS " rc=%d", + pcmk_rc_str(pcmk__corosync2rc(cs_rc)), retries, cs_rc); + sleep(retries); } while (retries < 5); - if (rc != CS_OK) { + if (cs_rc != CS_OK) { pcmk__crit("Could not connect to Corosync CMAP: %s " - QB_XS " rc=%d", pcmk_rc_str(pcmk__corosync2rc(rc)), rc); - return FALSE; + QB_XS " rc=%d", pcmk_rc_str(pcmk__corosync2rc(cs_rc)), cs_rc); + return success; } - rc = cmap_fd_get(local_handle, &fd); - if (rc != CS_OK) { + cs_rc = cmap_fd_get(local_handle, &fd); + if (cs_rc != CS_OK) { pcmk__crit("Could not get Corosync CMAP descriptor: %s " QB_XS " rc=%d", - pcmk_rc_str(pcmk__corosync2rc(rc)), rc); - cmap_finalize(local_handle); - return FALSE; + pcmk_rc_str(pcmk__corosync2rc(cs_rc)), cs_rc); + goto done; } /* CMAP provider run as root (in given user namespace, anyway)? */ - if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, - &found_uid, &found_gid))) { + rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, + &found_uid, &found_gid); + if (rv == 0) { pcmk__crit("Rejecting Corosync CMAP provider because process %lld " "is running as uid %lld gid %lld, not root", (long long) PCMK__SPECIAL_PID_AS_0(found_pid), (long long) found_uid, (long long) found_gid); - cmap_finalize(local_handle); - return FALSE; - } else if (rv < 0) { + goto done; + } + + if (rv < 0) { pcmk__crit("Could not authenticate Corosync CMAP provider: %s " QB_XS " rc=%d", strerror(-rv), -rv); - cmap_finalize(local_handle); - return FALSE; + goto done; } cluster_layer = pcmk_get_cluster_layer(); @@ -343,7 +343,7 @@ pacemakerd_read_config(void) pcmk__crit("Expected Corosync cluster layer but detected %s " QB_XS " cluster_layer=%d", cluster_layer_s, cluster_layer); - return FALSE; + goto done; } pcmk__info("Reading configuration for %s cluster layer", cluster_layer_s); @@ -369,8 +369,9 @@ pacemakerd_read_config(void) free(debug_enabled); } - if(local_handle){ + if (local_handle) { gid_t gid = 0; + if (pcmk__daemon_user(NULL, &gid) != pcmk_rc_ok) { pcmk__warn("Could not authorize group with Corosync " QB_XS " No group found for user " CRM_DAEMON_USER); @@ -379,17 +380,20 @@ pacemakerd_read_config(void) char *key = pcmk__assert_asprintf("uidgid.gid.%lld", (long long) gid); - rc = cmap_set_uint8(local_handle, key, 1); + cs_rc = cmap_set_uint8(local_handle, key, 1); free(key); - if (rc != CS_OK) { + if (cs_rc != CS_OK) { pcmk__warn("Could not authorize group with Corosync: %s " QB_XS " group=%u rc=%d", - pcmk_rc_str(pcmk__corosync2rc(rc)), gid, rc); + pcmk_rc_str(pcmk__corosync2rc(cs_rc)), gid, cs_rc); } } } - cmap_finalize(local_handle); - return TRUE; + success = true; + +done: + cmap_finalize(local_handle); + return success; } diff --git a/daemons/pacemakerd/pcmkd_corosync.h b/daemons/pacemakerd/pcmkd_corosync.h index bad71022ab6..0fd75b37e43 100644 --- a/daemons/pacemakerd/pcmkd_corosync.h +++ b/daemons/pacemakerd/pcmkd_corosync.h @@ -1,5 +1,5 @@ /* - * Copyright 2010-2025 the Pacemaker project contributors + * Copyright 2010-2026 the Pacemaker project contributors * * The version control history for this file may have further details. * @@ -13,6 +13,6 @@ gboolean cluster_connect_cfg(void); void cluster_disconnect_cfg(void); -gboolean pacemakerd_read_config(void); +bool pcmkd_read_config(void); bool pcmkd_corosync_connected(void); void pcmkd_shutdown_corosync(void); diff --git a/daemons/pacemakerd/pcmkd_ipc.c b/daemons/pacemakerd/pcmkd_ipc.c index 41d823261c9..ab8a0ac24f3 100644 --- a/daemons/pacemakerd/pcmkd_ipc.c +++ b/daemons/pacemakerd/pcmkd_ipc.c @@ -199,8 +199,9 @@ pacemakerd_ipc_cleanup(void) * \internal * \brief Set up pacemakerd IPC communication */ -void +bool pacemakerd_ipc_init(void) { pcmk__serve_pacemakerd_ipc(&ipcs, &ipc_callbacks); + return ipcs != NULL; } diff --git a/daemons/pacemakerd/pcmkd_subdaemons.c b/daemons/pacemakerd/pcmkd_subdaemons.c index 63152d29df0..9bd868d8499 100644 --- a/daemons/pacemakerd/pcmkd_subdaemons.c +++ b/daemons/pacemakerd/pcmkd_subdaemons.c @@ -97,8 +97,6 @@ bool shutdown_complete_state_reported_client_closed = false; const char *pacemakerd_state = PCMK__VALUE_INIT; bool running_with_sbd = false; -GMainLoop *mainloop = NULL; - static bool fatal_error = false; static int child_liveness(pcmkd_child_t *child); @@ -107,7 +105,6 @@ static int start_child(pcmkd_child_t *child); static void pcmk_child_exit(mainloop_child_t *p, int core, int signo, int exitcode); static void pcmk_process_exit(pcmkd_child_t *child); -static gboolean pcmk_shutdown_worker(void *user_data); static void stop_child(pcmkd_child_t *child, int signal); static void @@ -224,8 +221,8 @@ check_next_subdaemon(void *user_data) pcmk_process_exit(child); break; default: - crm_exit(CRM_EX_FATAL); - break; /* static analysis/noreturn */ + pcmk__daemon_quit(&pacemakerd, CRM_EX_FATAL); + return G_SOURCE_REMOVE; } if (++next_child >= PCMK__NELEM(pcmk_children)) { @@ -409,17 +406,21 @@ pcmk_shutdown_worker(void *user_data) return G_SOURCE_CONTINUE; } - g_main_loop_quit(mainloop); - if (fatal_error) { pcmk__notice("Shutting down and staying down after fatal error"); + pcmk__daemon_quit(&pacemakerd, CRM_EX_FATAL); + #if SUPPORT_COROSYNC + /* @FIXME Should this be moved to pacemakerd_cleanup? This is the only + * caller, so maybe not. + */ pcmkd_shutdown_corosync(); #endif - crm_exit(CRM_EX_FATAL); + } else { + pcmk__daemon_quit(&pacemakerd, CRM_EX_OK); } - return G_SOURCE_CONTINUE; + return G_SOURCE_REMOVE; } /* TODO once libqb is taught to juggle with IPC end-points carried over as @@ -537,9 +538,15 @@ start_child(pcmkd_child_t * child) execlp(path, path, (char *) NULL); } + /* If we reach this point, execlp has failed. It's okay to call crm_exit + * here to prevent the fork()ed child process from returning and continuing + * to run. + */ free(path); pcmk__crit("Could not execute subdaemon %s: %s", name, strerror(errno)); crm_exit(CRM_EX_FATAL); + + // Never reached, but makes static analysis happy return pcmk_rc_ok; // Never reached } diff --git a/daemons/schedulerd/pacemaker-schedulerd.c b/daemons/schedulerd/pacemaker-schedulerd.c index e3ce0ca7c4a..d421353b1f5 100644 --- a/daemons/schedulerd/pacemaker-schedulerd.c +++ b/daemons/schedulerd/pacemaker-schedulerd.c @@ -29,15 +29,22 @@ #define SUMMARY PCMK__SERVER_SCHEDULERD " - daemon for calculating a " \ "Pacemaker cluster's response to events" -struct { - gchar **remainder; -} options; +static pcmk__daemon_ipc_fns_t ipc_fns = { + .already_running = pcmk__daemon_ipc_running, +}; + +static pcmk__daemon_t schedulerd = { + .type = pcmk_ipc_schedulerd, + .ec = CRM_EX_OK, + .ipc_fns = &ipc_fns, +}; pcmk__output_t *logger_out = NULL; static pcmk__output_t *out = NULL; -static GMainLoop *mainloop = NULL; -static crm_exit_t exit_code = CRM_EX_OK; +static gchar **processed_args = NULL; +static GOptionContext *context = NULL; +static gchar **remainder = NULL; pcmk__supported_format_t formats[] = { PCMK__SUPPORTED_FORMAT_NONE, @@ -46,8 +53,6 @@ pcmk__supported_format_t formats[] = { { NULL, NULL, NULL } }; -void pengine_shutdown(int nsig); - /* @COMPAT Deprecated since 2.1.8. Use pcmk_list_cluster_options() or * crm_attribute --list-options=cluster instead of querying daemon metadata. * @@ -68,7 +73,7 @@ build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { GOptionContext *context = NULL; GOptionEntry extra_prog_entries[] = { - { G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder, + { G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &remainder, NULL, NULL }, @@ -80,6 +85,27 @@ build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { return context; } +static void +schedulerd_cleanup_cmdline(void) +{ + g_clear_pointer(&processed_args, g_strfreev); + g_clear_pointer(&context, g_option_context_free); + g_clear_pointer(&remainder, g_strfreev); +} + +static void +schedulerd_cleanup(void) +{ + schedulerd_ipc_cleanup(); + schedulerd_unregister_handlers(); +} + +static void +schedulerd_shutdown(int nsig) +{ + pcmk__daemon_quit(&schedulerd, CRM_EX_OK); +} + int main(int argc, char **argv) { @@ -87,23 +113,27 @@ main(int argc, char **argv) int rc = pcmk_rc_ok; GOptionGroup *output_group = NULL; - pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); - gchar **processed_args = pcmk__cmdline_preproc(argv, NULL); - GOptionContext *context = build_arg_context(args, &output_group); + pcmk__common_args_t *args = NULL; + + atexit(schedulerd_cleanup_cmdline); + + args = pcmk__new_common_args(SUMMARY); + processed_args = pcmk__cmdline_preproc(argv, NULL); + context = build_arg_context(args, &output_group); crm_log_preinit(NULL, argc, argv); - mainloop_add_signal(SIGTERM, pengine_shutdown); pcmk__register_formats(output_group, formats); if (!g_option_context_parse_strv(context, &processed_args, &error)) { - exit_code = CRM_EX_USAGE; + schedulerd.ec = CRM_EX_USAGE; goto done; } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if ((rc != pcmk_rc_ok) || (out == NULL)) { - exit_code = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s", + schedulerd.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, schedulerd.ec, + "Error creating output format %s: %s", args->output_ty, pcmk_rc_str(rc)); goto done; } @@ -111,20 +141,20 @@ main(int argc, char **argv) pe__register_messages(out); pcmk__register_lib_messages(out); - if (options.remainder) { - if (g_strv_length(options.remainder) == 1 && - pcmk__str_eq("metadata", options.remainder[0], pcmk__str_casei)) { + if (remainder != NULL) { + if (g_strv_length(remainder) == 1 && + pcmk__str_eq("metadata", remainder[0], pcmk__str_casei)) { rc = scheduler_metadata(out); if (rc != pcmk_rc_ok) { - exit_code = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + schedulerd.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, schedulerd.ec, "Unable to display metadata: %s", pcmk_rc_str(rc)); } } else { - exit_code = CRM_EX_USAGE; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + schedulerd.ec = CRM_EX_USAGE; + g_set_error(&error, PCMK__EXITC_ERROR, schedulerd.ec, "Unsupported extra command line parameters"); } goto done; @@ -137,59 +167,69 @@ main(int argc, char **argv) pcmk__cli_init_logging(PCMK__SERVER_SCHEDULERD, args->verbosity); crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); + + if (schedulerd.ipc_fns->already_running(&schedulerd)) { + schedulerd.ec = CRM_EX_OK; + g_set_error(&error, PCMK__EXITC_ERROR, schedulerd.ec, + "Aborting start-up because a scheduler instance is " + "already active"); + pcmk__crit("%s", error->message); + goto done; + } + pcmk__notice("Starting Pacemaker scheduler"); if (pcmk__daemon_can_write(PCMK_SCHEDULER_INPUT_DIR, NULL) == FALSE) { pcmk__err("Terminating due to bad permissions on " PCMK_SCHEDULER_INPUT_DIR); - exit_code = CRM_EX_FATAL; - g_set_error(&error, PCMK__EXITC_ERROR, exit_code, + schedulerd.ec = CRM_EX_FATAL; + g_set_error(&error, PCMK__EXITC_ERROR, schedulerd.ec, "ERROR: Bad permissions on %s (see logs for details)", PCMK_SCHEDULER_INPUT_DIR); goto done; } - schedulerd_ipc_init(); + if (!schedulerd_ipc_init()) { + schedulerd.ec = CRM_EX_FATAL; + goto done; + } if (pcmk__log_output_new(&logger_out) != pcmk_rc_ok) { - exit_code = CRM_EX_FATAL; + schedulerd.ec = CRM_EX_FATAL; goto done; } pe__register_messages(logger_out); pcmk__register_lib_messages(logger_out); pcmk__output_set_log_level(logger_out, LOG_TRACE); - /* Create the mainloop and run it... */ - mainloop = g_main_loop_new(NULL, FALSE); - pcmk__notice("Pacemaker scheduler successfully started and accepting " - "connections"); - g_main_loop_run(mainloop); + rc = pcmk__daemon_init(&schedulerd); + if (rc != pcmk_rc_ok) { + schedulerd.ec = CRM_EX_ERROR; + g_set_error(&error, PCMK__EXITC_ERROR, schedulerd.ec, + "Error initializing daemon object: %s", + pcmk_rc_str(rc)); + goto done; + } + + mainloop_add_signal(SIGTERM, schedulerd_shutdown); + + pcmk__daemon_run(&schedulerd); done: - g_strfreev(options.remainder); - g_strfreev(processed_args); - pcmk__free_arg_context(context); + schedulerd_cleanup(); pcmk__output_and_clear_error(&error, out); - pengine_shutdown(0); -} - -void -pengine_shutdown(int nsig) -{ - schedulerd_ipc_cleanup(); - schedulerd_unregister_handlers(); if (logger_out != NULL) { - logger_out->finish(logger_out, exit_code, true, NULL); + logger_out->finish(logger_out, schedulerd.ec, true, NULL); g_clear_pointer(&logger_out, pcmk__output_free); } if (out != NULL) { - out->finish(out, exit_code, true, NULL); + out->finish(out, schedulerd.ec, true, NULL); g_clear_pointer(&out, pcmk__output_free); } pcmk__unregister_formats(); - crm_exit(exit_code); + crm_exit(schedulerd.ec); } diff --git a/daemons/schedulerd/pacemaker-schedulerd.h b/daemons/schedulerd/pacemaker-schedulerd.h index fff31e44a02..009d4d7ba97 100644 --- a/daemons/schedulerd/pacemaker-schedulerd.h +++ b/daemons/schedulerd/pacemaker-schedulerd.h @@ -1,5 +1,5 @@ /* - * Copyright 2004-2025 the Pacemaker project contributors + * Copyright 2004-2026 the Pacemaker project contributors * * The version control history for this file may have further details. * @@ -14,7 +14,7 @@ extern pcmk__output_t *logger_out; -void schedulerd_ipc_init(void); +bool schedulerd_ipc_init(void); void schedulerd_ipc_cleanup(void); void schedulerd_unregister_handlers(void); void schedulerd_handle_request(pcmk__request_t *request); diff --git a/daemons/schedulerd/schedulerd_ipc.c b/daemons/schedulerd/schedulerd_ipc.c index 102f4049329..adeba852712 100644 --- a/daemons/schedulerd/schedulerd_ipc.c +++ b/daemons/schedulerd/schedulerd_ipc.c @@ -206,8 +206,9 @@ schedulerd_ipc_cleanup(void) * \internal * \brief Set up schedulerd IPC communication */ -void +bool schedulerd_ipc_init(void) { pcmk__serve_schedulerd_ipc(&ipcs, &ipc_callbacks); + return ipcs != NULL; } diff --git a/include/crm/common/daemon_internal.h b/include/crm/common/daemon_internal.h new file mode 100644 index 00000000000..742ae862f5c --- /dev/null +++ b/include/crm/common/daemon_internal.h @@ -0,0 +1,113 @@ +/* + * Copyright 2026 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +#ifndef PCMK__INCLUDED_CRM_COMMON_INTERNAL_H +#error "Include instead of directly" +#endif + +#ifndef PCMK__CRM_COMMON_DAEMON_INTERNAL__H +#define PCMK__CRM_COMMON_DAEMON_INTERNAL__H + +#include // bool +#include // time_t + +#include // GMainLoop + +#include // pcmk_ipc_server +#include // crm_exit_t + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct pcmk__daemon_s pcmk__daemon_t; + +/*! + * \internal + * \brief Daemon-specific general operations + */ +typedef struct { + /*! + * \internal + * \brief Perform daemon-specific quitting tasks + * + * This function should not perform any cleanup or memory freeing tasks. + * It is meant to terminate anything that needs to happen before the + * main loop quits, as well as to determine whether or not that happens + * at all. + * + * \param[in,out] srv The daemon object + * + * \return \c true if quitting should continue, and \c false if not + */ + bool (*quit)(pcmk__daemon_t *); +} pcmk__daemon_fns_t; + +/*! + * \internal + * \brief Daemon-specific IPC operations + */ +typedef struct { + /*! + * \internal + * \brief Determine if an instance of an IPC server is already running + * + * \param[in,out] d The daemon object + * + * \return \c true if an instance of the daemon is already running, and + * \c false if not + */ + bool (*already_running)(pcmk__daemon_t *); +} pcmk__daemon_ipc_fns_t; + +/*! + * \internal + * \brief This structure describes and manages a single pacemaker daemon + */ +struct pcmk__daemon_s { + //! Daemon type, indexed by the IPC enum + enum pcmk_ipc_server type; + + //! Is the daemon currently shutting down? + bool shutting_down; + + // NOTE: This is set by glib command line processing, hence gboolean + //! Is the daemon running in stand alone mode? + gboolean stand_alone; + + //! When did the daemon start running? + time_t start_time; + + //! What is the exit code of the daemon? + crm_exit_t ec; + + //! Main loop + GMainLoop *mainloop; + + pcmk__daemon_fns_t *fns; + + pcmk__daemon_ipc_fns_t *ipc_fns; +}; + +// IPC functions + +bool pcmk__daemon_ipc_running(pcmk__daemon_t *srv); +bool pcmk__generic_ipc_running(pcmk__daemon_t *srv); + +// Mainloop management functions + +int pcmk__daemon_init(pcmk__daemon_t *srv); +void pcmk__daemon_quit(pcmk__daemon_t *srv, crm_exit_t ec); +void pcmk__daemon_run(pcmk__daemon_t *srv); + +#ifdef __cplusplus +} +#endif + +#endif // PCMK__CRM_COMMON_DAEMON_INTERNAL__H diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h index 88bdac5d242..b3b560f88ec 100644 --- a/include/crm/common/internal.h +++ b/include/crm/common/internal.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am index 2292aef0d1b..290a57db638 100644 --- a/lib/common/Makefile.am +++ b/lib/common/Makefile.am @@ -56,6 +56,7 @@ if BUILD_CIBSECRETS libcrmcommon_la_SOURCES += cib_secrets.c endif libcrmcommon_la_SOURCES += cmdline.c +libcrmcommon_la_SOURCES += daemon.c libcrmcommon_la_SOURCES += digest.c libcrmcommon_la_SOURCES += health.c libcrmcommon_la_SOURCES += io.c diff --git a/lib/common/daemon.c b/lib/common/daemon.c new file mode 100644 index 00000000000..57a067b1fcb --- /dev/null +++ b/lib/common/daemon.c @@ -0,0 +1,173 @@ +/* + * Copyright 2026 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +#include + +#include // SIG* +#include // bool, false, true +#include // NULL +#include // time + +#include // g_clear_pointer, g_main_loop_* + +#include // crm_ipc_*, pcmk_ipc_api_t, pcmk_*_ipc_api +#include // CRM_CHECK +#include // CRM_EX_*, crm_exit, pcmk_rc_* + +/*! + * \internal + * \brief Initialize a previously allocated daemon object + * + * \param[in,out] srv The daemon object + * + * \return Standard Pacemaker return code + */ +int +pcmk__daemon_init(pcmk__daemon_t *srv) +{ + srv->start_time = time(NULL); + + srv->mainloop = g_main_loop_new(NULL, false); + return pcmk_rc_ok; +} + +/*! + * \internal + * \brief Determine if an instance of an IPC server is already running + * + * \param[in,out] srv The daemon object + * + * \return \c true if an instance of \p srv is already running, and \c false if not + * + * \note This function can be used to determine if a daemon is up and running + * since all daemons use IPC. + * + * \note This function only works for those daemons that have been converted + * to use \c pcmk_ipc_api_t as the client interface. Older daemons will + * have to use their own daemon specific method to figure this out. + */ +bool +pcmk__daemon_ipc_running(pcmk__daemon_t *srv) +{ + pcmk_ipc_api_t *old_instance = NULL; + int rc = pcmk_rc_ok; + + rc = pcmk_new_ipc_api(&old_instance, srv->type); + if (rc != pcmk_rc_ok) { + return false; + } + + rc = pcmk__connect_ipc(old_instance, pcmk_ipc_dispatch_sync, 2); + if (rc != pcmk_rc_ok) { + pcmk__debug("No existing %s instance found: %s", + pcmk_ipc_name(old_instance, true), pcmk_rc_str(rc)); + pcmk_free_ipc_api(old_instance); + return false; + } + + pcmk_disconnect_ipc(old_instance); + pcmk_free_ipc_api(old_instance); + return true; +} + +/*! + * \internal + * \brief Quit the daemon's main loop + * + * \param[in,out] srv The daemon object + * \param[in] ec The exit code to assign to the daemon + */ +void +pcmk__daemon_quit(pcmk__daemon_t *srv, crm_exit_t ec) +{ + if (srv->shutting_down) { + return; + } + + if ((srv->fns != NULL) && (srv->fns->quit != NULL)) { + if (!srv->fns->quit(srv)) { + return; + } + } + + pcmk__info("Shutting down %s", pcmk__server_log_name(srv->type)); + + // Tell various functions not to do anything + srv->shutting_down = true; + + srv->ec = ec; + + // Don't respond to signals while shutting down + mainloop_destroy_signal(SIGTERM); + mainloop_destroy_signal(SIGCHLD); + mainloop_destroy_signal(SIGPIPE); + mainloop_destroy_signal(SIGUSR1); + mainloop_destroy_signal(SIGUSR2); + mainloop_destroy_signal(SIGTRAP); + + CRM_CHECK((srv->mainloop != NULL) && g_main_loop_is_running(srv->mainloop), + return); + + g_main_loop_quit(srv->mainloop); +} + +/*! + * \internal + * \brief Run a daemon + * + * \param[in,out] srv The daemon object + */ +void +pcmk__daemon_run(pcmk__daemon_t *srv) +{ + pcmk__notice("Pacemaker %s successfully started and accepting connections", + pcmk__server_log_name(srv->type)); + g_main_loop_run(srv->mainloop); + g_clear_pointer(&srv->mainloop, g_main_loop_unref); +} + +/*! + * \internal + * \brief Determine if an instance of an IPC server is already running + * + * \param[in,out] srv The daemon object + * + * \return \c true if an instance of \p srv is already running, and \c false if not + * + * \note This function only works for older daemons that have not yet been + * converted to use the \c pcmk_ipc_api_t client interface. Once all have + * been updated, this function can be removed. + */ +bool +pcmk__generic_ipc_running(pcmk__daemon_t *srv) +{ + const char *ipc_name = pcmk__server_ipc_name(srv->type); + crm_ipc_t *old_instance = NULL; + int rc = pcmk_rc_ok; + + old_instance = crm_ipc_new(ipc_name, 0); + if (old_instance == NULL) { + /* This is an error - memory allocation failed, etc. - but crm_ipc_new + * will have already logged an error message. + */ + return false; + } + + rc = pcmk__connect_generic_ipc(old_instance); + if (rc != pcmk_rc_ok) { + pcmk__debug("No existing %s instance found: %s", ipc_name, + pcmk_rc_str(rc)); + crm_ipc_destroy(old_instance); + return false; + } + + crm_ipc_close(old_instance); + crm_ipc_destroy(old_instance); + return true; +} diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c index ec92392918f..7fdc8841333 100644 --- a/lib/common/ipc_server.c +++ b/lib/common/ipc_server.c @@ -1065,8 +1065,6 @@ pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c, * \param[out] ipcs_rw New IPC server for read/write CIB manager API * \param[in] ro_cb IPC callbacks for read-only API * \param[in] rw_cb IPC callbacks for read/write and shared-memory APIs - * - * \note This function exits fatally on error. */ void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro, qb_ipcs_service_t **ipcs_rw, @@ -1087,7 +1085,6 @@ pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro, qb_ipcs_service_t **ipcs_rw, pcmk__server_log_name(pcmk_ipc_based)); pcmk__crit("Verify pacemaker and pacemaker_remote are not both " "enabled"); - crm_exit(CRM_EX_FATAL); } } @@ -1125,8 +1122,6 @@ pcmk__serve_controld_ipc(qb_ipcs_service_t **ipcs, * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks - * - * \note This function exits fatally on error. */ void pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, @@ -1142,7 +1137,6 @@ pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, pcmk__server_log_name(pcmk_ipc_attrd)); pcmk__crit("Verify pacemaker and pacemaker_remote are not both " "enabled"); - crm_exit(CRM_EX_FATAL); } } @@ -1152,8 +1146,6 @@ pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs, * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks - * - * \note This function exits fatally on error. */ void pcmk__serve_execd_ipc(qb_ipcs_service_t **ipcs, @@ -1165,7 +1157,6 @@ pcmk__serve_execd_ipc(qb_ipcs_service_t **ipcs, if (*ipcs == NULL) { pcmk__crit("Failed to create %s IPC server; shutting down", pcmk__server_log_name(pcmk_ipc_execd)); - crm_exit(CRM_EX_FATAL); } } @@ -1175,8 +1166,6 @@ pcmk__serve_execd_ipc(qb_ipcs_service_t **ipcs, * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks - * - * \note This function exits fatally on error. */ void pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, @@ -1192,7 +1181,6 @@ pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, pcmk__server_log_name(pcmk_ipc_fenced)); pcmk__crit("Verify pacemaker and pacemaker_remote are not both " "enabled"); - crm_exit(CRM_EX_FATAL); } } @@ -1202,8 +1190,6 @@ pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs, * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks - * - * \note This function exits with CRM_EX_OSERR on error. */ void pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs, @@ -1219,13 +1205,6 @@ pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs, pcmk__server_log_name(pcmk_ipc_pacemakerd)); pcmk__crit("Verify pacemaker and pacemaker_remote are not both " "enabled"); - - /* sub-daemons are observed by pacemakerd. Thus we exit CRM_EX_FATAL - * if we want to prevent pacemakerd from restarting them. - * With pacemakerd we leave the exit-code shown to e.g. systemd - * to what it was prior to moving the code here from pacemakerd.c - */ - crm_exit(CRM_EX_OSERR); } } @@ -1235,9 +1214,6 @@ pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs, * * \param[out] ipcs Where to store newly created IPC server * \param[in] cb IPC callbacks - * - * \return Newly created IPC server - * \note This function exits fatally on error. */ void pcmk__serve_schedulerd_ipc(qb_ipcs_service_t **ipcs, @@ -1251,6 +1227,5 @@ pcmk__serve_schedulerd_ipc(qb_ipcs_service_t **ipcs, if (*ipcs == NULL) { pcmk__crit("Failed to create %s IPC server; shutting down", pcmk__server_log_name(pcmk_ipc_schedulerd)); - crm_exit(CRM_EX_FATAL); } }