diff --git a/jobs/prometheus/monit b/jobs/prometheus/monit index db389a43..ad3f5122 100644 --- a/jobs/prometheus/monit +++ b/jobs/prometheus/monit @@ -1,5 +1,5 @@ check process prometheus with pidfile /var/vcap/sys/run/prometheus/prometheus.pid start program "/var/vcap/jobs/prometheus/bin/prometheus_ctl start" - stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop" + stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + p('prometheus.stop_timeout_extra') %> seconds group vcap diff --git a/jobs/prometheus/spec b/jobs/prometheus/spec index 62a5ad57..015fc253 100644 --- a/jobs/prometheus/spec +++ b/jobs/prometheus/spec @@ -61,6 +61,12 @@ properties: prometheus.enable_features: description: "List of features to enable" default: [] + prometheus.stop_timeout: + description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL" + default: 110 + prometheus.stop_timeout_extra: + description: "Additional seconds monit waits beyond stop_timeout before declaring the stop failed. Gives bosh-agent headroom so a slow graceful shutdown isn't reported as a failure." + default: 10 prometheus.alertmanager.notification_queue_capacity: description: "The capacity of the queue for pending alert manager notifications" diff --git a/jobs/prometheus/templates/bin/prometheus_ctl b/jobs/prometheus/templates/bin/prometheus_ctl index 9155b877..f51569aa 100644 --- a/jobs/prometheus/templates/bin/prometheus_ctl +++ b/jobs/prometheus/templates/bin/prometheus_ctl @@ -153,7 +153,7 @@ case $1 in ;; stop) - kill_and_wait ${PIDFILE} + kill_and_wait ${PIDFILE} <%= p('prometheus.stop_timeout') %> ;; *) diff --git a/jobs/prometheus2/monit b/jobs/prometheus2/monit index 935dbfb4..1e1569d6 100644 --- a/jobs/prometheus2/monit +++ b/jobs/prometheus2/monit @@ -1,5 +1,5 @@ check process prometheus2 with pidfile /var/vcap/sys/run/prometheus2/prometheus.pid start program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl start" - stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop" + stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + p('prometheus.stop_timeout_extra') %> seconds group vcap diff --git a/jobs/prometheus2/spec b/jobs/prometheus2/spec index 709307d9..014d2019 100644 --- a/jobs/prometheus2/spec +++ b/jobs/prometheus2/spec @@ -61,6 +61,12 @@ properties: prometheus.enable_features: description: "List of features to enable" default: [] + prometheus.stop_timeout: + description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL" + default: 110 + prometheus.stop_timeout_extra: + description: "Additional seconds monit waits beyond stop_timeout before declaring the stop failed. Gives bosh-agent headroom so a slow graceful shutdown isn't reported as a failure." + default: 10 prometheus.alertmanager.notification_queue_capacity: description: "The capacity of the queue for pending alert manager notifications" diff --git a/jobs/prometheus2/templates/bin/prometheus_ctl b/jobs/prometheus2/templates/bin/prometheus_ctl index 5ae20d17..80215dc1 100644 --- a/jobs/prometheus2/templates/bin/prometheus_ctl +++ b/jobs/prometheus2/templates/bin/prometheus_ctl @@ -153,7 +153,7 @@ case $1 in ;; stop) - kill_and_wait ${PIDFILE} + kill_and_wait ${PIDFILE} <%= p('prometheus.stop_timeout') %> ;; *)