From 7d59d3eb4b278a689d0c09be75f9e5345b137063 Mon Sep 17 00:00:00 2001 From: Yevgen Bykov Date: Mon, 6 Jul 2026 16:19:14 +0200 Subject: [PATCH 1/2] Increase graceful shutdown timeout for prometheus/prometheus2 jobs --- jobs/prometheus/monit | 2 +- jobs/prometheus/spec | 3 +++ jobs/prometheus/templates/bin/prometheus_ctl | 2 +- jobs/prometheus2/monit | 2 +- jobs/prometheus2/spec | 3 +++ jobs/prometheus2/templates/bin/prometheus_ctl | 2 +- 6 files changed, 10 insertions(+), 4 deletions(-) diff --git a/jobs/prometheus/monit b/jobs/prometheus/monit index db389a435..71f6b9473 100644 --- a/jobs/prometheus/monit +++ b/jobs/prometheus/monit @@ -1,5 +1,5 @@ check process prometheus with pidfile /var/vcap/sys/run/prometheus/prometheus.pid start program "/var/vcap/jobs/prometheus/bin/prometheus_ctl start" - stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop" + stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + 10 %> seconds group vcap diff --git a/jobs/prometheus/spec b/jobs/prometheus/spec index 62a5ad57f..da5fa6e47 100644 --- a/jobs/prometheus/spec +++ b/jobs/prometheus/spec @@ -61,6 +61,9 @@ properties: prometheus.enable_features: description: "List of features to enable" default: [] + prometheus.stop_timeout: + description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL" + default: 110 prometheus.alertmanager.notification_queue_capacity: description: "The capacity of the queue for pending alert manager notifications" diff --git a/jobs/prometheus/templates/bin/prometheus_ctl b/jobs/prometheus/templates/bin/prometheus_ctl index 9155b8770..f51569aaf 100644 --- a/jobs/prometheus/templates/bin/prometheus_ctl +++ b/jobs/prometheus/templates/bin/prometheus_ctl @@ -153,7 +153,7 @@ case $1 in ;; stop) - kill_and_wait ${PIDFILE} + kill_and_wait ${PIDFILE} <%= p('prometheus.stop_timeout') %> ;; *) diff --git a/jobs/prometheus2/monit b/jobs/prometheus2/monit index 935dbfb4d..e1a091135 100644 --- a/jobs/prometheus2/monit +++ b/jobs/prometheus2/monit @@ -1,5 +1,5 @@ check process prometheus2 with pidfile /var/vcap/sys/run/prometheus2/prometheus.pid start program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl start" - stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop" + stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + 10 %> seconds group vcap diff --git a/jobs/prometheus2/spec b/jobs/prometheus2/spec index 709307d9b..44db5842e 100644 --- a/jobs/prometheus2/spec +++ b/jobs/prometheus2/spec @@ -61,6 +61,9 @@ properties: prometheus.enable_features: description: "List of features to enable" default: [] + prometheus.stop_timeout: + description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL" + default: 110 prometheus.alertmanager.notification_queue_capacity: description: "The capacity of the queue for pending alert manager notifications" diff --git a/jobs/prometheus2/templates/bin/prometheus_ctl b/jobs/prometheus2/templates/bin/prometheus_ctl index 5ae20d17c..80215dc19 100644 --- a/jobs/prometheus2/templates/bin/prometheus_ctl +++ b/jobs/prometheus2/templates/bin/prometheus_ctl @@ -153,7 +153,7 @@ case $1 in ;; stop) - kill_and_wait ${PIDFILE} + kill_and_wait ${PIDFILE} <%= p('prometheus.stop_timeout') %> ;; *) From 1370bea2f5d7b13d0372e5bea17e86d436976a95 Mon Sep 17 00:00:00 2001 From: Yevgen Bykov Date: Wed, 5 Aug 2026 14:13:12 +0200 Subject: [PATCH 2/2] Make extra shutdown timeout for prometheus/prometheus2 jobs also configurable --- jobs/prometheus/monit | 2 +- jobs/prometheus/spec | 3 +++ jobs/prometheus2/monit | 2 +- jobs/prometheus2/spec | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jobs/prometheus/monit b/jobs/prometheus/monit index 71f6b9473..ad3f5122a 100644 --- a/jobs/prometheus/monit +++ b/jobs/prometheus/monit @@ -1,5 +1,5 @@ check process prometheus with pidfile /var/vcap/sys/run/prometheus/prometheus.pid start program "/var/vcap/jobs/prometheus/bin/prometheus_ctl start" - stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + 10 %> seconds + stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + p('prometheus.stop_timeout_extra') %> seconds group vcap diff --git a/jobs/prometheus/spec b/jobs/prometheus/spec index da5fa6e47..015fc2533 100644 --- a/jobs/prometheus/spec +++ b/jobs/prometheus/spec @@ -64,6 +64,9 @@ properties: prometheus.stop_timeout: description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL" default: 110 + prometheus.stop_timeout_extra: + description: "Additional seconds monit waits beyond stop_timeout before declaring the stop failed. Gives bosh-agent headroom so a slow graceful shutdown isn't reported as a failure." + default: 10 prometheus.alertmanager.notification_queue_capacity: description: "The capacity of the queue for pending alert manager notifications" diff --git a/jobs/prometheus2/monit b/jobs/prometheus2/monit index e1a091135..1e1569d64 100644 --- a/jobs/prometheus2/monit +++ b/jobs/prometheus2/monit @@ -1,5 +1,5 @@ check process prometheus2 with pidfile /var/vcap/sys/run/prometheus2/prometheus.pid start program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl start" - stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + 10 %> seconds + stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + p('prometheus.stop_timeout_extra') %> seconds group vcap diff --git a/jobs/prometheus2/spec b/jobs/prometheus2/spec index 44db5842e..014d2019d 100644 --- a/jobs/prometheus2/spec +++ b/jobs/prometheus2/spec @@ -64,6 +64,9 @@ properties: prometheus.stop_timeout: description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL" default: 110 + prometheus.stop_timeout_extra: + description: "Additional seconds monit waits beyond stop_timeout before declaring the stop failed. Gives bosh-agent headroom so a slow graceful shutdown isn't reported as a failure." + default: 10 prometheus.alertmanager.notification_queue_capacity: description: "The capacity of the queue for pending alert manager notifications"