Increase graceful shutdown timeout for prometheus/prometheus2 jobs

jobs/prometheus/monit

@@ -1,5 +1,5 @@
 check process prometheus
   with pidfile /var/vcap/sys/run/prometheus/prometheus.pid
   start program "/var/vcap/jobs/prometheus/bin/prometheus_ctl start"
-  stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop"
+  stop program "/var/vcap/jobs/prometheus/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + p('prometheus.stop_timeout_extra') %> seconds
   group vcap

jobs/prometheus/spec

@@ -61,6 +61,12 @@ properties:
   prometheus.enable_features:
     description: "List of features to enable"
     default: []
+  prometheus.stop_timeout:
+    description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL"
+    default: 110
+  prometheus.stop_timeout_extra:
+    description: "Additional seconds monit waits beyond stop_timeout before declaring the stop failed. Gives bosh-agent headroom so a slow graceful shutdown isn't reported as a failure."
+    default: 10
 
   prometheus.alertmanager.notification_queue_capacity:
     description: "The capacity of the queue for pending alert manager notifications"

jobs/prometheus/templates/bin/prometheus_ctl

@@ -153,7 +153,7 @@ case $1 in
     ;;
 
   stop)
-    kill_and_wait ${PIDFILE}
+    kill_and_wait ${PIDFILE} <%= p('prometheus.stop_timeout') %>
     ;;
 
   *)

jobs/prometheus2/monit

@@ -1,5 +1,5 @@
 check process prometheus2
   with pidfile /var/vcap/sys/run/prometheus2/prometheus.pid
   start program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl start"
-  stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop"
+  stop program "/var/vcap/jobs/prometheus2/bin/prometheus_ctl stop" with timeout <%= p('prometheus.stop_timeout') + p('prometheus.stop_timeout_extra') %> seconds
   group vcap

jobs/prometheus2/spec

@@ -61,6 +61,12 @@ properties:
   prometheus.enable_features:
     description: "List of features to enable"
     default: []
+  prometheus.stop_timeout:
+    description: "Seconds to wait for prometheus to shut down gracefully before sending SIGKILL"
+    default: 110
+  prometheus.stop_timeout_extra:
+    description: "Additional seconds monit waits beyond stop_timeout before declaring the stop failed. Gives bosh-agent headroom so a slow graceful shutdown isn't reported as a failure."
+    default: 10
 
   prometheus.alertmanager.notification_queue_capacity:
     description: "The capacity of the queue for pending alert manager notifications"

jobs/prometheus2/templates/bin/prometheus_ctl

@@ -153,7 +153,7 @@ case $1 in
     ;;
 
   stop)
-    kill_and_wait ${PIDFILE}
+    kill_and_wait ${PIDFILE} <%= p('prometheus.stop_timeout') %>
     ;;
 
   *)