From 493f3283f495572d7d314e51eab53dd0175a9884 Mon Sep 17 00:00:00 2001 From: Max Nitze Date: Tue, 8 Sep 2026 16:45:26 +0200 Subject: [PATCH 1/3] notify/webex: drain and close the response body Signed-off-by: Max Nitze --- notify/webex/webex.go | 1 + 1 file changed, 1 insertion(+) diff --git a/notify/webex/webex.go b/notify/webex/webex.go index 949bb614f8..fe1bc477d0 100644 --- a/notify/webex/webex.go +++ b/notify/webex/webex.go @@ -105,6 +105,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) if err != nil { return true, notify.RedactURL(err) } + defer notify.Drain(resp) shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body) if err != nil { From 54967523c9d66c660c4fa2ee6efe16f47e67308c Mon Sep 17 00:00:00 2001 From: Max Nitze Date: Tue, 8 Sep 2026 16:45:45 +0200 Subject: [PATCH 2/3] notify/webex: retry on 429 Signed-off-by: Max Nitze --- notify/webex/webex.go | 2 +- notify/webex/webex_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/notify/webex/webex.go b/notify/webex/webex.go index fe1bc477d0..84dfece8eb 100644 --- a/notify/webex/webex.go +++ b/notify/webex/webex.go @@ -54,7 +54,7 @@ func New(c *config.WebexConfig, t *template.Template, l *slog.Logger, httpOpts . tmpl: t, logger: l, client: client, - retrier: ¬ify.Retrier{}, + retrier: ¬ify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}}, } return n, nil diff --git a/notify/webex/webex_test.go b/notify/webex/webex_test.go index eb12cfc2b6..029789a650 100644 --- a/notify/webex/webex_test.go +++ b/notify/webex/webex_test.go @@ -49,7 +49,8 @@ func TestWebexRetry(t *testing.T) { ) require.NoError(t, err) - for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) { + retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests) + for statusCode, expected := range test.RetryTests(retryCodes) { actual, _ := notifier.retrier.Check(statusCode, nil) require.Equal(t, expected, actual, "error on status %d", statusCode) } From 0d1e2d73df2f57c8704aeef375b88a17807ef3cb Mon Sep 17 00:00:00 2001 From: Max Nitze Date: Tue, 8 Sep 2026 16:51:20 +0200 Subject: [PATCH 3/3] notify/webex: honor Retry-After header on 429 Signed-off-by: Max Nitze --- notify/webex/webex.go | 11 +++++ notify/webex/webex_test.go | 82 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/notify/webex/webex.go b/notify/webex/webex.go index 84dfece8eb..9f3571981c 100644 --- a/notify/webex/webex.go +++ b/notify/webex/webex.go @@ -19,6 +19,7 @@ import ( "encoding/json" "log/slog" "net/http" + "time" commoncfg "github.com/prometheus/common/config" @@ -102,6 +103,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) } resp, err := notify.PostJSON(ctx, n.client, n.conf.APIURL.String(), &payload) + received := time.Now() if err != nil { return true, notify.RedactURL(err) } @@ -109,6 +111,15 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body) if err != nil { + if resp.StatusCode == http.StatusTooManyRequests { + if d := notify.ParseRetryAfter(resp.Header, received); d > 0 { + logger.Warn("Rate limited by Webex, waiting before retry", "retry_after_secs", d.Seconds()) + select { + case <-time.After(d): + case <-ctx.Done(): + } + } + } return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err) } diff --git a/notify/webex/webex_test.go b/notify/webex/webex_test.go index 029789a650..ef5e2d2315 100644 --- a/notify/webex/webex_test.go +++ b/notify/webex/webex_test.go @@ -170,6 +170,88 @@ func TestWebexTemplating(t *testing.T) { } } +func TestWebexRetryAfterSleep(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + + notifier, err := New( + &config.WebexConfig{ + HTTPConfig: &commoncfg.HTTPClientConfig{}, + APIURL: &amcommoncfg.URL{URL: u}, + }, + test.CreateTmpl(t), + promslog.NewNopLogger(), + ) + require.NoError(t, err) + + ctx := notify.WithGroupKey(context.Background(), "1") + alert := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{"lbl1": "val1"}, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + }, + } + + start := time.Now() + retry, err := notifier.Notify(ctx, alert) + elapsed := time.Since(start) + + require.True(t, retry) + require.Error(t, err) + require.GreaterOrEqual(t, elapsed, 1*time.Second, "should have waited at least 1 second for Retry-After") +} + +func TestWebexRetryAfterContextCancelled(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + u, err := url.Parse(srv.URL) + require.NoError(t, err) + + notifier, err := New( + &config.WebexConfig{ + HTTPConfig: &commoncfg.HTTPClientConfig{}, + APIURL: &amcommoncfg.URL{URL: u}, + }, + test.CreateTmpl(t), + promslog.NewNopLogger(), + ) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + ctx = notify.WithGroupKey(ctx, "1") + + // Cancel context after a short delay to interrupt the Retry-After sleep. + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() + + alert := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{"lbl1": "val1"}, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + }, + } + + start := time.Now() + retry, err := notifier.Notify(ctx, alert) + elapsed := time.Since(start) + + require.True(t, retry) + require.Error(t, err) + require.Less(t, elapsed, 2*time.Second, "should not have waited the full Retry-After duration") +} + func TestWebexFailureReason(t *testing.T) { for _, tc := range []struct { name string