From 9f31a20c21ed52ba7814ae829bbc2245d29dd7d0 Mon Sep 17 00:00:00 2001 From: LivXue Date: Sat, 29 Aug 2026 19:32:31 +0800 Subject: [PATCH 1/2] fix(channels): back off exponentially on feishu ws reconnect The reconnect loop retried on a fixed 5s sleep forever. lark-oapi's ws Client.start() blocks while the connection lives and only returns by raising, almost always a ClientException (bad credentials or conn limit exceeded), so the old loop hammered Feishu's auth endpoint every 5s on permanent failures. Retries now use exponential backoff capped at 300s, and stop() can interrupt the sleep early. Co-authored-by: Claude (deepseek-v4-pro) --- raven/channels/adapters/feishu/channel.py | 32 ++++++++++++-- tests/test_channels_feishu.py | 54 ++++++++++++++++++++++- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/raven/channels/adapters/feishu/channel.py b/raven/channels/adapters/feishu/channel.py index 9855c48e..005733ac 100644 --- a/raven/channels/adapters/feishu/channel.py +++ b/raven/channels/adapters/feishu/channel.py @@ -39,6 +39,9 @@ ".pptx": "ppt", } _DEDUP_CAP = 1000 +_RECONNECT_BACKOFF_INITIAL_S = 5.0 +_RECONNECT_BACKOFF_FACTOR = 2.0 +_RECONNECT_BACKOFF_MAX_S = 300.0 class FeishuChannel(ChannelBase): @@ -106,25 +109,48 @@ def _run_ws_supervised(self) -> None: lark_oapi grabs a module-level ``loop = asyncio.get_event_loop()``; giving this thread its own idle loop (and pointing lark's module at - it) avoids clashing with the already-running main loop. Reconnects - with a fixed backoff until the channel is stopped. + it) avoids clashing with the already-running main loop. ``start()`` + blocks while the connection lives (the SDK reconnects transient + drops internally), so it only returns by raising — almost always a + ``ClientException``, i.e. a permanent auth/config failure. Retries + use exponential backoff capped at ``_RECONNECT_BACKOFF_MAX_S``. """ import lark_oapi.ws.client as lark_ws + from lark_oapi.ws.exception import ClientException ws_loop = asyncio.new_event_loop() asyncio.set_event_loop(ws_loop) lark_ws.loop = ws_loop try: + attempt = 0 while self._running: try: self._ws_client.start() + except ClientException as e: + # Permanent auth/config failure: an immediate retry only + # hammers Feishu's auth endpoint, so back off instead. + logger.error("Feishu WebSocket permanent error: {}", e) except Exception as e: logger.warning("Feishu WebSocket error: {}", e) if self._running: - time.sleep(5) + delay = min( + _RECONNECT_BACKOFF_MAX_S, + _RECONNECT_BACKOFF_INITIAL_S * (_RECONNECT_BACKOFF_FACTOR**attempt), + ) + attempt += 1 + self._sleep_interruptibly(delay) finally: ws_loop.close() + def _sleep_interruptibly(self, seconds: float) -> None: + """Sleep in 1s slices so stop() can cut a long backoff short.""" + deadline = time.monotonic() + seconds + while self._running: + remaining = deadline - time.monotonic() + if remaining <= 0: + return + time.sleep(min(1.0, remaining)) + async def stop(self) -> None: # lark.ws.Client has no stop(); dropping references + exit closes it. self._running = False diff --git a/tests/test_channels_feishu.py b/tests/test_channels_feishu.py index b6694115..e4d29623 100644 --- a/tests/test_channels_feishu.py +++ b/tests/test_channels_feishu.py @@ -1,6 +1,7 @@ """Tests for the feishu adapter package — inbound content extraction -(content.py), outbound format detection/rendering (cards.py), and group -mention gating. Pure surface; no lark SDK / live connection.""" +(content.py), outbound format detection/rendering (cards.py), group +mention gating, and reconnect backoff supervision. Pure surface; no lark +SDK / live connection.""" import asyncio from types import SimpleNamespace @@ -10,6 +11,8 @@ pytest.importorskip("lark_oapi") +from lark_oapi.ws.exception import ClientException + from raven.channels.adapters.feishu import cards, content from raven.channels.adapters.feishu.channel import FeishuChannel @@ -298,6 +301,53 @@ def test_stop_blocks_zombie_inbound(monkeypatch): assert len(calls) == 1 # zombie delivery dropped +# ── reconnect supervision (exponential backoff) ─────────────────────── + + +def _run_supervised_with_failures(ch, exc, count): + """Drive _run_ws_supervised with start() always raising, recording the + backoff delays, and stopping after `count` failures.""" + ch._ws_client = MagicMock() + ch._ws_client.start.side_effect = exc + ch._running = True + delays = [] + + def fake_sleep(seconds): + delays.append(seconds) + if len(delays) >= count: + ch._running = False + + ch._sleep_interruptibly = fake_sleep + ch._run_ws_supervised() + return delays + + +def test_reconnect_backoff_grows_and_caps(): + delays = _run_supervised_with_failures(_channel(), RuntimeError("down"), count=9) + assert delays == [5.0, 10.0, 20.0, 40.0, 80.0, 160.0, 300.0, 300.0, 300.0] + + +def test_reconnect_permanent_client_exception_also_backs_off(): + """ClientException (bad credentials / conn limit) escapes start() and + must not be retried on the old fixed 5s hammer loop.""" + delays = _run_supervised_with_failures(_channel(), ClientException(99991663, "invalid app_secret"), count=3) + assert delays == [5.0, 10.0, 20.0] + + +def test_backoff_sleep_cut_short_by_stop(monkeypatch): + ch = _channel() + ch._running = True + slept = [] + + def fake_sleep(seconds): + slept.append(seconds) + ch._running = False + + monkeypatch.setattr("raven.channels.adapters.feishu.channel.time.sleep", fake_sleep) + ch._sleep_interruptibly(300.0) + assert slept == [1.0] # one 1s slice, then stop() ends the backoff early + + # ── contract conformance ─────────────────────────────────────────────── From 01411cc1418e82589568b1ee41426cc94eacfc02 Mon Sep 17 00:00:00 2001 From: LivXue Date: Sat, 29 Aug 2026 20:38:36 +0800 Subject: [PATCH 2/2] fix(channels): carry capped delay in feishu reconnect backoff The per-attempt exponent 2.0**attempt overflows after 1,024 failed starts, which raises OverflowError outside the try blocks and kills the reconnect thread while _running stays true. Grow the delay iteratively instead and clamp it to the cap, so sustained failures never overflow the backoff math. Co-authored-by: Claude (deepseek-v4-pro) --- raven/channels/adapters/feishu/channel.py | 8 ++------ tests/test_channels_feishu.py | 9 +++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/raven/channels/adapters/feishu/channel.py b/raven/channels/adapters/feishu/channel.py index 005733ac..388a097a 100644 --- a/raven/channels/adapters/feishu/channel.py +++ b/raven/channels/adapters/feishu/channel.py @@ -122,7 +122,7 @@ def _run_ws_supervised(self) -> None: asyncio.set_event_loop(ws_loop) lark_ws.loop = ws_loop try: - attempt = 0 + delay = _RECONNECT_BACKOFF_INITIAL_S while self._running: try: self._ws_client.start() @@ -133,12 +133,8 @@ def _run_ws_supervised(self) -> None: except Exception as e: logger.warning("Feishu WebSocket error: {}", e) if self._running: - delay = min( - _RECONNECT_BACKOFF_MAX_S, - _RECONNECT_BACKOFF_INITIAL_S * (_RECONNECT_BACKOFF_FACTOR**attempt), - ) - attempt += 1 self._sleep_interruptibly(delay) + delay = min(_RECONNECT_BACKOFF_MAX_S, delay * _RECONNECT_BACKOFF_FACTOR) finally: ws_loop.close() diff --git a/tests/test_channels_feishu.py b/tests/test_channels_feishu.py index e4d29623..87489878 100644 --- a/tests/test_channels_feishu.py +++ b/tests/test_channels_feishu.py @@ -334,6 +334,15 @@ def test_reconnect_permanent_client_exception_also_backs_off(): assert delays == [5.0, 10.0, 20.0] +def test_reconnect_backoff_never_overflows_beyond_cap(): + """Sustained failure must not overflow the backoff math: once the cap + is reached the delay stays flat and the supervisor keeps running.""" + delays = _run_supervised_with_failures(_channel(), ClientException(99991440, "conn limit"), count=2000) + assert delays[:7] == [5.0, 10.0, 20.0, 40.0, 80.0, 160.0, 300.0] + assert delays[7:] == [300.0] * (len(delays) - 7) + assert len(delays) == 2000 + + def test_backoff_sleep_cut_short_by_stop(monkeypatch): ch = _channel() ch._running = True