diff --git a/raven/channels/adapters/feishu/channel.py b/raven/channels/adapters/feishu/channel.py index 9855c48e..388a097a 100644 --- a/raven/channels/adapters/feishu/channel.py +++ b/raven/channels/adapters/feishu/channel.py @@ -39,6 +39,9 @@ ".pptx": "ppt", } _DEDUP_CAP = 1000 +_RECONNECT_BACKOFF_INITIAL_S = 5.0 +_RECONNECT_BACKOFF_FACTOR = 2.0 +_RECONNECT_BACKOFF_MAX_S = 300.0 class FeishuChannel(ChannelBase): @@ -106,25 +109,44 @@ def _run_ws_supervised(self) -> None: lark_oapi grabs a module-level ``loop = asyncio.get_event_loop()``; giving this thread its own idle loop (and pointing lark's module at - it) avoids clashing with the already-running main loop. Reconnects - with a fixed backoff until the channel is stopped. + it) avoids clashing with the already-running main loop. ``start()`` + blocks while the connection lives (the SDK reconnects transient + drops internally), so it only returns by raising — almost always a + ``ClientException``, i.e. a permanent auth/config failure. Retries + use exponential backoff capped at ``_RECONNECT_BACKOFF_MAX_S``. """ import lark_oapi.ws.client as lark_ws + from lark_oapi.ws.exception import ClientException ws_loop = asyncio.new_event_loop() asyncio.set_event_loop(ws_loop) lark_ws.loop = ws_loop try: + delay = _RECONNECT_BACKOFF_INITIAL_S while self._running: try: self._ws_client.start() + except ClientException as e: + # Permanent auth/config failure: an immediate retry only + # hammers Feishu's auth endpoint, so back off instead. + logger.error("Feishu WebSocket permanent error: {}", e) except Exception as e: logger.warning("Feishu WebSocket error: {}", e) if self._running: - time.sleep(5) + self._sleep_interruptibly(delay) + delay = min(_RECONNECT_BACKOFF_MAX_S, delay * _RECONNECT_BACKOFF_FACTOR) finally: ws_loop.close() + def _sleep_interruptibly(self, seconds: float) -> None: + """Sleep in 1s slices so stop() can cut a long backoff short.""" + deadline = time.monotonic() + seconds + while self._running: + remaining = deadline - time.monotonic() + if remaining <= 0: + return + time.sleep(min(1.0, remaining)) + async def stop(self) -> None: # lark.ws.Client has no stop(); dropping references + exit closes it. self._running = False diff --git a/tests/test_channels_feishu.py b/tests/test_channels_feishu.py index b6694115..87489878 100644 --- a/tests/test_channels_feishu.py +++ b/tests/test_channels_feishu.py @@ -1,6 +1,7 @@ """Tests for the feishu adapter package — inbound content extraction -(content.py), outbound format detection/rendering (cards.py), and group -mention gating. Pure surface; no lark SDK / live connection.""" +(content.py), outbound format detection/rendering (cards.py), group +mention gating, and reconnect backoff supervision. Pure surface; no lark +SDK / live connection.""" import asyncio from types import SimpleNamespace @@ -10,6 +11,8 @@ pytest.importorskip("lark_oapi") +from lark_oapi.ws.exception import ClientException + from raven.channels.adapters.feishu import cards, content from raven.channels.adapters.feishu.channel import FeishuChannel @@ -298,6 +301,62 @@ def test_stop_blocks_zombie_inbound(monkeypatch): assert len(calls) == 1 # zombie delivery dropped +# ── reconnect supervision (exponential backoff) ─────────────────────── + + +def _run_supervised_with_failures(ch, exc, count): + """Drive _run_ws_supervised with start() always raising, recording the + backoff delays, and stopping after `count` failures.""" + ch._ws_client = MagicMock() + ch._ws_client.start.side_effect = exc + ch._running = True + delays = [] + + def fake_sleep(seconds): + delays.append(seconds) + if len(delays) >= count: + ch._running = False + + ch._sleep_interruptibly = fake_sleep + ch._run_ws_supervised() + return delays + + +def test_reconnect_backoff_grows_and_caps(): + delays = _run_supervised_with_failures(_channel(), RuntimeError("down"), count=9) + assert delays == [5.0, 10.0, 20.0, 40.0, 80.0, 160.0, 300.0, 300.0, 300.0] + + +def test_reconnect_permanent_client_exception_also_backs_off(): + """ClientException (bad credentials / conn limit) escapes start() and + must not be retried on the old fixed 5s hammer loop.""" + delays = _run_supervised_with_failures(_channel(), ClientException(99991663, "invalid app_secret"), count=3) + assert delays == [5.0, 10.0, 20.0] + + +def test_reconnect_backoff_never_overflows_beyond_cap(): + """Sustained failure must not overflow the backoff math: once the cap + is reached the delay stays flat and the supervisor keeps running.""" + delays = _run_supervised_with_failures(_channel(), ClientException(99991440, "conn limit"), count=2000) + assert delays[:7] == [5.0, 10.0, 20.0, 40.0, 80.0, 160.0, 300.0] + assert delays[7:] == [300.0] * (len(delays) - 7) + assert len(delays) == 2000 + + +def test_backoff_sleep_cut_short_by_stop(monkeypatch): + ch = _channel() + ch._running = True + slept = [] + + def fake_sleep(seconds): + slept.append(seconds) + ch._running = False + + monkeypatch.setattr("raven.channels.adapters.feishu.channel.time.sleep", fake_sleep) + ch._sleep_interruptibly(300.0) + assert slept == [1.0] # one 1s slice, then stop() ends the backoff early + + # ── contract conformance ───────────────────────────────────────────────