fix(channels): back off exponentially on feishu ws reconnect

raven/channels/adapters/feishu/channel.py

@@ -39,6 +39,9 @@
     ".pptx": "ppt",
 }
 _DEDUP_CAP = 1000
+_RECONNECT_BACKOFF_INITIAL_S = 5.0
+_RECONNECT_BACKOFF_FACTOR = 2.0
+_RECONNECT_BACKOFF_MAX_S = 300.0
 
 
 class FeishuChannel(ChannelBase):
@@ -106,25 +109,44 @@ def _run_ws_supervised(self) -> None:
 
         lark_oapi grabs a module-level ``loop = asyncio.get_event_loop()``;
         giving this thread its own idle loop (and pointing lark's module at
-        it) avoids clashing with the already-running main loop. Reconnects
-        with a fixed backoff until the channel is stopped.
+        it) avoids clashing with the already-running main loop. ``start()``
+        blocks while the connection lives (the SDK reconnects transient
+        drops internally), so it only returns by raising — almost always a
+        ``ClientException``, i.e. a permanent auth/config failure. Retries
+        use exponential backoff capped at ``_RECONNECT_BACKOFF_MAX_S``.
         """
         import lark_oapi.ws.client as lark_ws
+        from lark_oapi.ws.exception import ClientException
 
         ws_loop = asyncio.new_event_loop()
         asyncio.set_event_loop(ws_loop)
         lark_ws.loop = ws_loop
         try:
+            delay = _RECONNECT_BACKOFF_INITIAL_S
             while self._running:
                 try:
                     self._ws_client.start()
+                except ClientException as e:
+                    # Permanent auth/config failure: an immediate retry only
+                    # hammers Feishu's auth endpoint, so back off instead.
+                    logger.error("Feishu WebSocket permanent error: {}", e)
                 except Exception as e:
                     logger.warning("Feishu WebSocket error: {}", e)
                 if self._running:
-                    time.sleep(5)
+                    self._sleep_interruptibly(delay)
+                    delay = min(_RECONNECT_BACKOFF_MAX_S, delay * _RECONNECT_BACKOFF_FACTOR)
         finally:
             ws_loop.close()
 
+    def _sleep_interruptibly(self, seconds: float) -> None:
+        """Sleep in 1s slices so stop() can cut a long backoff short."""
+        deadline = time.monotonic() + seconds
+        while self._running:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return
+            time.sleep(min(1.0, remaining))
+
     async def stop(self) -> None:
         # lark.ws.Client has no stop(); dropping references + exit closes it.
         self._running = False

tests/test_channels_feishu.py

@@ -1,6 +1,7 @@
 """Tests for the feishu adapter package — inbound content extraction
-(content.py), outbound format detection/rendering (cards.py), and group
-mention gating. Pure surface; no lark SDK / live connection."""
+(content.py), outbound format detection/rendering (cards.py), group
+mention gating, and reconnect backoff supervision. Pure surface; no lark
+SDK / live connection."""
 
 import asyncio
 from types import SimpleNamespace
@@ -10,6 +11,8 @@
 
 pytest.importorskip("lark_oapi")
 
+from lark_oapi.ws.exception import ClientException
+
 from raven.channels.adapters.feishu import cards, content
 from raven.channels.adapters.feishu.channel import FeishuChannel
 
@@ -298,6 +301,62 @@ def test_stop_blocks_zombie_inbound(monkeypatch):
     assert len(calls) == 1  # zombie delivery dropped
 
 
+# ── reconnect supervision (exponential backoff) ───────────────────────
+
+
+def _run_supervised_with_failures(ch, exc, count):
+    """Drive _run_ws_supervised with start() always raising, recording the
+    backoff delays, and stopping after `count` failures."""
+    ch._ws_client = MagicMock()
+    ch._ws_client.start.side_effect = exc
+    ch._running = True
+    delays = []
+
+    def fake_sleep(seconds):
+        delays.append(seconds)
+        if len(delays) >= count:
+            ch._running = False
+
+    ch._sleep_interruptibly = fake_sleep
+    ch._run_ws_supervised()
+    return delays
+
+
+def test_reconnect_backoff_grows_and_caps():
+    delays = _run_supervised_with_failures(_channel(), RuntimeError("down"), count=9)
+    assert delays == [5.0, 10.0, 20.0, 40.0, 80.0, 160.0, 300.0, 300.0, 300.0]
+
+
+def test_reconnect_permanent_client_exception_also_backs_off():
+    """ClientException (bad credentials / conn limit) escapes start() and
+    must not be retried on the old fixed 5s hammer loop."""
+    delays = _run_supervised_with_failures(_channel(), ClientException(99991663, "invalid app_secret"), count=3)
+    assert delays == [5.0, 10.0, 20.0]
+
+
+def test_reconnect_backoff_never_overflows_beyond_cap():
+    """Sustained failure must not overflow the backoff math: once the cap
+    is reached the delay stays flat and the supervisor keeps running."""
+    delays = _run_supervised_with_failures(_channel(), ClientException(99991440, "conn limit"), count=2000)
+    assert delays[:7] == [5.0, 10.0, 20.0, 40.0, 80.0, 160.0, 300.0]
+    assert delays[7:] == [300.0] * (len(delays) - 7)
+    assert len(delays) == 2000
+
+
+def test_backoff_sleep_cut_short_by_stop(monkeypatch):
+    ch = _channel()
+    ch._running = True
+    slept = []
+
+    def fake_sleep(seconds):
+        slept.append(seconds)
+        ch._running = False
+
+    monkeypatch.setattr("raven.channels.adapters.feishu.channel.time.sleep", fake_sleep)
+    ch._sleep_interruptibly(300.0)
+    assert slept == [1.0]  # one 1s slice, then stop() ends the backoff early
+
+
 # ── contract conformance ───────────────────────────────────────────────