From bd6d43a30531570d1a1e055d2c28523e697d3931 Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Sun, 9 Aug 2026 19:41:17 +0000
Subject: [PATCH 01/16] agent: fast queue rotation does not requiring the
current server to be online
---
plans/2026-08-09-fast-queue-rotation-plan.md | 69 +++++++++++++++++
rfcs/2026-08-09-fast-queue-rotation.md | 79 ++++++++++++++++++++
2 files changed, 148 insertions(+)
create mode 100644 plans/2026-08-09-fast-queue-rotation-plan.md
create mode 100644 rfcs/2026-08-09-fast-queue-rotation.md
diff --git a/plans/2026-08-09-fast-queue-rotation-plan.md b/plans/2026-08-09-fast-queue-rotation-plan.md
new file mode 100644
index 000000000..cc57ced51
--- /dev/null
+++ b/plans/2026-08-09-fast-queue-rotation-plan.md
@@ -0,0 +1,69 @@
+# Fast queue rotation — implementation plan
+
+Branch: ep/drop-agent-versions. RFC: ../rfcs/2026-08-09-fast-queue-rotation.md.
+
+No new agent message. Each side detects the rotation from the replacement
+reference (`dbReplaceQueueId`) on its own new queue.
+
+## New definitions
+
+Agent/Protocol.hs
+- Gate on the existing `rpcAddressSMPAgentVersion` (v8). Renaming it to cover both features can be considered separately; no alias.
+- `SndSwitchStatus` constructor `SSSecuringQueue` with StrEncoding/JSON/field instances.
+- `InternalCommand` constructor `ICQSndSecure SMP.SenderId` with tag.
+
+Agent/Store(.AgentStore)
+- `SSSecuringQueue` in `setSndSwitchStatus` mapping.
+- `ICQSndSecure` in internal command store/parse.
+
+Agent.hs
+- `storeRotationConfirmation` — variant of `storeConfirmation` that stores
+ `AgentConfirmation {e2eEncryption_ = Nothing, encConnInfo = ""}` with msgType
+ `AM_CONN_INFO`, `sndMsgPrepData_ = Nothing`, no ratchet step.
+
+## Sender B — Agent.hs
+
+`qAddMsg` (new-queue-sender branch)
+- After `addConnSndQueue` for the new snd queue, choose path:
+ - fast: `connAgentVersion cData' >= rpcAddressSMPAgentVersion && senderCanSecure (queueMode of new queue)`.
+ - else: current `QKEY` path (unchanged).
+- Fast: `enqueueCommand c "" connId (Just newSrv) $ AInternalCommand $ ICQSndSecure sndId`;
+ `setSndSwitchStatus db sq $ Just SSSecuringQueue`; notify `SWITCH QDSnd SPStarted`.
+
+`ICQSndSecure sId` (new, in runCommandProcessing, model on `ICQSecure`)
+- Find new snd queue `sq'` in sqs with `dbReplaceQueueId = Just _`, status `New`.
+- `agentSecureSndQueue` (SKEY) on `sq'`; set `Secured`.
+- `storeRotationConfirmation c cData sq'`; `submitPendingMsg c sq'`.
+
+runSmpQueueMsgDelivery — success `AM_CONN_INFO`
+- Branch on `sq` replacement reference:
+ - `dbReplaceQueueId = Just _`: rotation. Mirror `AM_QTEST_` — `checkSQSwchStatus sq' SSSecuringQueue`; remove old snd queue; `setSndQueuePrimary`; `deleteConnSndQueue`; notify `SWITCH QDSnd SPCompleted`.
+ - otherwise: current join completion (`CON`/`setStatus`).
+
+runSmpQueueMsgDelivery — AUTH `AM_CONN_INFO`
+- Same branch: rotation → `qError msgId "rotation confirmation: AUTH"`; otherwise current `connError NOT_AVAILABLE`.
+
+## Recipient A — Agent.hs
+
+`smpConfirmation`, `New` case, before the RcvConnection/DuplexConnection split
+- If `dbReplaceQueueId rq = Just replacedId` (rq is R'):
+ - `setRcvQueueConfirmedE2E db rq (C.dh' e2ePubKey e2ePrivKey) (min v phVer)` (empty encConnInfo not decrypted).
+ - `setRcvQueuePrimary`, `setRcvQueueStatus rq Active`.
+ - `setRcvSwitchStatus db replaced $ Just RSReceivedMessage`.
+ - finalize: update connection, notify `SWITCH QDRcv SPCompleted`, delete old queue via `deleteConnQueues` (bounded best-effort), not `ICQDelete`.
+ - no `CON`/`INFO`.
+
+## Switch state
+
+- A: `RSSendingQADD` → (confirmation on R') → `RSReceivedMessage` → done. `RSSendingQUSE` skipped for fast path.
+- B: `SSSecuringQueue` → (confirmation delivered) → done.
+
+## Unchanged
+
+QADD wire format, slow QKEY/QUSE/QTEST path, abort (`canAbortRcvSwitch` still allows through `RSSendingQADD`).
+
+## Tests
+
+- new/new: rotation completes with current server stopped after QADD.
+- new/old and old/new: fall back to QKEY path, complete.
+- redundancy: two current queues, rotate one, secret established on R' only.
diff --git a/rfcs/2026-08-09-fast-queue-rotation.md b/rfcs/2026-08-09-fast-queue-rotation.md
new file mode 100644
index 000000000..b8e87988c
--- /dev/null
+++ b/rfcs/2026-08-09-fast-queue-rotation.md
@@ -0,0 +1,79 @@
+---
+Proposed: 2026-08-09
+Protocol: agent-protocol v8
+Diagram: ./diagrams/duplex-messaging/queue-rotation-fast.svg
+---
+
+# Fast queue rotation (SKEY)
+
+## Problem
+
+In the current rotation the peer returns `QKEY` to the initiator over the
+initiator's current receiving queue. When that queue's server is unavailable the
+rotation cannot complete, so a client cannot move away from a failed server.
+
+## Solution
+
+When both agents support v8, the peer secures the new queue itself with `SKEY`,
+as in the fast connection handshake, and sends an empty confirmation to it
+instead of returning `QKEY`. This replaces the `QKEY` and `QUSE` exchange. The
+initiator's current server is used only to delete the old queue at the end, and
+that deletion does not block completion.
+
+Roles: A initiates (its receiving queue moves; A receives on R'); B is the peer
+(B holds the sending queue to A, and secures R' and sends to it).
+
+Sequence (as drawn in the diagram):
+
+ A -> R' : create new queue (messaging mode, SKEY allowed)
+ A -> S -> B : QADD(R') (over A's sending queue; A's current server untouched)
+ B -> R' : SKEY (B secures R')
+ B -> R' : confirmation (empty)
+ R' -> A : confirmation (A derives R' secret, completes the switch)
+ A -> R (old) : delete (best effort, skipped if the server is down)
+ B -> R' : messages
+
+## Confirmation
+
+The confirmation is the only message a recipient can read on a queue that is not
+yet secured, and it is what establishes the queue's shared secret. This is why
+it, rather than an ordinary message, carries the rotation: the earlier abandoned
+attempt sent the securing message as an ordinary message, which required the
+secret it was establishing.
+
+The confirmation body is empty: both parties already know each other, so no
+profile or reply queue is sent. The body is sealed by the queue's NaCl box
+(keyed by the shared secret being established) and is not additionally encrypted
+with the double ratchet, so the rotation does not advance the message ratchet.
+No ratchet key parameters are included, as in the SKEY handshake.
+
+No new agent message is defined. Both parties already know which queue is being
+replaced — the initiator because it created R' to replace a specific queue, the
+peer because `QADD` states the replaced sending-queue address — so no marker is
+needed on the wire.
+
+## Per-queue secret
+
+R' secret is a fresh Diffie-Hellman between the peer's queue key, sent in the
+confirmation header, and the initiator's R' key. It does not depend on any
+current queue, so redundancy (several current queues) is unaffected.
+
+## Compatibility
+
+Fast rotation runs only when the connection's agreed agent protocol version is 8
+or higher; the peer chooses it. Otherwise the `QKEY`/`QUSE` exchange is used.
+
+ new A / new B : fast (QADD, confirmation)
+ new A / old B : slow (old B returns QKEY; new A keeps the QKEY/QUSE handling)
+ old A / new B : slow (agreed version below 8; new B returns QKEY)
+ old A / old B : slow
+
+The recipient does not choose by version; it reacts to whichever message arrives
+(`QKEY` or a confirmation on R').
+
+## Dead current server
+
+The initiator completes the switch and stops using the old queue without waiting
+for the old queue to be deleted on its server. Deletion is retried a bounded
+number of times and then abandoned, so an unavailable current server never
+blocks the rotation.
From 849175c59b7df5e2cb20bd52735d0ebab9a3e1eb Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 10 Aug 2026 19:05:29 +0000
Subject: [PATCH 02/16] update plan
---
plans/2026-08-09-fast-queue-rotation-plan.md | 243 ++++++++++++++-----
1 file changed, 188 insertions(+), 55 deletions(-)
diff --git a/plans/2026-08-09-fast-queue-rotation-plan.md b/plans/2026-08-09-fast-queue-rotation-plan.md
index cc57ced51..34a83585d 100644
--- a/plans/2026-08-09-fast-queue-rotation-plan.md
+++ b/plans/2026-08-09-fast-queue-rotation-plan.md
@@ -2,68 +2,201 @@
Branch: ep/drop-agent-versions. RFC: ../rfcs/2026-08-09-fast-queue-rotation.md.
-No new agent message. Each side detects the rotation from the replacement
-reference (`dbReplaceQueueId`) on its own new queue.
+No new agent message. Both peers recognise the rotation from the replacement
+reference (`dbReplaceQueueId`) on their own new queue. The confirmation carries
+one value: `lastOldMsgId` — the last connection message id B committed to the old
+queue before the flip. A uses it to know exactly how much to drain.
+
+Roles: A initiates (its receiving queue moves; A receives on R'); B is the peer
+that sends to A and secures R'.
+
+## Delivery facts this relies on (from the reception loop)
+
+- Per subscribed queue the server sends the next message only as the response to
+ the ACK of the current one (`SMP.ACK _ -> … Right msg@SMP.MSG`). So **not ACKing
+ a message pauses that queue.**
+- `ack = enqueueCmd (ICAck rId srvMsgId)`; the ACK response (next `MSG`, or none)
+ returns to `STResponse (Cmd SRecipient (SMP.ACK _))`, where "none" is currently
+ `pure ()`.
+- The confirmation on an unsecured queue is read via the `(Nothing, Just e2ePubKey)`
+ + `PHEmpty AgentConfirmation | senderCanSecure` branch; it is not in the numbered
+ message sequence.
## New definitions
Agent/Protocol.hs
-- Gate on the existing `rpcAddressSMPAgentVersion` (v8). Renaming it to cover both features can be considered separately; no alias.
-- `SndSwitchStatus` constructor `SSSecuringQueue` with StrEncoding/JSON/field instances.
-- `InternalCommand` constructor `ICQSndSecure SMP.SenderId` with tag.
+- Condition fast rotation on the existing `rpcAddressSMPAgentVersion` (v8). No alias.
+- `SndSwitchStatus` constructor `SSSecuringQueue`.
+- `RcvSwitchStatus` constructor `RSDrainingOld`.
+- `InternalCommand` constructor `ICQSndSecure SMP.SenderId`.
+- Confirmation body for rotation: `lastOldMsgId :: Int64`, placed in `encConnInfo`,
+ protected by the per-queue box, not ratchet-encrypted; read directly by A.
Agent/Store(.AgentStore)
-- `SSSecuringQueue` in `setSndSwitchStatus` mapping.
+- `SSSecuringQueue`, `RSDrainingOld` in the switch-status mappings.
- `ICQSndSecure` in internal command store/parse.
-
-Agent.hs
-- `storeRotationConfirmation` — variant of `storeConfirmation` that stores
- `AgentConfirmation {e2eEncryption_ = Nothing, encConnInfo = ""}` with msgType
- `AM_CONN_INFO`, `sndMsgPrepData_ = Nothing`, no ratchet step.
-
-## Sender B — Agent.hs
-
-`qAddMsg` (new-queue-sender branch)
-- After `addConnSndQueue` for the new snd queue, choose path:
- - fast: `connAgentVersion cData' >= rpcAddressSMPAgentVersion && senderCanSecure (queueMode of new queue)`.
- - else: current `QKEY` path (unchanged).
-- Fast: `enqueueCommand c "" connId (Just newSrv) $ AInternalCommand $ ICQSndSecure sndId`;
- `setSndSwitchStatus db sq $ Just SSSecuringQueue`; notify `SWITCH QDSnd SPStarted`.
-
-`ICQSndSecure sId` (new, in runCommandProcessing, model on `ICQSecure`)
-- Find new snd queue `sq'` in sqs with `dbReplaceQueueId = Just _`, status `New`.
-- `agentSecureSndQueue` (SKEY) on `sq'`; set `Secured`.
-- `storeRotationConfirmation c cData sq'`; `submitPendingMsg c sq'`.
-
-runSmpQueueMsgDelivery — success `AM_CONN_INFO`
-- Branch on `sq` replacement reference:
- - `dbReplaceQueueId = Just _`: rotation. Mirror `AM_QTEST_` — `checkSQSwchStatus sq' SSSecuringQueue`; remove old snd queue; `setSndQueuePrimary`; `deleteConnSndQueue`; notify `SWITCH QDSnd SPCompleted`.
- - otherwise: current join completion (`CON`/`setStatus`).
-
-runSmpQueueMsgDelivery — AUTH `AM_CONN_INFO`
-- Same branch: rotation → `qError msgId "rotation confirmation: AUTH"`; otherwise current `connError NOT_AVAILABLE`.
-
-## Recipient A — Agent.hs
-
-`smpConfirmation`, `New` case, before the RcvConnection/DuplexConnection split
-- If `dbReplaceQueueId rq = Just replacedId` (rq is R'):
- - `setRcvQueueConfirmedE2E db rq (C.dh' e2ePubKey e2ePrivKey) (min v phVer)` (empty encConnInfo not decrypted).
- - `setRcvQueuePrimary`, `setRcvQueueStatus rq Active`.
- - `setRcvSwitchStatus db replaced $ Just RSReceivedMessage`.
- - finalize: update connection, notify `SWITCH QDRcv SPCompleted`, delete old queue via `deleteConnQueues` (bounded best-effort), not `ICQDelete`.
- - no `CON`/`INFO`.
-
-## Switch state
-
-- A: `RSSendingQADD` → (confirmation on R') → `RSReceivedMessage` → done. `RSSendingQUSE` skipped for fast path.
-- B: `SSSecuringQueue` → (confirmation delivered) → done.
-
-## Unchanged
-
-QADD wire format, slow QKEY/QUSE/QTEST path, abort (`canAbortRcvSwitch` still allows through `RSSendingQADD`).
+- New: `moveSndDeliveries db connId fromQ toQ` —
+ `UPDATE snd_message_deliveries SET snd_queue_id = toQ WHERE conn_id = ? AND snd_queue_id = fromQ AND failed = 0`
+ (used both for the flip, old → R', and for rollback, R' → old).
+- New: `getFirstPendingSndMsgId db connId oldQ` —
+ `SELECT MIN(m.internal_snd_id) FROM snd_message_deliveries d JOIN messages m ON m.conn_id = d.conn_id AND m.internal_id = d.internal_id WHERE d.conn_id = ? AND d.snd_queue_id = oldQ AND d.failed = 0`.
+- New: `setRcvSwitchDrain db oldRq srvMsgId lastOldMsgId` and `getRcvSwitchDrain db oldRq`
+ to persist and read the recipient's deferred-ACK id and drain target.
+
+## Schema
+
+Migration adds columns to `rcv_queues`, carried on the old (replaced) queue row, which
+already holds `rcv_switch_status`:
+
+```
+ALTER TABLE rcv_queues ADD COLUMN switch_confirm_srv_msg_id BLOB; -- R' confirmation id to ACK at the flip
+ALTER TABLE rcv_queues ADD COLUMN switch_drain_to_msg_id BIGINT; -- lastOldMsgId: drain old up to this
+```
+
+Both are NULL except while `rcv_switch_status = RSDrainingOld`. R''s `rcvId` (needed for the
+ACK) is the connection's queue whose `dbReplaceQueueId` is the old queue. No new send-side
+column: `ICQSndSecure` recomputes `lastOldMsgId` on each attempt and rolls the move back on
+failure, so its only persisted state is `snd_switch_status = SSSecuringQueue`.
+
+New config: `switchDrainTimeout` — the deadline after which A flips even though old has not
+fully drained (see recipient Step 2).
+
+## Sender B
+
+### Synchronisation
+
+Three tasks touch this connection's send state; two must be excluded during the flip,
+the third is handled by transaction ordering:
+
+1. **User enqueue** — `sendMessagesB_ c reqs connIds = withConnLocks c connIds "sendMessages" $ …`.
+ The `getConn`, the id assignment (`updateSndIds`), and the delivery creation
+ (`createSndMsgDelivery`) all run **inside** the per-connection lock (`connLocks c`, keyed by
+ `connId`).
+2. **Reception** — `processSMP … withConnLock c connId "processSMP"`.
+3. **Delivery worker** — `runSmpQueueMsgDelivery`; for `A_MSG` it does **not** take the
+ connection lock, and it only *removes* an old delivery on a successful send
+ (`delMsgKeep → deleteSndMsgDelivery`), never creates one.
+
+The primitive is the **per-connection lock**. `ICQSndSecure` holds it across the whole flip —
+including the confirmation network send — via `tryWithLock` (`= tryCommand . withConnLock c connId`),
+which is the same thing `ICQSecure` already does across the `secureQueue` network call today. Because
+(1) and (2) take that lock, and (1) re-reads the connection under it, an enqueue cannot interleave
+with the flip: it either fully commits **before** the flip takes the lock — so its message is pending
+on old and is moved to R' — or runs **after** the flip releases it, re-reads the connection, sees R'
+primary, and enqueues to R'. It can never write to old mid-flip.
+
+Holding the lock across the send is specific to the **rotation** confirmation, because only it carries
+`lastOldMsgId` and must exclude concurrent sends to old. It is sent **directly** from `ICQSndSecure`
+(`sendConfirmation`), so ordinary connection-setup confirmations — `secureConfirmQueue` /
+`storeConfirmation` and the delivery worker's `AM_CONN_INFO` path — are unchanged and are **not**
+sent under the connection lock.
+
+Only (3) runs outside the lock. It is excluded not by the lock but by ordering: `lastOldMsgId` is
+read in the **same DB transaction** as the move, so SQLite serialises the worker's row removal
+against it, and the move relabels rows rather than deleting them. So a message the worker sends
+concurrently is either removed before the transaction (counted on old, A drains it) or after (it was
+already relabelled to R', the worker's old-scoped delete matches nothing, A reads it on R'); at worst
+it exists on both old and R' and is de-duplicated by `sndMsgId`. Never lost.
+
+### Steps
+
+`qAddMsg`
+- After `addConnSndQueue` for the new send queue (created `New` by `newSndQueue … Nothing`):
+ - fast: `connAgentVersion cData' >= rpcAddressSMPAgentVersion` (rotation queues are always
+ messaging mode, so `senderCanSecure` is a redundant guard). Then:
+ `enqueueCommand … (Just newSrv) (ICQSndSecure sndId)`; `setSndSwitchStatus SSSecuringQueue`;
+ notify `SWITCH QDSnd SPStarted`.
+ - else: current `QKEY` path, unchanged.
+- The new send queue stays `New`, so `isActiveSndQ` is false and no user message is enqueued to
+ it while securing — user messages go only to old.
+
+`ICQSndSecure sId` (new internal command, retryable). Runs under `tryWithLock` — the connection
+lock is held for the entire body, steps 1–5:
+1. If R' is already primary and old is gone → the flip completed on a prior attempt; return.
+ Otherwise find R' by `dbReplaceQueueId = Just _`, status `New`; `secureSndQueue` (SKEY) on R'
+ (idempotent, retry-safe).
+2. `atomically $ TM.delete (qAddress oldQ) (smpDeliveryWorkers c)` — the old worker starts no new
+ iteration; any in-flight send is handled by step 3's transaction.
+3. **One DB transaction:** `lastOldMsgId = getFirstPendingSndMsgId oldQ − 1` (or the connection's
+ last snd id if nothing is pending); `moveSndDeliveries oldQ → R'` (`failed = 0` only; preserves
+ `internal_id`, so the moved suffix keeps its order and `sndMsgId`). R' is not primary and its
+ worker is not started, so nothing is sent on R' yet.
+4. Send the confirmation to R' carrying `lastOldMsgId` (`sendConfirmation`; direct, not stored, no
+ ratchet step, `e2eEncryption_ = Nothing`). Still under the lock, so no enqueue and no reception
+ for this connection during the send (bounded by the send's network timeout — same as `ICQSecure`).
+5. On success — one transaction: `setSndQueuePrimary R'`; `setSndQueueStatus R' Active`;
+ `deleteConnSndQueue oldQ` (deliveries already moved; B never deletes old on the server). Then
+ `submitPendingMsg c R'` (starts R''s worker, which sends the moved suffix — after the confirmation),
+ notify `SWITCH QDSnd SPCompleted`.
+ On failure — one transaction: `moveSndDeliveries R' → oldQ`; `submitPendingMsg c oldQ` (restart old
+ worker). Temporary error → the command retries from step 1. Permanent error (AUTH) → also
+ `deleteConnSndQueue R'`, leave old as sole primary, surface `A_QUEUE`.
+
+Order of arrival on R': the confirmation (sent directly at step 4, before R''s worker exists), then
+the moved suffix (> `lastOldMsgId`, in `internal_id` order once the worker starts at step 5), then new
+user messages (higher ids, enqueued to R' after the lock is released).
+
+## Recipient A — receive-side state machine
+
+Preconditions: A is subscribed to old (primary) and R' (created at rotation start), at
+`RSSendingQADD`, and tracks `lastExternalSndId` (existing per-connection counter).
+
+`RSSendingQADD` → (confirmation on R') → `RSDrainingOld` → (drained) → complete.
+
+Step 1 — confirmation on R' (`smpConfirmation`, new branch when `dbReplaceQueueId rq = Just replacedId`):
+- Derive R''s secret from the header: `setRcvQueueConfirmedE2E rq (C.dh' e2ePubKey e2ePrivKey) …`.
+- Read `lastOldMsgId` from the confirmation body.
+- Persist the deferred ACK `(rcvId rq, srvMsgId)` and `lastOldMsgId`; set the old queue
+ `rcvSwchStatus = RSDrainingOld`; keep R' status `New`; keep old primary.
+- **Return without ACKing** (like `A_MSG` returns `ACKPending`). The server then holds R''s
+ backlog; A processes no numbered R' message.
+- If `lastExternalSndId >= lastOldMsgId` → go to Step 3 (nothing to drain).
+- Idempotent on redelivery: if already `RSDrainingOld`, re-derive (no-op) and again return
+ without ACKing; re-evaluate the flip condition. **Never ACK a redelivered confirmation
+ until Step 3.**
+
+Step 2 — drain old (best-effort):
+- A keeps receiving old's messages through its existing subscription; each is processed and
+ ACKed normally, advancing `lastExternalSndId`. (No polling; the server pushes.)
+- Extend the ACK path (`ackQueueMessage`/`sendAck` and the `SMP.ACK _ -> … _ -> pure ()`
+ response branch) to report whether the server has more messages for the queue.
+- Perform the flip (Step 3) when any holds:
+ - `lastExternalSndId >= lastOldMsgId`, or
+ - an old-queue ACK response reports no more messages, or
+ - the drain deadline (new config, e.g. a few seconds) passes, or
+ - old is unreachable (subscription failed).
+- Dead old server: nothing is pushed; the deadline fires; A flips, losing
+ `lastExternalSndId+1 … lastOldMsgId` (inherent — on a dead server). When
+ `lastExternalSndId >= lastOldMsgId` already, Step 1 flips immediately, with no delay even
+ if old is unreachable.
+
+Step 3 — flip (finalize, holds the connection lock):
+- `setRcvQueuePrimary R'`; `setRcvQueueStatus R' Active`; remove old from the connection;
+ clear `RSDrainingOld`; notify `SWITCH QDRcv SPCompleted`.
+- Delete old on the server best-effort (bounded `deleteConnQueues`), not blocking.
+- ACK the deferred confirmation: `enqueueCommand … (Just R'server) (ICAck (rcvId R') srvMsgId)`.
+ The server then releases R''s backlog; A processes it and new messages (> `lastOldMsgId`),
+ contiguous after `<= lastOldMsgId`.
+
+## Abort / version
+
+- `canAbortRcvSwitch` returns false for `RSSendingQADD` when `connAgentVersion >= v8`
+ (A cannot see whether B answers with `QKEY` or a confirmation, but at v8 B always chooses
+ fast, so A treats `RSSendingQADD` as committed). Pass `connAgentVersion` to the check.
+- No send-side abort exists; B rolls back internally on permanent failure (sender step 6).
+
+## Losses and duplication
+
+- Old reachable: no loss — A drains `<= lastOldMsgId` from old, then reads `> lastOldMsgId` from R'.
+- Old dead: `lastExternalSndId+1 … lastOldMsgId` stranded on the dead server are lost; inherent.
+- Duplication (old worker race on B; a message present on both old server and R') is
+ de-duplicated by `sndMsgId` (`checkMsgIntegrity` → `MsgDuplicate`).
## Tests
-- new/new: rotation completes with current server stopped after QADD.
-- new/old and old/new: fall back to QKEY path, complete.
-- redundancy: two current queues, rotate one, secret established on R' only.
+- new/new, old server stopped right after QADD, with unsent backlog: rotation completes;
+ backlog delivered on R'; loss only of messages left unfetched on the stopped server.
+- nothing to drain (`lastExternalSndId >= lastOldMsgId`): flip is immediate with old server down.
+- draining: messages pending on old at flip are received before any R' content.
+- new/old and old/new: fall back to the `QKEY`/`QUSE` path.
+- crash mid-drain: reconnect redelivers the confirmation; A re-defers and resumes; completes.
From 0bd32d7e1108956d3730c6e591eb6ff77536feae Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Sat, 15 Aug 2026 16:43:33 +0000
Subject: [PATCH 03/16] update plan and rfc
---
plans/2026-08-09-fast-queue-rotation-plan.md | 320 +++++++++----------
rfcs/2026-08-09-fast-queue-rotation.md | 94 +++---
2 files changed, 201 insertions(+), 213 deletions(-)
diff --git a/plans/2026-08-09-fast-queue-rotation-plan.md b/plans/2026-08-09-fast-queue-rotation-plan.md
index 34a83585d..79111ef8d 100644
--- a/plans/2026-08-09-fast-queue-rotation-plan.md
+++ b/plans/2026-08-09-fast-queue-rotation-plan.md
@@ -2,201 +2,177 @@
Branch: ep/drop-agent-versions. RFC: ../rfcs/2026-08-09-fast-queue-rotation.md.
-No new agent message. Both peers recognise the rotation from the replacement
-reference (`dbReplaceQueueId`) on their own new queue. The confirmation carries
-one value: `lastOldMsgId` — the last connection message id B committed to the old
-queue before the flip. A uses it to know exactly how much to drain.
-
-Roles: A initiates (its receiving queue moves; A receives on R'); B is the peer
-that sends to A and secures R'.
-
-## Delivery facts this relies on (from the reception loop)
-
-- Per subscribed queue the server sends the next message only as the response to
- the ACK of the current one (`SMP.ACK _ -> … Right msg@SMP.MSG`). So **not ACKing
- a message pauses that queue.**
-- `ack = enqueueCmd (ICAck rId srvMsgId)`; the ACK response (next `MSG`, or none)
- returns to `STResponse (Cmd SRecipient (SMP.ACK _))`, where "none" is currently
- `pure ()`.
-- The confirmation on an unsecured queue is read via the `(Nothing, Just e2ePubKey)`
- + `PHEmpty AgentConfirmation | senderCanSecure` branch; it is not in the numbered
- message sequence.
+Model: redundant delivery, no flip. `QADD` adds the new receive queue R'. From `QADD` until R' is
+secured the sender writes every message to both old and R' (double delivery, not a move); once R' is
+secured the sender writes new messages to R' only, while old delivers its already-scheduled tail and
+the `QEND` appended to it. `QEND` removes a named queue. The recipient drops duplicates (double
+ratchet), so the order and which queue delivers do not matter, as long as every message arrives on at
+least one queue. Rotation away from a dead server works because every message up to securing is
+scheduled on R' too.
+
+Roles: A initiates (its receive queue rotates; A receives on R'). B sends to A and secures R'.
+
+## Why redundant delivery removes the hard parts
+
+- No boundary, no drain, no last-message id. A never decides how much of old to read.
+- A dead old server loses nothing: every undelivered message is scheduled on R' as well.
+- A dead new server does not suspend delivery: old keeps delivering until R' is secured.
+- The double ratchet already drops duplicates (`AGENT A_DUPLICATE`) and tolerates bounded reordering,
+ and the delivery schema already writes one message to several send queues (`enqueueMessageB` +
+ `enqueueSavedMessageB`).
+
+## The one ordering constraint
+
+A must hold R''s secret before it reads any R' data message. A data message reaching R' before the
+confirmation is dropped as "no keys" (`processClientMsg`, `(Nothing, Nothing)` arm, line 3611), which
+loses it when old is dead. So the confirmation is the first message B sends on R'. R''s delivery
+worker does not start while R' is securing, so its accumulated rows cannot outrun the confirmation.
+`ICQSndSecure` sends the confirmation and only then starts the worker. This holds on restart too (see
+Worker gate).
## New definitions
Agent/Protocol.hs
-- Condition fast rotation on the existing `rpcAddressSMPAgentVersion` (v8). No alias.
-- `SndSwitchStatus` constructor `SSSecuringQueue`.
-- `RcvSwitchStatus` constructor `RSDrainingOld`.
+- Condition fast rotation on the existing `rpcAddressSMPAgentVersion` (v8, `Protocol.hs:322`).
+- New `AMessage` constructor `QEND SndQAddr` (tag `QE`), the address of the queue to remove. v8-only,
+ and only sent during fast rotation, so peers below v8 never parse it.
+- `SndSwitchStatus` constructors `SSSecuringQueue` (old, while R' secures) and `SSSendingQEND` (old,
+ after R' is secured — it drains its tail and `QEND` but takes no new messages).
- `InternalCommand` constructor `ICQSndSecure SMP.SenderId`.
-- Confirmation body for rotation: `lastOldMsgId :: Int64`, placed in `encConnInfo`,
- protected by the per-queue box, not ratchet-encrypted; read directly by A.
-
-Agent/Store(.AgentStore)
-- `SSSecuringQueue`, `RSDrainingOld` in the switch-status mappings.
-- `ICQSndSecure` in internal command store/parse.
-- New: `moveSndDeliveries db connId fromQ toQ` —
- `UPDATE snd_message_deliveries SET snd_queue_id = toQ WHERE conn_id = ? AND snd_queue_id = fromQ AND failed = 0`
- (used both for the flip, old → R', and for rollback, R' → old).
-- New: `getFirstPendingSndMsgId db connId oldQ` —
- `SELECT MIN(m.internal_snd_id) FROM snd_message_deliveries d JOIN messages m ON m.conn_id = d.conn_id AND m.internal_id = d.internal_id WHERE d.conn_id = ? AND d.snd_queue_id = oldQ AND d.failed = 0`.
-- New: `setRcvSwitchDrain db oldRq srvMsgId lastOldMsgId` and `getRcvSwitchDrain db oldRq`
- to persist and read the recipient's deferred-ACK id and drain target.
+
+No receive-side switch status, no boundary, no drain state.
## Schema
-Migration adds columns to `rcv_queues`, carried on the old (replaced) queue row, which
-already holds `rcv_switch_status`:
+None. `SSSecuringQueue` uses `snd_queues.switch_status`; R' is secured into `rcv_queues.e2e_dh_secret`.
+No new columns, no migration.
+
+## Sender B
-```
-ALTER TABLE rcv_queues ADD COLUMN switch_confirm_srv_msg_id BLOB; -- R' confirmation id to ACK at the flip
-ALTER TABLE rcv_queues ADD COLUMN switch_drain_to_msg_id BIGINT; -- lastOldMsgId: drain old up to this
-```
+### Dual scheduling from QADD
-Both are NULL except while `rcv_switch_status = RSDrainingOld`. R''s `rcvId` (needed for the
-ACK) is the connection's queue whose `dbReplaceQueueId` is the old queue. No new send-side
-column: `ICQSndSecure` recomputes `lastOldMsgId` on each attempt and rolls the move back on
-failure, so its only persisted state is `snd_switch_status = SSSecuringQueue`.
+`enqueueMessageB` writes a delivery row for the head send queue and for each `filter isActiveSndQ`
+tail queue (`Agent.hs:2345`). Adjust the selection two ways: additionally include a securing
+replacement queue on a v8 connection (`connAgentVersion cData >= rpcAddressSMPAgentVersion && status == New && isJust dbReplaceQueueId`),
+and exclude a terminating queue (`sndSwchStatus == Just SSSendingQEND`). The version guard keeps the
+slow path unchanged — there R' is also `New` with a replace reference during `QKEY`/`QUSE`, but it must
+not be dual-scheduled. `SSSendingQEND` is a fast-path-only status, so the exclusion never affects the
+slow path. The gate below is inert for the slow path anyway, since it never starts R''s worker while
+`New`.
-New config: `switchDrainTimeout` — the deadline after which A flips even though old has not
-fully drained (see recipient Step 2).
+- `QADD` until R' secured: old is the head (active) and R' is the securing replacement, so every `SEND`
+ writes both rows. old delivers at once; R''s rows accumulate behind its gate.
+- R' secured: R' is the head (primary) and old is `SSSendingQEND` (excluded), so a `SEND` writes R'
+ only. old keeps its worker and delivers whatever was already scheduled on it, plus `QEND`.
-## Sender B
+### Worker gate
-### Synchronisation
-
-Three tasks touch this connection's send state; two must be excluded during the flip,
-the third is handled by transaction ordering:
-
-1. **User enqueue** — `sendMessagesB_ c reqs connIds = withConnLocks c connIds "sendMessages" $ …`.
- The `getConn`, the id assignment (`updateSndIds`), and the delivery creation
- (`createSndMsgDelivery`) all run **inside** the per-connection lock (`connLocks c`, keyed by
- `connId`).
-2. **Reception** — `processSMP … withConnLock c connId "processSMP"`.
-3. **Delivery worker** — `runSmpQueueMsgDelivery`; for `A_MSG` it does **not** take the
- connection lock, and it only *removes* an old delivery on a successful send
- (`delMsgKeep → deleteSndMsgDelivery`), never creates one.
-
-The primitive is the **per-connection lock**. `ICQSndSecure` holds it across the whole flip —
-including the confirmation network send — via `tryWithLock` (`= tryCommand . withConnLock c connId`),
-which is the same thing `ICQSecure` already does across the `secureQueue` network call today. Because
-(1) and (2) take that lock, and (1) re-reads the connection under it, an enqueue cannot interleave
-with the flip: it either fully commits **before** the flip takes the lock — so its message is pending
-on old and is moved to R' — or runs **after** the flip releases it, re-reads the connection, sees R'
-primary, and enqueues to R'. It can never write to old mid-flip.
-
-Holding the lock across the send is specific to the **rotation** confirmation, because only it carries
-`lastOldMsgId` and must exclude concurrent sends to old. It is sent **directly** from `ICQSndSecure`
-(`sendConfirmation`), so ordinary connection-setup confirmations — `secureConfirmQueue` /
-`storeConfirmation` and the delivery worker's `AM_CONN_INFO` path — are unchanged and are **not**
-sent under the connection lock.
-
-Only (3) runs outside the lock. It is excluded not by the lock but by ordering: `lastOldMsgId` is
-read in the **same DB transaction** as the move, so SQLite serialises the worker's row removal
-against it, and the move relabels rows rather than deleting them. So a message the worker sends
-concurrently is either removed before the transaction (counted on old, A drains it) or after (it was
-already relabelled to R', the worker's old-scoped delete matches nothing, A reads it on R'); at worst
-it exists on both old and R' and is de-duplicated by `sndMsgId`. Never lost.
+`submitPendingMsg` (`Agent.hs:2437`) and `resumeMsgDelivery` (`2421`) — the two `getDeliveryWorker`
+callers that start delivery — skip a queue with `status == New && isJust dbReplaceQueueId`, so neither
+a `SEND` nor startup starts R''s worker while it secures. Startup resumes delivery through
+`resumeMsgDelivery` (`resumeDelivery` line 1848, and `getAllSndQueuesForDelivery` line 1943), so R' is
+skipped there; `resumeAllCommands` (1883) resumes R''s `ICQSndSecure`, which secures R' and only then
+starts its worker.
### Steps
-`qAddMsg`
-- After `addConnSndQueue` for the new send queue (created `New` by `newSndQueue … Nothing`):
- - fast: `connAgentVersion cData' >= rpcAddressSMPAgentVersion` (rotation queues are always
- messaging mode, so `senderCanSecure` is a redundant guard). Then:
- `enqueueCommand … (Just newSrv) (ICQSndSecure sndId)`; `setSndSwitchStatus SSSecuringQueue`;
- notify `SWITCH QDSnd SPStarted`.
- - else: current `QKEY` path, unchanged.
-- The new send queue stays `New`, so `isActiveSndQ` is false and no user message is enqueued to
- it while securing — user messages go only to old.
-
-`ICQSndSecure sId` (new internal command, retryable). Runs under `tryWithLock` — the connection
-lock is held for the entire body, steps 1–5:
-1. If R' is already primary and old is gone → the flip completed on a prior attempt; return.
- Otherwise find R' by `dbReplaceQueueId = Just _`, status `New`; `secureSndQueue` (SKEY) on R'
- (idempotent, retry-safe).
-2. `atomically $ TM.delete (qAddress oldQ) (smpDeliveryWorkers c)` — the old worker starts no new
- iteration; any in-flight send is handled by step 3's transaction.
-3. **One DB transaction:** `lastOldMsgId = getFirstPendingSndMsgId oldQ − 1` (or the connection's
- last snd id if nothing is pending); `moveSndDeliveries oldQ → R'` (`failed = 0` only; preserves
- `internal_id`, so the moved suffix keeps its order and `sndMsgId`). R' is not primary and its
- worker is not started, so nothing is sent on R' yet.
-4. Send the confirmation to R' carrying `lastOldMsgId` (`sendConfirmation`; direct, not stored, no
- ratchet step, `e2eEncryption_ = Nothing`). Still under the lock, so no enqueue and no reception
- for this connection during the send (bounded by the send's network timeout — same as `ICQSecure`).
-5. On success — one transaction: `setSndQueuePrimary R'`; `setSndQueueStatus R' Active`;
- `deleteConnSndQueue oldQ` (deliveries already moved; B never deletes old on the server). Then
- `submitPendingMsg c R'` (starts R''s worker, which sends the moved suffix — after the confirmation),
- notify `SWITCH QDSnd SPCompleted`.
- On failure — one transaction: `moveSndDeliveries R' → oldQ`; `submitPendingMsg c oldQ` (restart old
- worker). Temporary error → the command retries from step 1. Permanent error (AUTH) → also
- `deleteConnSndQueue R'`, leave old as sole primary, surface `A_QUEUE`.
-
-Order of arrival on R': the confirmation (sent directly at step 4, before R''s worker exists), then
-the moved suffix (> `lastOldMsgId`, in `internal_id` order once the worker starts at step 5), then new
-user messages (higher ids, enqueued to R' after the lock is released).
-
-## Recipient A — receive-side state machine
-
-Preconditions: A is subscribed to old (primary) and R' (created at rotation start), at
-`RSSendingQADD`, and tracks `lastExternalSndId` (existing per-connection counter).
-
-`RSSendingQADD` → (confirmation on R') → `RSDrainingOld` → (drained) → complete.
-
-Step 1 — confirmation on R' (`smpConfirmation`, new branch when `dbReplaceQueueId rq = Just replacedId`):
-- Derive R''s secret from the header: `setRcvQueueConfirmedE2E rq (C.dh' e2ePubKey e2ePrivKey) …`.
-- Read `lastOldMsgId` from the confirmation body.
-- Persist the deferred ACK `(rcvId rq, srvMsgId)` and `lastOldMsgId`; set the old queue
- `rcvSwchStatus = RSDrainingOld`; keep R' status `New`; keep old primary.
-- **Return without ACKing** (like `A_MSG` returns `ACKPending`). The server then holds R''s
- backlog; A processes no numbered R' message.
-- If `lastExternalSndId >= lastOldMsgId` → go to Step 3 (nothing to drain).
-- Idempotent on redelivery: if already `RSDrainingOld`, re-derive (no-op) and again return
- without ACKing; re-evaluate the flip condition. **Never ACK a redelivered confirmation
- until Step 3.**
-
-Step 2 — drain old (best-effort):
-- A keeps receiving old's messages through its existing subscription; each is processed and
- ACKed normally, advancing `lastExternalSndId`. (No polling; the server pushes.)
-- Extend the ACK path (`ackQueueMessage`/`sendAck` and the `SMP.ACK _ -> … _ -> pure ()`
- response branch) to report whether the server has more messages for the queue.
-- Perform the flip (Step 3) when any holds:
- - `lastExternalSndId >= lastOldMsgId`, or
- - an old-queue ACK response reports no more messages, or
- - the drain deadline (new config, e.g. a few seconds) passes, or
- - old is unreachable (subscription failed).
-- Dead old server: nothing is pushed; the deadline fires; A flips, losing
- `lastExternalSndId+1 … lastOldMsgId` (inherent — on a dead server). When
- `lastExternalSndId >= lastOldMsgId` already, Step 1 flips immediately, with no delay even
- if old is unreachable.
-
-Step 3 — flip (finalize, holds the connection lock):
-- `setRcvQueuePrimary R'`; `setRcvQueueStatus R' Active`; remove old from the connection;
- clear `RSDrainingOld`; notify `SWITCH QDRcv SPCompleted`.
-- Delete old on the server best-effort (bounded `deleteConnQueues`), not blocking.
-- ACK the deferred confirmation: `enqueueCommand … (Just R'server) (ICAck (rcvId R') srvMsgId)`.
- The server then releases R''s backlog; A processes it and new messages (> `lastOldMsgId`),
- contiguous after `<= lastOldMsgId`.
+`qAddMsg` (fast branch, under the connection lock, `Agent.hs:3855`):
+- Add R' as the slow path does (line 3870): `addConnSndQueue (sq_) {primary = True, dbReplaceQueueId = Just old}`, `New`.
+- Duplicate **every** undelivered message on old to R': for each pending row on old
+ (`SELECT internal_id FROM snd_message_deliveries WHERE conn_id = ? AND snd_queue_id = old AND failed = 0`),
+ `createSndMsgDelivery db R' internalId`. This is the loss-prevention step: old's not-yet-sent
+ messages are duplicated onto R', so if old later fails they are already on R'. If old is already
+ down, nothing was sent and the whole backlog is duplicated.
+- `enqueueCommand (Just newSrv) (ICQSndSecure sndId)`; `setSndSwitchStatus SSSecuringQueue` on old
+ (where the slow path sets `SSSendingQKEY`, line 3874); notify `SWITCH QDSnd SPStarted`. old keeps
+ delivering; R''s worker is gated.
+
+`ICQSndSecure sId` (retryable, under `tryWithLock`):
+1. If old is already gone, a prior attempt finished; return. Otherwise find R' by `sId`.
+2. `secureSndQueue` (SKEY) R' — idempotent, since `sndPrivateKey` was persisted by `qAddMsg`
+ (`QueueStore/STM.hs:213`: same key → `Right ()`, different key → `AUTH`).
+3. Send the confirmation on R' (`sendConfirmation`; empty body, both peers already know each other;
+ `e2eEncryption_ = Nothing`, no ratchet step). It is the first message on R'.
+4. On success, in one transaction: `setSndQueueStatus R' Active` (the gate lifts), `setSndQueuePrimary R'`
+ (R' becomes the head; its replace reference is cleared), and `setSndSwitchStatus old (Just SSSendingQEND)`
+ (old takes no new messages but keeps its worker). Then `submitPendingMsg c R'` (the worker starts and
+ flushes the accumulated rows after the confirmation), `enqueueMessages [old, R'] (QEND oldAddr)`
+ (appended after old's tail, delivered on both), and notify `SWITCH QDSnd SPSecured`. From here a
+ `SEND` goes to R' only; old delivers its tail then `QEND` and is removed when `QEND` is sent.
+
+A temporary error retries from step 1; nothing is torn down. A permanent `AUTH` (should not occur, R'
+was secured with B's own key) leaves both queues and surfaces `A_QUEUE`.
+
+`QEND` sent — new `AM_QEND_` arm in `runSmpQueueMsgDelivery`, modelled on `AM_QTEST_` (`Agent.hs:2567`):
+on a successful send of `QEND addr`, remove the named send queue (`TM.delete` its worker,
+`deleteConnSndQueue addr`), make the remaining queue the sole primary (`setSndQueuePrimary`, which
+clears its `replace_snd_queue_id`), and notify `SWITCH QDSnd SPCompleted` (as `AM_QTEST_` does, line 2591). The handler is idempotent — a
+second `QEND` send finds the named queue already gone and does nothing. `QEND` is sent on both queues;
+removing old's send queue also drops any `QEND` still pending on old. The R' copy reliably removes old
+and reaches A even when old is dead; the old copy is best effort. Once old's send queue is gone, `SEND`
+schedules to R' only.
+
+## Recipient A
+
+A is subscribed to old (primary) and R' (created at rotation start, `dbReplaceQueueId = old`,
+`RSSendingQADD`).
+
+- **Confirmation on R'.** In `processClientMsg`, `(Nothing, Just e2ePubKey)` case, add an arm before the
+ `senderCanSecure` arm (`Agent.hs:3476`), guarded by `isJust (dbReplaceQueueId rq)`. In one
+ transaction: `setRcvQueueConfirmedE2E rq (C.dh' e2ePubKey e2ePrivKey) (min v phVer)` (secures R') and
+ `setRcvQueuePrimary R'` (clears R''s replace reference). Then `ack`, and notify `SWITCH QDRcv SPConfirmed`.
+ No conn-info processing, no ratchet step, no deferral. Redelivery is idempotent: R' now has `e2e_dh_secret`, so a re-sent
+ confirmation reaches the `(Just e2eDh, Just _)` arm (line 3608) and is acked — correct here, since
+ there is no backlog to hold.
+- **Data on R'.** With R''s replace reference cleared, a data message on R' takes the ordinary path
+ (`(_, dbReplaceQueueId=Nothing)`, line 3503) — no old-deletion, no `RSSendingQUSE` check. A copy
+ already read on old is dropped as `A_DUPLICATE`; a copy read first on R' advances the ratchet and
+ old's copy is then the duplicate.
+- **`QEND oldAddr` on either queue.** New `AMessage` handler (`qEndMsg`, a `qDuplex` handler like
+ `qAddMsg`): `findRQ oldAddr` the receive queue to remove. Mark it deleted (`setRcvQueueDeleted`, so
+ `getRcvQueuesByConnId_`'s `deleted = 0` filter excludes it at once and a restart does not resurrect
+ it) and `enqueueCommand (Just oldServer) (ICDeleteRcvQueue oldRcvId)` for the server `DEL` and record
+ removal — the async, crash-safe path `abortConnectionSwitch` uses, which resumes on restart and does
+ not block `QEND`, **not** the synchronous `deleteQueue` of `finalizeSwitch`, which would stall if old
+ is unreachable. `ICDeleteRcvQueue` (`Agent.hs:2224`) currently retries a temporary error forever;
+ bound it with the same persisted `rcv_queues.delete_errors`/`deleteErrorCount` mechanism `deleteQueueRec`
+ uses (2884): on a temporary error `incRcvDeleteErrors`, and at the limit `deleteConnRcvQueue` and
+ stop. The count is in the database, so the bound survives restarts, and its only other caller
+ (`abortConnectionSwitch'`, 2739) deletes an alive queue that succeeds well before the limit. If the removed
+ queue was primary, make the remaining one primary (`setRcvQueuePrimary`); re-create the notification
+ subscription (`when enableNtfs $ sendNtfSubCommand ns (NSCCreate, [connId])`); notify
+ `SWITCH QDRcv SPCompleted`; `ackDel` the `QEND`. Received on both queues, the second finds it already
+ marked deleted and is a no-op.
+
+No drain, no boundary, no finalize command. Old is removed when `QEND` arrives, not by counting.
## Abort / version
-- `canAbortRcvSwitch` returns false for `RSSendingQADD` when `connAgentVersion >= v8`
- (A cannot see whether B answers with `QKEY` or a confirmation, but at v8 B always chooses
- fast, so A treats `RSSendingQADD` as committed). Pass `connAgentVersion` to the check.
-- No send-side abort exists; B rolls back internally on permanent failure (sender step 6).
+- Fast rotation runs only when `connAgentVersion >= v8`; otherwise the `QKEY`/`QUSE` slow path runs
+ unchanged.
+- `canAbortRcvSwitch` (`Agent/Store.hs:210`) returns false for `RSSendingQADD` when `connAgentVersion >= v8`
+ (at v8 B always chooses fast, so A treats a sent `QADD` as committed). Its signature gains
+ `connAgentVersion`; both callers pass it from `cData` — `abortConnectionSwitch'` (2730) and
+ `rcvQueueInfo` in `connectionStats` (2976).
## Losses and duplication
-- Old reachable: no loss — A drains `<= lastOldMsgId` from old, then reads `> lastOldMsgId` from R'.
-- Old dead: `lastExternalSndId+1 … lastOldMsgId` stranded on the dead server are lost; inherent.
-- Duplication (old worker race on B; a message present on both old server and R') is
- de-duplicated by `sndMsgId` (`checkMsgIntegrity` → `MsgDuplicate`).
+- No boundary loss: the `QADD` step **duplicates** old's entire undelivered backlog onto R', and every
+ later message up to securing is scheduled on both queues, so if old fails it loses nothing B still
+ holds. The only messages old can strand are those its server already accepted but had not handed to
+ A — the ordinary store-and-forward risk, present whenever a server fails with unread messages, and
+ empty if old was already down (a down server accepted nothing).
+- Duplicates: the double ratchet drops them (`A_DUPLICATE`); `checkMsgIntegrity`'s `MsgDuplicate` is
+ only a flag, not the mechanism.
+- The 512 skip bound (`Crypto/Ratchet.hs:953`) does not bite: each queue delivers in order and every
+ message is on R', so A reads a contiguous stream with only small cross-queue reordering.
## Tests
-- new/new, old server stopped right after QADD, with unsent backlog: rotation completes;
- backlog delivered on R'; loss only of messages left unfetched on the stopped server.
-- nothing to drain (`lastExternalSndId >= lastOldMsgId`): flip is immediate with old server down.
-- draining: messages pending on old at flip are received before any R' content.
+- new/new, old stopped right after `QADD`: rotation completes; all messages delivered on R'; old
+ removed by `QEND` on R'.
+- new/new, both alive: messages delivered on both, deduped; old removed by `QEND`.
- new/old and old/new: fall back to the `QKEY`/`QUSE` path.
-- crash mid-drain: reconnect redelivers the confirmation; A re-defers and resumes; completes.
+- crash during securing: restart does not start R''s worker; `ICQSndSecure` resumes, secures R',
+ starts the worker, sends `QEND`.
+- `QEND` received on both queues: old removed once, the second receipt is a no-op.
diff --git a/rfcs/2026-08-09-fast-queue-rotation.md b/rfcs/2026-08-09-fast-queue-rotation.md
index b8e87988c..83dcb2904 100644
--- a/rfcs/2026-08-09-fast-queue-rotation.md
+++ b/rfcs/2026-08-09-fast-queue-rotation.md
@@ -4,76 +4,88 @@ Protocol: agent-protocol v8
Diagram: ./diagrams/duplex-messaging/queue-rotation-fast.svg
---
-# Fast queue rotation (SKEY)
+# Fast queue rotation (redundant delivery)
## Problem
-In the current rotation the peer returns `QKEY` to the initiator over the
-initiator's current receiving queue. When that queue's server is unavailable the
-rotation cannot complete, so a client cannot move away from a failed server.
+In the current rotation the peer returns `QKEY` to the initiator over the initiator's current
+receiving queue. When that queue's server is unavailable the rotation cannot complete, so a client
+cannot move away from a failed server.
## Solution
-When both agents support v8, the peer secures the new queue itself with `SKEY`,
-as in the fast connection handshake, and sends an empty confirmation to it
-instead of returning `QKEY`. This replaces the `QKEY` and `QUSE` exchange. The
-initiator's current server is used only to delete the old queue at the end, and
-that deletion does not block completion.
+When both agents support v8, rotation is redundant delivery rather than a switch. The initiator adds
+a new queue with `QADD`; from that point the peer writes every message to both the current queue and
+the new queue, and secures the new queue in parallel. Once the new queue is secured the peer writes
+new messages to it alone, while the current queue delivers whatever was already scheduled on it and a
+final `QEND`, and is then removed. The recipient already drops duplicate messages, so the order of
+delivery and which queue delivers a given message do not matter, provided every message arrives on at
+least one queue. There is no boundary and no flip.
-Roles: A initiates (its receiving queue moves; A receives on R'); B is the peer
-(B holds the sending queue to A, and secures R' and sends to it).
+Rotation away from a dead current server works because every message is also scheduled on the new
+queue: the recipient reads it there. A dead new queue does not stop delivery either, because the
+current queue keeps delivering until the new one is secured.
+
+Roles: A initiates (its receiving queue rotates; A receives on the new queue R'). B is the peer (B
+holds the sending queue to A, secures R', and delivers to both).
Sequence (as drawn in the diagram):
A -> R' : create new queue (messaging mode, SKEY allowed)
A -> S -> B : QADD(R') (over A's sending queue; A's current server untouched)
+ B : from now, schedule every message on both the current queue and R'
+ B -> both : messages (current queue and R', duplicates dropped by A)
B -> R' : SKEY (B secures R')
- B -> R' : confirmation (empty)
- R' -> A : confirmation (A derives R' secret, completes the switch)
- A -> R (old) : delete (best effort, skipped if the server is down)
- B -> R' : messages
+ B -> R' : confirmation (empty; establishes R' secret; first message on R')
+ B : R' secured — new messages now go to R' only
+ B -> current : remaining tail, then QEND(current)
+ B -> R' : QEND(current), then new messages
+ A : on QEND, delete the current queue; keep receiving on R'
## Confirmation
-The confirmation is the only message a recipient can read on a queue that is not
-yet secured, and it is what establishes the queue's shared secret. This is why
-it, rather than an ordinary message, carries the rotation: the earlier abandoned
-attempt sent the securing message as an ordinary message, which required the
-secret it was establishing.
+The confirmation is the only message a recipient can read on a queue that is not yet secured, and it
+establishes the queue's shared secret without depending on the current queue. Because a data message
+that reached R' before the confirmation could not be read, the confirmation is the first message the
+peer sends on R', and the peer does not start ordinary delivery on R' until the confirmation has been
+sent.
+
+The confirmation body is empty: both parties already know each other, so no profile or reply queue is
+sent. It is sealed by the queue's box (keyed by the shared secret being established) and is not
+additionally encrypted with the double ratchet, so rotation does not advance the message ratchet.
-The confirmation body is empty: both parties already know each other, so no
-profile or reply queue is sent. The body is sealed by the queue's NaCl box
-(keyed by the shared secret being established) and is not additionally encrypted
-with the double ratchet, so the rotation does not advance the message ratchet.
-No ratchet key parameters are included, as in the SKEY handshake.
+## Termination
-No new agent message is defined. Both parties already know which queue is being
-replaced — the initiator because it created R' to replace a specific queue, the
-peer because `QADD` states the replaced sending-queue address — so no marker is
-needed on the wire.
+`QEND` names a queue to remove and is delivered on both queues. On receipt the recipient deletes the
+named queue; on send the peer removes its sending queue of that address. `QEND` is a general
+queue-removal message — the peer can remove either queue with it — so rotation is the addition of a
+queue (`QADD`) followed by the removal of a queue (`QEND`), with no distinct switch step. Delivering
+`QEND` on the removed queue is best effort; the copy on the surviving queue removes it and reaches the
+recipient even when the removed server is dead.
## Per-queue secret
-R' secret is a fresh Diffie-Hellman between the peer's queue key, sent in the
-confirmation header, and the initiator's R' key. It does not depend on any
-current queue, so redundancy (several current queues) is unaffected.
+R' secret is a fresh Diffie-Hellman between the peer's queue key, sent in the confirmation header, and
+the initiator's R' key. It does not depend on any current queue, so redundancy of current queues is
+unaffected.
## Compatibility
-Fast rotation runs only when the connection's agreed agent protocol version is 8
-or higher; the peer chooses it. Otherwise the `QKEY`/`QUSE` exchange is used.
+Fast rotation runs only when the connection's agreed agent protocol version is 8 or higher; the peer
+chooses it. Otherwise the `QKEY`/`QUSE` exchange is used. `QEND` is defined at version 8 and is only
+sent during fast rotation, so peers below version 8 never receive it.
- new A / new B : fast (QADD, confirmation)
+ new A / new B : fast (QADD, confirmation, redundant delivery, QEND)
new A / old B : slow (old B returns QKEY; new A keeps the QKEY/QUSE handling)
old A / new B : slow (agreed version below 8; new B returns QKEY)
old A / old B : slow
-The recipient does not choose by version; it reacts to whichever message arrives
-(`QKEY` or a confirmation on R').
+The recipient does not choose by version; it reacts to whichever message arrives — `QKEY`, or a
+confirmation on R' followed later by `QEND`.
## Dead current server
-The initiator completes the switch and stops using the old queue without waiting
-for the old queue to be deleted on its server. Deletion is retried a bounded
-number of times and then abandoned, so an unavailable current server never
-blocks the rotation.
+The initiator keeps reading messages on the new queue and removes the current queue when `QEND`
+arrives there, without waiting for the current server. Nothing is lost, because every message is
+scheduled on the new queue as well; the only cleanup that a dead current server delays is the deletion
+of its queue, which is retried a bounded number of times and then abandoned.
From e750e453b53378cb43d02de014f763b1fdfa4dab Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Sat, 15 Aug 2026 18:20:26 +0000
Subject: [PATCH 04/16] corrections
---
.../duplex-messaging/queue-rotation-fast.mmd | 23 ++++---
.../duplex-messaging/queue-rotation-fast.svg | 2 +-
rfcs/2026-08-09-fast-queue-rotation.md | 66 ++++++++++---------
3 files changed, 51 insertions(+), 40 deletions(-)
diff --git a/protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd b/protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd
index 75887dd0b..952729f4a 100644
--- a/protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd
+++ b/protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd
@@ -5,13 +5,18 @@ sequenceDiagram
participant S as Server that has A's send queue (B's receive queue)
participant B as Bob
- A ->> R': NEW: create new queue (allow SKEY)
- A ->> S: SEND: QADD (R'): send address of the new queue(s)
+ A ->> R': NEW: create new queue (SKEY allowed)
+ A ->> S: SEND: QADD (R')
S ->> B: MSG: QADD (R')
- B ->> R': SKEY: secure new queue
- B ->> R': SEND: QTEST
- R' ->> A: MSG: QTEST
- A ->> R: DEL: delete the old queue
- B ->> R': SEND: send messages to the new queue
- R' ->> A: MSG: receive messages from the new queue
-
\ No newline at end of file
+ B ->> R: SEND: messages (also scheduled on R')
+ R ->> A: MSG: messages
+ B ->> R': SKEY: authorize B as sender
+ B ->> R': SEND: confirmation (establishes R' secret)
+ R' ->> A: MSG: confirmation (A secures R')
+ B ->> R': SEND: held copies (deduped), then new messages
+ R' ->> A: MSG: messages
+ B ->> R: SEND: remaining tail, then QEND
+ R ->> A: MSG: QEND
+ B ->> R': SEND: QEND
+ R' ->> A: MSG: QEND
+ A ->> R: DEL: delete the current queue
diff --git a/protocol/diagrams/duplex-messaging/queue-rotation-fast.svg b/protocol/diagrams/duplex-messaging/queue-rotation-fast.svg
index 823074823..77b986d75 100644
--- a/protocol/diagrams/duplex-messaging/queue-rotation-fast.svg
+++ b/protocol/diagrams/duplex-messaging/queue-rotation-fast.svg
@@ -1,3 +1,3 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/rfcs/2026-08-09-fast-queue-rotation.md b/rfcs/2026-08-09-fast-queue-rotation.md
index 83dcb2904..d5cb3cf55 100644
--- a/rfcs/2026-08-09-fast-queue-rotation.md
+++ b/rfcs/2026-08-09-fast-queue-rotation.md
@@ -1,10 +1,10 @@
---
Proposed: 2026-08-09
Protocol: agent-protocol v8
-Diagram: ./diagrams/duplex-messaging/queue-rotation-fast.svg
+Diagram: ../protocol/diagrams/duplex-messaging/queue-rotation-fast.svg
---
-# Fast queue rotation (redundant delivery)
+# Fast queue rotation
## Problem
@@ -14,33 +14,38 @@ cannot move away from a failed server.
## Solution
-When both agents support v8, rotation is redundant delivery rather than a switch. The initiator adds
-a new queue with `QADD`; from that point the peer writes every message to both the current queue and
-the new queue, and secures the new queue in parallel. Once the new queue is secured the peer writes
-new messages to it alone, while the current queue delivers whatever was already scheduled on it and a
-final `QEND`, and is then removed. The recipient already drops duplicate messages, so the order of
-delivery and which queue delivers a given message do not matter, provided every message arrives on at
-least one queue. There is no boundary and no flip.
-
-Rotation away from a dead current server works because every message is also scheduled on the new
-queue: the recipient reads it there. A dead new queue does not stop delivery either, because the
-current queue keeps delivering until the new one is secured.
+Both the current rotation and v8 add a queue, deliver to both queues while the rotation is in
+progress, and remove the old queue; the recipient drops duplicates in both. The main difference is where
+the new queue's secret is established. In the current rotation it is established over the current
+queue, by `QKEY`, so it cannot complete when the current server is down. In v8 the peer establishes the
+new queue's secret over the new queue itself — a confirmation it sends on R' — so establishing the
+secret no longer depends on the current queue, and the rotation completes even when the current server
+is down.
+
+v8 also starts writing to both queues earlier: from the moment the queue is added, including the
+current queue's not-yet-delivered backlog. So the initiator adds the new queue with `QADD`; from that
+point the peer writes every message to both the current queue and R'. Once R' is secured the peer
+writes new messages to it alone, while the current queue delivers whatever was already scheduled on it
+and a final `QEND`, and is then removed. Because the recipient drops duplicates, neither the order of
+arrival nor which queue carries a message matters, provided each message arrives on at least one queue
+— with one exception, the confirmation, which is always the first message on R'. A dead new queue does
+not stop delivery either, because the current queue keeps delivering until R' is secured.
Roles: A initiates (its receiving queue rotates; A receives on the new queue R'). B is the peer (B
holds the sending queue to A, secures R', and delivers to both).
-Sequence (as drawn in the diagram):
+Sequence:
- A -> R' : create new queue (messaging mode, SKEY allowed)
- A -> S -> B : QADD(R') (over A's sending queue; A's current server untouched)
- B : from now, schedule every message on both the current queue and R'
- B -> both : messages (current queue and R', duplicates dropped by A)
- B -> R' : SKEY (B secures R')
- B -> R' : confirmation (empty; establishes R' secret; first message on R')
- B : R' secured — new messages now go to R' only
- B -> current : remaining tail, then QEND(current)
- B -> R' : QEND(current), then new messages
- A : on QEND, delete the current queue; keep receiving on R'
+ A -> R' : create new queue (messaging mode, SKEY allowed)
+ A -> S -> B : QADD(R') (over A's sending queue; A's current server untouched)
+ B : from QADD, schedule every message and the current backlog on both queues
+ B -> current : deliver the scheduled messages (R' holds its copies while securing)
+ B -> R' : SKEY (authorize B as sender)
+ B -> R' : confirmation (empty; establishes R' secret; first message on R')
+ B -> R' : deliver R''s held copies (A dedups), then new messages to R' only
+ B -> current : deliver the remaining tail, then QEND(current)
+ B -> R' : QEND(current)
+ A : on QEND, delete the current queue; keep receiving on R'
## Confirmation
@@ -58,10 +63,11 @@ additionally encrypted with the double ratchet, so rotation does not advance the
`QEND` names a queue to remove and is delivered on both queues. On receipt the recipient deletes the
named queue; on send the peer removes its sending queue of that address. `QEND` is a general
-queue-removal message — the peer can remove either queue with it — so rotation is the addition of a
-queue (`QADD`) followed by the removal of a queue (`QEND`), with no distinct switch step. Delivering
-`QEND` on the removed queue is best effort; the copy on the surviving queue removes it and reaches the
-recipient even when the removed server is dead.
+queue-removal message — the peer can remove either queue with it — so on the wire a rotation is the
+addition of a queue (`QADD`) and the later removal of the replaced one (`QEND`), each an ordinary
+operation on the queue set rather than a `QTEST`-style completion. Delivering `QEND` on the removed
+queue is best effort; the copy on the surviving queue removes it and reaches the recipient even when
+the removed server is dead.
## Per-queue secret
@@ -75,7 +81,7 @@ Fast rotation runs only when the connection's agreed agent protocol version is 8
chooses it. Otherwise the `QKEY`/`QUSE` exchange is used. `QEND` is defined at version 8 and is only
sent during fast rotation, so peers below version 8 never receive it.
- new A / new B : fast (QADD, confirmation, redundant delivery, QEND)
+ new A / new B : fast (QADD, confirmation on R', QEND)
new A / old B : slow (old B returns QKEY; new A keeps the QKEY/QUSE handling)
old A / new B : slow (agreed version below 8; new B returns QKEY)
old A / old B : slow
@@ -87,5 +93,5 @@ confirmation on R' followed later by `QEND`.
The initiator keeps reading messages on the new queue and removes the current queue when `QEND`
arrives there, without waiting for the current server. Nothing is lost, because every message is
-scheduled on the new queue as well; the only cleanup that a dead current server delays is the deletion
+scheduled on the new queue; the only cleanup that a dead current server delays is the deletion
of its queue, which is retried a bounded number of times and then abandoned.
From 407ee4a5ead16641baed481e1baa12982caa93db Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Sun, 16 Aug 2026 10:50:19 +0000
Subject: [PATCH 05/16] update plan
---
plans/2026-08-09-fast-queue-rotation-plan.md | 31 +++++++++++++-------
1 file changed, 20 insertions(+), 11 deletions(-)
diff --git a/plans/2026-08-09-fast-queue-rotation-plan.md b/plans/2026-08-09-fast-queue-rotation-plan.md
index 79111ef8d..2af69ba5a 100644
--- a/plans/2026-08-09-fast-queue-rotation-plan.md
+++ b/plans/2026-08-09-fast-queue-rotation-plan.md
@@ -15,7 +15,7 @@ Roles: A initiates (its receive queue rotates; A receives on R'). B sends to A a
## Why redundant delivery removes the hard parts
- No boundary, no drain, no last-message id. A never decides how much of old to read.
-- A dead old server loses nothing: every undelivered message is scheduled on R' as well.
+- A dead old server loses nothing B still holds: every undelivered message is scheduled on R' as well.
- A dead new server does not suspend delivery: old keeps delivering until R' is secured.
- The double ratchet already drops duplicates (`AGENT A_DUPLICATE`) and tolerates bounded reordering,
and the delivery schema already writes one message to several send queues (`enqueueMessageB` +
@@ -106,16 +106,18 @@ was secured with B's own key) leaves both queues and surfaces `A_QUEUE`.
`QEND` sent — new `AM_QEND_` arm in `runSmpQueueMsgDelivery`, modelled on `AM_QTEST_` (`Agent.hs:2567`):
on a successful send of `QEND addr`, remove the named send queue (`TM.delete` its worker,
`deleteConnSndQueue addr`), make the remaining queue the sole primary (`setSndQueuePrimary`, which
-clears its `replace_snd_queue_id`), and notify `SWITCH QDSnd SPCompleted` (as `AM_QTEST_` does, line 2591). The handler is idempotent — a
-second `QEND` send finds the named queue already gone and does nothing. `QEND` is sent on both queues;
-removing old's send queue also drops any `QEND` still pending on old. The R' copy reliably removes old
+clears its `replace_snd_queue_id`), and notify `SWITCH QDSnd SPCompleted` (as `AM_QTEST_` does, line
+2591). This re-primary is a no-op on the send side — step 4 already made R' primary before `QEND` was
+enqueued. The handler is idempotent — a second `QEND` send finds the named queue already gone and does
+nothing. `QEND` is sent on both queues; removing old's send queue also drops any `QEND` still pending
+on old. The R' copy reliably removes old
and reaches A even when old is dead; the old copy is best effort. Once old's send queue is gone, `SEND`
schedules to R' only.
## Recipient A
-A is subscribed to old (primary) and R' (created at rotation start, `dbReplaceQueueId = old`,
-`RSSendingQADD`).
+A is subscribed to old (primary, `RSSendingQADD`) and R' (created at rotation start,
+`dbReplaceQueueId = old`).
- **Confirmation on R'.** In `processClientMsg`, `(Nothing, Just e2ePubKey)` case, add an arm before the
`senderCanSecure` arm (`Agent.hs:3476`), guarded by `isJust (dbReplaceQueueId rq)`. In one
@@ -138,9 +140,13 @@ A is subscribed to old (primary) and R' (created at rotation start, `dbReplaceQu
bound it with the same persisted `rcv_queues.delete_errors`/`deleteErrorCount` mechanism `deleteQueueRec`
uses (2884): on a temporary error `incRcvDeleteErrors`, and at the limit `deleteConnRcvQueue` and
stop. The count is in the database, so the bound survives restarts, and its only other caller
- (`abortConnectionSwitch'`, 2739) deletes an alive queue that succeeds well before the limit. If the removed
- queue was primary, make the remaining one primary (`setRcvQueuePrimary`); re-create the notification
- subscription (`when enableNtfs $ sendNtfSubCommand ns (NSCCreate, [connId])`); notify
+ (`abortConnectionSwitch'`, 2739) deletes an alive queue that succeeds well before the limit. `qEndMsg`
+ does **not** re-primary R' — the confirmation arm (above) owns R''s primary flag and replace
+ reference. `QEND` on old and the confirmation on R' travel on different queues with no order between
+ them, so `QEND` on old can be processed first (it sits only behind old's tail); re-primarying then
+ would clear R''s `dbReplaceQueueId` and the later confirmation would miss the rotation arm and never
+ secure R'. So `qEndMsg` only removes the named queue. Re-create the notification subscription
+ (`when enableNtfs $ sendNtfSubCommand ns (NSCCreate, [connId])`); notify
`SWITCH QDRcv SPCompleted`; `ackDel` the `QEND`. Received on both queues, the second finds it already
marked deleted and is a no-op.
@@ -164,8 +170,9 @@ No drain, no boundary, no finalize command. Old is removed when `QEND` arrives,
empty if old was already down (a down server accepted nothing).
- Duplicates: the double ratchet drops them (`A_DUPLICATE`); `checkMsgIntegrity`'s `MsgDuplicate` is
only a flag, not the mechanism.
-- The 512 skip bound (`Crypto/Ratchet.hs:953`) does not bite: each queue delivers in order and every
- message is on R', so A reads a contiguous stream with only small cross-queue reordering.
+- The 512 skip bound (`Crypto/Ratchet.hs:953`) does not bite on the rotation: each queue delivers in
+ order and every message B still holds is on R', so A reads a contiguous stream with only small
+ cross-queue reordering. A store-and-forward residual (above) is an ordinary loss, not introduced here.
## Tests
@@ -176,3 +183,5 @@ No drain, no boundary, no finalize command. Old is removed when `QEND` arrives,
- crash during securing: restart does not start R''s worker; `ICQSndSecure` resumes, secures R',
starts the worker, sends `QEND`.
- `QEND` received on both queues: old removed once, the second receipt is a no-op.
+- `QEND` on old processed before the confirmation on R': R' still secures, because `qEndMsg` does not
+ clear R''s replace reference; rotation completes.
From 4e060c084de238284213971edc3c1dce06fb6823 Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Sun, 16 Aug 2026 16:53:20 +0000
Subject: [PATCH 06/16] implementation
---
src/Simplex/Messaging/Agent.hs | 143 +++++++++++++++---
src/Simplex/Messaging/Agent/Protocol.hs | 17 +++
src/Simplex/Messaging/Agent/Store.hs | 14 +-
.../Messaging/Agent/Store/AgentStore.hs | 14 ++
tests/AgentTests/FunctionalAPITests.hs | 54 +++++--
tests/AgentTests/NotificationTests.hs | 5 +-
6 files changed, 205 insertions(+), 42 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index 371e1fac7..2984dc6a7 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2224,8 +2224,14 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
ICDeleteConn -> withStore' c (`deleteCommand` cmdId)
ICDeleteRcvQueue rId -> withServer $ \srv -> tryWithLock "ICDeleteRcvQueue" $ do
rq <- withStore c (\db -> getDeletedRcvQueue db connId srv rId)
- deleteQueue c NRMBackground rq
- withStore' c (`deleteConnRcvQueue` rq)
+ maxErrs <- asks $ deleteErrorCount . config
+ tryAllErrors (deleteQueue c NRMBackground rq) >>= \case
+ Right () -> withStore' c (`deleteConnRcvQueue` rq)
+ Left e
+ | temporaryOrHostError e && deleteErrors rq + 1 < maxErrs -> do
+ withStore' c (`incRcvDeleteErrors` rq)
+ throwE e
+ | otherwise -> withStore' c (`deleteConnRcvQueue` rq)
ICQSecure rId senderKey ->
withServer $ \srv -> tryWithLock "ICQSecure" . withDuplexConn $ \(DuplexConnection cData rqs sqs) ->
case find (sameQueue (srv, rId)) rqs of
@@ -2268,10 +2274,33 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
cStats <- connectionStats c conn'
notify $ SWITCH QDRcv SPCompleted cStats
_ -> internalErr "ICQDelete: cannot delete the only queue in connection"
+ ICQSndSecure sId ->
+ withServer $ \srv -> tryWithLock "ICQSndSecure" . withDuplexConn $ \(DuplexConnection cData@ConnData {connAgentVersion} _rqs sqs) ->
+ case findQ (srv, sId) sqs of
+ Nothing -> internalErr "ICQSndSecure: queue address not found in connection"
+ Just sq'@SndQueue {dbReplaceQueueId} ->
+ case dbReplaceQueueId >>= \replaceQId -> find ((replaceQId ==) . dbQId) sqs of
+ Nothing -> pure ()
+ Just oldSq@SndQueue {server = oldServer, sndId = oldSndId} -> do
+ secureSndQueue c NRMBackground sq'
+ let confMsg = smpEncode $ AgentConfirmation {agentVersion = connAgentVersion, e2eEncryption_ = Nothing, encConnInfo = ""}
+ void $ sendConfirmation c NRMBackground sq' confMsg
+ withStore' c $ \db -> do
+ setSndQueueStatus db sq' Active
+ setSndQueuePrimary db connId sq'
+ void $ setSndSwitchStatus db oldSq $ Just SSSendingQEND
+ let sq'' = (sq' :: SndQueue) {status = Active, primary = True, dbReplaceQueueId = Nothing}
+ lift $ submitPendingMsg c sq''
+ void $ enqueueMessages c cData (oldSq :| [sq'']) SMP.noMsgFlags $ QEND (oldServer, oldSndId)
+ SomeConn _ conn' <- withStore c (`getConn` connId)
+ cStats <- connectionStats c conn'
+ notify $ SWITCH QDSnd SPSecured cStats
where
- ack srv rId srvMsgId = do
- rq <- withStore c $ \db -> getRcvQueue db connId srv rId
- ackQueueMessage c rq srvMsgId
+ ack srv rId srvMsgId =
+ withStore' c (\db -> getRcvQueue db connId srv rId) >>= \case
+ Right rq -> ackQueueMessage c rq srvMsgId
+ -- the queue was removed (e.g. old queue after QEND); nothing to ack
+ Left _ -> pure Nothing
secure :: RcvQueue -> SMP.SndPublicAuthKey -> AM ()
secure rq@RcvQueue {server} senderKey = do
secureQueue c NRMBackground rq senderKey
@@ -2341,11 +2370,15 @@ enqueueMessageB c reqs = do
cfg <- asks config
(_, reqMids) <- unsafeWithStore c $ \db -> do
mapAccumLM (\ids r -> storeSentMsg db cfg ids r `E.catchAny` \e -> (ids,) <$> handleInternal e) IM.empty reqs
- forME reqMids $ \((csqs_, _, _, _), InternalId msgId, pqSecr) -> forM csqs_ $ \(_, sq :| sqs) -> do
+ forME reqMids $ \((csqs_, _, _, _), InternalId msgId, pqSecr) -> forM csqs_ $ \(cData, sq :| sqs) -> do
submitPendingMsg c sq
- let sqs' = filter isActiveSndQ sqs
+ let sqs' = filter (sndDeliverTo cData) sqs
pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId))
where
+ sndDeliverTo :: ConnData -> SndQueue -> Bool
+ sndDeliverTo ConnData {connAgentVersion} sq@SndQueue {status, sndSwchStatus} =
+ sndSwchStatus /= Just SSSendingQEND
+ && (isActiveSndQ sq || (connAgentVersion >= rpcAddressSMPAgentVersion && status == New && isJust (dbReplaceQId sq)))
storeSentMsg ::
DB.Connection ->
AgentConfig ->
@@ -2423,9 +2456,16 @@ resumeMsgDelivery :: AgentClient -> SndQueue -> AM' ()
-- hasWork is passed as False to avoid unnecessary write to TMVar:
-- - new worker is always created by "some work to do".
-- - if the worker already exists, there is no need to "push" it again.
-resumeMsgDelivery = void .: getDeliveryWorker False
+resumeMsgDelivery c sq
+ | securingSndQueue sq = pure ()
+ | otherwise = void $ getDeliveryWorker False c sq
{-# INLINE resumeMsgDelivery #-}
+-- a replacement queue holds its worker until it is secured and its confirmation is sent
+securingSndQueue :: SndQueue -> Bool
+securingSndQueue SndQueue {status, dbReplaceQueueId} = status == New && isJust dbReplaceQueueId
+{-# INLINE securingSndQueue #-}
+
getDeliveryWorker :: Bool -> AgentClient -> SndQueue -> AM' (Worker, TMVar ())
getDeliveryWorker hasWork c sq =
getAgentWorker' fst mkLock "msg_delivery" hasWork c (qAddress sq) (smpDeliveryWorkers c) (runSmpQueueMsgDelivery c sq)
@@ -2435,9 +2475,11 @@ getDeliveryWorker hasWork c sq =
pure (w, retryLock)
submitPendingMsg :: AgentClient -> SndQueue -> AM' ()
-submitPendingMsg c sq = do
- atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + 1}
- void $ getDeliveryWorker True c sq
+submitPendingMsg c sq
+ | securingSndQueue sq = pure ()
+ | otherwise = do
+ atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + 1}
+ void $ getDeliveryWorker True c sq
runSmpQueueMsgDelivery :: AgentClient -> SndQueue -> (Worker, TMVar ()) -> AM ()
runSmpQueueMsgDelivery c@AgentClient {subQ} sq@SndQueue {userId, connId, server, queueMode} (Worker {doWork}, qLock) = do
@@ -2506,6 +2548,8 @@ runSmpQueueMsgDelivery c@AgentClient {subQ} sq@SndQueue {userId, connId, server,
AM_QKEY_ -> qError msgId "QKEY: AUTH"
AM_QUSE_ -> qError msgId "QUSE: AUTH"
AM_QTEST_ -> qError msgId "QTEST: AUTH"
+ -- QEND removal is best effort and redundant; drop a copy that cannot be sent
+ AM_QEND_ -> delMsg msgId
AM_EREADY_ -> notifyDel msgId err
AM_SRV_REQ -> logError "AM_SRV_REQ: unexpected stored message" >> delMsg msgId
AM_SRV_RESP -> notifyDel msgId err
@@ -2593,6 +2637,23 @@ runSmpQueueMsgDelivery c@AgentClient {subQ} sq@SndQueue {userId, connId, server,
_ -> internalErr msgId "sent QTEST: there is only one queue in connection"
_ -> internalErr msgId "sent QTEST: queue not in connection or not replacing another queue"
_ -> internalErr msgId "QTEST sent not in duplex connection"
+ AM_QEND_ -> withConnLockNotify c connId "runSmpQueueMsgDelivery AM_QEND_" $ do
+ SomeConn _ conn <- withStore c (`getConn` connId)
+ case conn of
+ DuplexConnection cData' rqs sqs ->
+ case removeQP (\sq' -> sndSwchStatus sq' == Just SSSendingQEND) sqs of
+ Nothing -> pure Nothing
+ Just (oldSq, sq'' : sqs') -> do
+ atomically $ TM.delete (qAddress oldSq) $ smpDeliveryWorkers c
+ withStore' c $ \db -> do
+ setSndQueuePrimary db connId sq''
+ deletePendingMsgs db connId oldSq
+ deleteConnSndQueue db connId oldSq
+ let conn' = DuplexConnection cData' rqs (sq'' :| sqs')
+ cStats <- connectionStats c conn'
+ pure $ Just ("", connId, AEvt SAEConn $ SWITCH QDSnd SPCompleted cStats)
+ Just (_, []) -> internalErr msgId "sent QEND: no remaining queue in connection"
+ _ -> internalErr msgId "QEND sent not in duplex connection"
AM_EREADY_ -> pure ()
AM_SRV_REQ -> logError "AM_SRV_REQ: unexpected stored message"
AM_SRV_RESP -> notify $ SSENT mId proxySrv_
@@ -2726,9 +2787,9 @@ abortConnectionSwitch' :: AgentClient -> ConnId -> AM ConnectionStats
abortConnectionSwitch' c connId =
withConnLock c connId "abortConnectionSwitch" $
withStore c (`getConn` connId) >>= \case
- SomeConn _ (DuplexConnection cData rqs sqs) -> case switchingRQ rqs of
+ SomeConn _ (DuplexConnection cData@ConnData {connAgentVersion} rqs sqs) -> case switchingRQ rqs of
Just rq
- | canAbortRcvSwitch rq -> do
+ | canAbortRcvSwitch connAgentVersion rq -> do
when (ratchetSyncSendProhibited cData) $ throwE $ CMD PROHIBITED "abortConnectionSwitch: send prohibited"
-- multiple queues to which the connections switches were possible when repeating switch was allowed
let (delRqs, keepRqs) = L.partition ((Just (dbQId rq) ==) . dbReplaceQId) rqs
@@ -2943,12 +3004,12 @@ getConnectionRatchetAdHash' c connId = do
connectionStats :: AgentClient -> Connection c -> AM ConnectionStats
connectionStats c = \case
RcvConnection cData rq -> do
- rcvQueuesInfo <- (: []) <$> rcvQueueInfo rq
+ rcvQueuesInfo <- (: []) <$> rcvQueueInfo cData rq
pure (stats cData) {rcvQueuesInfo, subStatus = connSubStatus rcvQueuesInfo}
SndConnection cData sq -> do
pure (stats cData) {sndQueuesInfo = [sndQueueInfo sq]}
DuplexConnection cData rqs sqs -> do
- rcvQueuesInfo <- mapM rcvQueueInfo (L.toList rqs)
+ rcvQueuesInfo <- mapM (rcvQueueInfo cData) (L.toList rqs)
pure
(stats cData)
{ rcvQueuesInfo,
@@ -2956,7 +3017,7 @@ connectionStats c = \case
subStatus = connSubStatus rcvQueuesInfo
}
ContactConnection cData rq -> do
- rcvQueuesInfo <- (: []) <$> rcvQueueInfo rq
+ rcvQueuesInfo <- (: []) <$> rcvQueueInfo cData rq
pure (stats cData) {rcvQueuesInfo, subStatus = connSubStatus rcvQueuesInfo}
NewConnection cData ->
pure $ stats cData
@@ -2971,10 +3032,10 @@ connectionStats c = \case
ratchetSyncSupported = True,
subStatus = Nothing
}
- rcvQueueInfo :: RcvQueue -> AM RcvQueueInfo
- rcvQueueInfo rq@RcvQueue {server, status, rcvSwchStatus} = do
+ rcvQueueInfo :: ConnData -> RcvQueue -> AM RcvQueueInfo
+ rcvQueueInfo ConnData {connAgentVersion} rq@RcvQueue {server, status, rcvSwchStatus} = do
subStatus <- atomically checkQueueSubStatus
- pure $ RcvQueueInfo {rcvServer = server, status, rcvSwitchStatus = rcvSwchStatus, canAbortSwitch = canAbortRcvSwitch rq, subStatus}
+ pure $ RcvQueueInfo {rcvServer = server, status, rcvSwitchStatus = rcvSwchStatus, canAbortSwitch = canAbortRcvSwitch connAgentVersion rq, subStatus}
where
checkQueueSubStatus :: STM SubscriptionStatus
checkQueueSubStatus =
@@ -3475,6 +3536,15 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
(SMP.PHConfirmation senderKey, AgentConfirmation {e2eEncryption_, encConnInfo, agentVersion}) ->
smpConfirmation srvMsgId conn (Just senderKey) e2ePubKey e2eEncryption_ encConnInfo phVer agentVersion >> ack
(SMP.PHEmpty, AgentConfirmation {e2eEncryption_, encConnInfo, agentVersion})
+ | isJust (dbReplaceQId rq) -> do
+ logServer "<--" c srv rId $ "MSG :" <> logSecret' srvMsgId
+ withStore' c $ \db -> do
+ setRcvQueueConfirmedE2E db rq e2eDh $ min agreedClientVerion phVer
+ setRcvQueuePrimary db connId rq
+ SomeConn _ conn' <- withStore c (`getConn` connId)
+ cStats <- connectionStats c conn'
+ notify $ SWITCH QDRcv SPConfirmed cStats
+ ack
| senderCanSecure queueMode -> smpConfirmation srvMsgId conn Nothing e2ePubKey e2eEncryption_ encConnInfo phVer agentVersion >> ack
| otherwise -> prohibited "handshake: missing sender key" >> ack
(SMP.PHEmpty, AgentInvitation {connReq, connInfo}) ->
@@ -3522,6 +3592,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
-- no action needed for QTEST
-- any message in the new queue will mark it active and trigger deletion of the old queue
QTEST _ -> logServer "<--" c srv rId ("MSG :" <> logSecret' srvMsgId) >> ackDel msgId
+ QEND addr -> qDuplexAckDel conn'' "QEND" $ qEndMsg srvMsgId addr
EREADY _ -> qDuplexAckDel conn'' "EREADY" $ ereadyMsg rcPrev
where
qDuplexAckDel :: Connection c -> String -> (Connection 'CDuplex -> AM ()) -> AM ACKd
@@ -3853,7 +3924,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
-- processed by queue sender
qAddMsg :: SMP.MsgId -> NonEmpty (SMPQueueUri, Maybe SndQAddr) -> Connection 'CDuplex -> AM ()
qAddMsg _ ((_, Nothing) :| _) _ = qError "adding queue without switching is not supported"
- qAddMsg srvMsgId ((qUri, Just addr) :| _) (DuplexConnection cData' rqs sqs) = do
+ qAddMsg srvMsgId ((qUri, Just addr) :| _) (DuplexConnection cData'@ConnData {connAgentVersion} rqs sqs) = do
when (ratchetSyncSendProhibited cData') $ throwE $ AGENT (A_QUEUE "ratchet is not synchronized")
clientVRange <- asks $ smpClientVRange . config
case qUri `compatibleVersion` clientVRange of
@@ -3870,9 +3941,19 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
liftIO $ mapM_ (deleteConnSndQueue db connId) delSqs
addConnSndQueue db connId (sq_ :: NewSndQueue) {primary = True, dbReplaceQueueId = Just dbQueueId}
logServer "<--" c srv rId $ "MSG :" <> logSecret' srvMsgId <> " " <> logSecret (senderId queueAddress)
- let sqInfo' = (sqInfo :: SMPQueueInfo) {queueAddress = queueAddress {dhPublicKey}}
- void . enqueueMessages c cData' sqs SMP.noMsgFlags $ QKEY [(sqInfo', C.toPublic sndPrivateKey)]
- sq1 <- withStore' c $ \db -> setSndSwitchStatus db sq $ Just SSSendingQKEY
+ swchStatus <-
+ if connAgentVersion >= rpcAddressSMPAgentVersion
+ then do
+ withStore' c $ \db -> do
+ pending <- getPendingSndDeliveries db connId sq
+ mapM_ (createSndMsgDelivery db sq2) pending
+ enqueueCommand c "" connId (Just $ qServer sq2) $ AInternalCommand $ ICQSndSecure (snd $ qAddress sq2)
+ pure SSSecuringQueue
+ else do
+ let sqInfo' = (sqInfo :: SMPQueueInfo) {queueAddress = queueAddress {dhPublicKey}}
+ void . enqueueMessages c cData' sqs SMP.noMsgFlags $ QKEY [(sqInfo', C.toPublic sndPrivateKey)]
+ pure SSSendingQKEY
+ sq1 <- withStore' c $ \db -> setSndSwitchStatus db sq $ Just swchStatus
let sqs'' = updatedQs sq1 sqs' <> [sq2]
conn' = DuplexConnection cData' rqs sqs''
cStats <- connectionStats c conn'
@@ -3926,6 +4007,22 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
_ -> qError "QUSE: switching SndQueue not found in connection"
_ -> qError "QUSE: switched queue address not found in connection"
+ -- processed by queue recipient: remove the queue named by QEND
+ qEndMsg :: SMP.MsgId -> SndQAddr -> Connection 'CDuplex -> AM ()
+ qEndMsg srvMsgId addr (DuplexConnection cData'@ConnData {enableNtfs} rqs sqs) =
+ case removeQP (\rq' -> sameQAddress addr (sndAddress rq')) rqs of
+ Just (rq'@RcvQueue {server = rmServer, rcvId}, rq'' : rqs') -> do
+ logServer "<--" c srv rId $ "MSG :" <> logSecret' srvMsgId <> " " <> logSecret (snd addr)
+ withStore' c $ \db -> setRcvQueueDeleted db rq'
+ enqueueCommand c "" connId (Just rmServer) $ AInternalCommand $ ICDeleteRcvQueue rcvId
+ when enableNtfs $ do
+ ns <- asks ntfSupervisor
+ liftIO $ sendNtfSubCommand ns (NSCCreate, [connId])
+ let conn' = DuplexConnection cData' (rq'' :| rqs') sqs
+ cStats <- connectionStats c conn'
+ notify $ SWITCH QDRcv SPCompleted cStats
+ _ -> pure ()
+
qError :: String -> AM a
qError = throwE . AGENT . A_QUEUE
diff --git a/src/Simplex/Messaging/Agent/Protocol.hs b/src/Simplex/Messaging/Agent/Protocol.hs
index cfdb94a6b..4d205608a 100644
--- a/src/Simplex/Messaging/Agent/Protocol.hs
+++ b/src/Simplex/Messaging/Agent/Protocol.hs
@@ -639,16 +639,22 @@ instance FromJSON RcvSwitchStatus where
data SndSwitchStatus
= SSSendingQKEY
| SSSendingQTEST
+ | SSSecuringQueue
+ | SSSendingQEND
deriving (Eq, Show)
instance StrEncoding SndSwitchStatus where
strEncode = \case
SSSendingQKEY -> "sending_qkey"
SSSendingQTEST -> "sending_qtest"
+ SSSecuringQueue -> "securing_queue"
+ SSSendingQEND -> "sending_qend"
strP =
A.takeTill (== ' ') >>= \case
"sending_qkey" -> pure SSSendingQKEY
"sending_qtest" -> pure SSSendingQTEST
+ "securing_queue" -> pure SSSecuringQueue
+ "sending_qend" -> pure SSSendingQEND
_ -> fail "bad SndSwitchStatus"
instance ToField SndSwitchStatus where toField = toField . decodeLatin1 . strEncode
@@ -960,6 +966,7 @@ data AgentMessageType
| AM_QKEY_
| AM_QUSE_
| AM_QTEST_
+ | AM_QEND_
| AM_EREADY_
| AM_SRV_REQ
| AM_SRV_RESP
@@ -979,6 +986,7 @@ instance Encoding AgentMessageType where
AM_QKEY_ -> "QK"
AM_QUSE_ -> "QU"
AM_QTEST_ -> "QT"
+ AM_QEND_ -> "QE"
AM_EREADY_ -> "E"
AM_SRV_REQ -> "A"
AM_SRV_RESP -> "P"
@@ -998,6 +1006,7 @@ instance Encoding AgentMessageType where
'K' -> pure AM_QKEY_
'U' -> pure AM_QUSE_
'T' -> pure AM_QTEST_
+ 'E' -> pure AM_QEND_
_ -> fail "bad AgentMessageType"
'E' -> pure AM_EREADY_
'A' -> pure AM_SRV_REQ
@@ -1037,6 +1046,7 @@ data AMsgType
| QKEY_
| QUSE_
| QTEST_
+ | QEND_
| EREADY_
deriving (Eq)
@@ -1050,6 +1060,7 @@ instance Encoding AMsgType where
QKEY_ -> "QK"
QUSE_ -> "QU"
QTEST_ -> "QT"
+ QEND_ -> "QE"
EREADY_ -> "E"
smpP =
A.anyChar >>= \case
@@ -1063,6 +1074,7 @@ instance Encoding AMsgType where
'K' -> pure QKEY_
'U' -> pure QUSE_
'T' -> pure QTEST_
+ 'E' -> pure QEND_
_ -> fail "bad AMsgType"
'E' -> pure EREADY_
_ -> fail "bad AMsgType"
@@ -1087,6 +1099,8 @@ data AMessage
QUSE (NonEmpty (SndQAddr, Bool))
| -- sent by the sender to test new queues and to complete switching
QTEST (NonEmpty SndQAddr)
+ | -- sent by the sender to remove a queue from the connection (fast rotation, v8)
+ QEND SndQAddr
| -- ratchet re-synchronization is complete, with last decrypted sender message id (recipient's `last_external_snd_msg_id`)
EREADY AgentMsgId
deriving (Show)
@@ -1105,6 +1119,7 @@ aMessageType = \case
QKEY _ -> AM_QKEY_
QUSE _ -> AM_QUSE_
QTEST _ -> AM_QTEST_
+ QEND _ -> AM_QEND_
EREADY _ -> AM_EREADY_
-- | this type is used to send as part of the protocol between different clients
@@ -1157,6 +1172,7 @@ instance Encoding AMessage where
QKEY qs -> smpEncode (QKEY_, qs)
QUSE qs -> smpEncode (QUSE_, qs)
QTEST qs -> smpEncode (QTEST_, qs)
+ QEND addr -> smpEncode (QEND_, addr)
EREADY lastDecryptedMsgId -> smpEncode (EREADY_, lastDecryptedMsgId)
smpP =
smpP
@@ -1169,6 +1185,7 @@ instance Encoding AMessage where
QKEY_ -> QKEY <$> smpP
QUSE_ -> QUSE <$> smpP
QTEST_ -> QTEST <$> smpP
+ QEND_ -> QEND <$> smpP
EREADY_ -> EREADY <$> smpP
instance ToField AMessage where toField = toField . Binary . smpEncode
diff --git a/src/Simplex/Messaging/Agent/Store.hs b/src/Simplex/Messaging/Agent/Store.hs
index 371fae798..1054d1d17 100644
--- a/src/Simplex/Messaging/Agent/Store.hs
+++ b/src/Simplex/Messaging/Agent/Store.hs
@@ -207,12 +207,13 @@ rcvSMPQueueAddress :: RcvQueue -> SMPQueueAddress
rcvSMPQueueAddress RcvQueue {server, sndId, e2ePrivKey, queueMode} =
SMPQueueAddress server sndId (C.publicKey e2ePrivKey) queueMode
-canAbortRcvSwitch :: RcvQueue -> Bool
-canAbortRcvSwitch = maybe False canAbort . rcvSwchStatus
+canAbortRcvSwitch :: VersionSMPA -> RcvQueue -> Bool
+canAbortRcvSwitch connAgentVersion = maybe False canAbort . rcvSwchStatus
where
canAbort = \case
RSSwitchStarted -> True
- RSSendingQADD -> True
+ -- at agent version 8 and above the peer always chooses fast rotation, so a sent QADD is committed
+ RSSendingQADD -> connAgentVersion < rpcAddressSMPAgentVersion
-- if switch is in RSSendingQUSE, a race condition with sender deleting the original queue is possible
RSSendingQUSE -> False
-- if switch is in RSReceivedMessage status, aborting switch (deleting new queue)
@@ -539,6 +540,7 @@ data InternalCommand
| ICDeleteRcvQueue SMP.RecipientId
| ICQSecure SMP.RecipientId SMP.SndPublicAuthKey
| ICQDelete SMP.RecipientId
+ | ICQSndSecure SMP.SenderId
| ICReplyDel
data InternalCommandTag
@@ -550,6 +552,7 @@ data InternalCommandTag
| ICDeleteRcvQueue_
| ICQSecure_
| ICQDelete_
+ | ICQSndSecure_
| ICReplyDel_
deriving (Show)
@@ -563,6 +566,7 @@ instance StrEncoding InternalCommand where
ICDeleteRcvQueue rId -> strEncode (ICDeleteRcvQueue_, rId)
ICQSecure rId senderKey -> strEncode (ICQSecure_, rId, senderKey)
ICQDelete rId -> strEncode (ICQDelete_, rId)
+ ICQSndSecure sId -> strEncode (ICQSndSecure_, sId)
ICReplyDel -> strEncode ICReplyDel_
strP =
strP >>= \case
@@ -574,6 +578,7 @@ instance StrEncoding InternalCommand where
ICDeleteRcvQueue_ -> ICDeleteRcvQueue <$> _strP
ICQSecure_ -> ICQSecure <$> _strP <*> _strP
ICQDelete_ -> ICQDelete <$> _strP
+ ICQSndSecure_ -> ICQSndSecure <$> _strP
ICReplyDel_ -> pure ICReplyDel
instance StrEncoding InternalCommandTag where
@@ -586,6 +591,7 @@ instance StrEncoding InternalCommandTag where
ICDeleteRcvQueue_ -> "DELETE_RCV_QUEUE"
ICQSecure_ -> "QSECURE"
ICQDelete_ -> "QDELETE"
+ ICQSndSecure_ -> "QSND_SECURE"
ICReplyDel_ -> "REPLY_DEL"
strP =
A.takeTill (== ' ') >>= \case
@@ -597,6 +603,7 @@ instance StrEncoding InternalCommandTag where
"DELETE_RCV_QUEUE" -> pure ICDeleteRcvQueue_
"QSECURE" -> pure ICQSecure_
"QDELETE" -> pure ICQDelete_
+ "QSND_SECURE" -> pure ICQSndSecure_
"REPLY_DEL" -> pure ICReplyDel_
_ -> fail "bad InternalCommandTag"
@@ -615,6 +622,7 @@ internalCmdTag = \case
ICDeleteRcvQueue {} -> ICDeleteRcvQueue_
ICQSecure {} -> ICQSecure_
ICQDelete _ -> ICQDelete_
+ ICQSndSecure {} -> ICQSndSecure_
ICReplyDel -> ICReplyDel_
-- * Confirmation types
diff --git a/src/Simplex/Messaging/Agent/Store/AgentStore.hs b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
index dacd3c602..c7a07c727 100644
--- a/src/Simplex/Messaging/Agent/Store/AgentStore.hs
+++ b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
@@ -131,6 +131,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
createSndMsg,
updateSndMsgHash,
createSndMsgDelivery,
+ getPendingSndDeliveries,
getSndMsgViaRcpt,
updateSndMsgRcpt,
getPendingQueueMsg,
@@ -1041,6 +1042,19 @@ createSndMsgDelivery :: DB.Connection -> SndQueue -> InternalId -> IO ()
createSndMsgDelivery db SndQueue {connId, dbQueueId} msgId =
DB.execute db "INSERT INTO snd_message_deliveries (conn_id, snd_queue_id, internal_id) VALUES (?, ?, ?)" (connId, dbQueueId, msgId)
+getPendingSndDeliveries :: DB.Connection -> ConnId -> SndQueue -> IO [InternalId]
+getPendingSndDeliveries db connId SndQueue {dbQueueId} =
+ map fromOnly
+ <$> DB.query
+ db
+ [sql|
+ SELECT internal_id
+ FROM snd_message_deliveries
+ WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0
+ ORDER BY internal_id ASC
+ |]
+ (connId, dbQueueId)
+
getSndMsgViaRcpt :: DB.Connection -> ConnId -> InternalSndId -> IO (Either StoreError SndMsg)
getSndMsgViaRcpt db connId sndMsgId =
firstRow toSndMsg (SEMsgNotFound "getSndMsgViaRcpt") $
diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs
index 8e6f1a08c..502193c83 100644
--- a/tests/AgentTests/FunctionalAPITests.hs
+++ b/tests/AgentTests/FunctionalAPITests.hs
@@ -52,6 +52,7 @@ module AgentTests.FunctionalAPITests
pattern Msg',
pattern SENT,
agentCfgVPrevPQ,
+ agentCfgV7,
)
where
@@ -527,6 +528,8 @@ functionalAPITests ps = do
it "should handle service unavailable on startup" $ testServiceUnavailableOnStartup ps
it "migrate connections to and from service" $ testMigrateConnectionsToService ps
describe "Connection switch" $ do
+ describe "should switch delivery to the new queue with fast rotation" $
+ testServerMatrix2 ps testFastSwitchConnection
describe "should switch delivery to the new queue" $
testServerMatrix2 ps testSwitchConnection
describe "should switch to new queue asynchronously" $
@@ -3517,9 +3520,13 @@ testUsersNoServer ps = withAgentClientsCfg2 aCfg agentCfg $ \a b -> do
where
aCfg = agentCfg {initialCleanupDelay = 10000, cleanupInterval = 10000, deleteErrorCount = 3}
+-- fast rotation runs at agent version 8+; these tests pin to v7 to exercise the QKEY/QUSE slow path and switch abort
+agentCfgV7 :: AgentConfig
+agentCfgV7 = agentCfg {smpAgentVRange = mkVersionRange 6 7}
+
testSwitchConnection :: InitialAgentServers -> IO ()
testSwitchConnection servers =
- withAgentClientsCfgServers2 agentCfg agentCfg servers $ \a b -> runRight_ $ do
+ withAgentClientsCfgServers2 agentCfgV7 agentCfgV7 servers $ \a b -> runRight_ $ do
(aId, bId) <- makeConnection a b
exchangeGreetings a bId b aId
testFullSwitch a bId b aId 8
@@ -3543,6 +3550,25 @@ switchComplete a bId b aId = do
phaseSnd b aId SPCompleted [Nothing]
phaseRcv a bId SPCompleted [Nothing]
+testFastSwitchConnection :: InitialAgentServers -> IO ()
+testFastSwitchConnection servers =
+ withAgentClientsCfgServers2 agentCfg agentCfg servers $ \a b -> runRight_ $ do
+ (aId, bId) <- makeConnection a b
+ exchangeGreetings a bId b aId
+ stats <- switchConnectionAsync a "" bId
+ liftIO $ rcvSwchStatuses' stats `shouldMatchList` [Just RSSwitchStarted]
+ fastSwitchComplete a bId b aId
+ exchangeGreetingsMsgId 6 a bId b aId
+
+fastSwitchComplete :: AgentClient -> ByteString -> AgentClient -> ByteString -> ExceptT AgentErrorType IO ()
+fastSwitchComplete a bId b aId = do
+ phaseRcv a bId SPStarted [Just RSSendingQADD, Nothing]
+ phaseSnd b aId SPStarted [Just SSSecuringQueue, Nothing]
+ phaseSnd b aId SPSecured [Just SSSendingQEND, Nothing]
+ phaseRcv a bId SPConfirmed [Just RSSendingQADD, Nothing]
+ phaseRcv a bId SPCompleted [Nothing]
+ phaseSnd b aId SPCompleted [Nothing]
+
phaseRcv :: AgentClient -> ByteString -> SwitchPhase -> [Maybe RcvSwitchStatus] -> ExceptT AgentErrorType IO ()
phaseRcv c connId p swchStatuses = phase c connId QDRcv p (\stats -> rcvSwchStatuses' stats `shouldMatchList` swchStatuses)
@@ -3599,9 +3625,9 @@ testSwitchAsync servers = do
testFullSwitch a bId b aId 14
where
withA :: (AgentClient -> IO a) -> IO a
- withA = withAgent 1 agentCfg servers testDB
+ withA = withAgent 1 agentCfgV7 servers testDB
withB :: (AgentClient -> IO a) -> IO a
- withB = withAgent 2 agentCfg servers testDB2
+ withB = withAgent 2 agentCfgV7 servers testDB2
withAgent :: HasCallStack => Int -> AgentConfig -> InitialAgentServers -> String -> (HasCallStack => AgentClient -> IO a) -> IO a
withAgent clientId cfg' servers dbPath = bracket (getSMPAgentClient' clientId cfg' servers dbPath) (\a -> disposeAgentClient a >> threadDelay 100000)
@@ -3617,7 +3643,7 @@ sessionSubscribe withC connIds a =
testSwitchDelete :: InitialAgentServers -> IO ()
testSwitchDelete servers =
- withAgentClientsCfgServers2 agentCfg agentCfg servers $ \a b -> runRight_ $ do
+ withAgentClientsCfgServers2 agentCfgV7 agentCfgV7 servers $ \a b -> runRight_ $ do
(aId, bId) <- makeConnection a b
exchangeGreetings a bId b aId
liftIO $ disposeAgentClient b
@@ -3675,9 +3701,9 @@ testAbortSwitchStarted servers = do
testFullSwitch a bId b aId 16
where
withA :: (AgentClient -> IO a) -> IO a
- withA = withAgent 1 agentCfg servers testDB
+ withA = withAgent 1 agentCfgV7 servers testDB
withB :: (AgentClient -> IO a) -> IO a
- withB = withAgent 2 agentCfg servers testDB2
+ withB = withAgent 2 agentCfgV7 servers testDB2
testAbortSwitchStartedReinitiate :: HasCallStack => InitialAgentServers -> IO ()
testAbortSwitchStartedReinitiate servers = do
@@ -3726,9 +3752,9 @@ testAbortSwitchStartedReinitiate servers = do
testFullSwitch a bId b aId 16
where
withA :: (AgentClient -> IO a) -> IO a
- withA = withAgent 1 agentCfg servers testDB
+ withA = withAgent 1 agentCfgV7 servers testDB
withB :: (AgentClient -> IO a) -> IO a
- withB = withAgent 2 agentCfg servers testDB2
+ withB = withAgent 2 agentCfgV7 servers testDB2
switchPhaseRcvP :: ConnId -> SwitchPhase -> [Maybe RcvSwitchStatus] -> ATransmission -> Bool
switchPhaseRcvP cId sphase swchStatuses = switchPhaseP cId QDRcv sphase (\stats -> rcvSwchStatuses' stats == swchStatuses)
@@ -3780,9 +3806,9 @@ testCannotAbortSwitchSecured servers = do
testFullSwitch a bId b aId 14
where
withA :: (AgentClient -> IO a) -> IO a
- withA = withAgent 1 agentCfg servers testDB
+ withA = withAgent 1 agentCfgV7 servers testDB
withB :: (AgentClient -> IO a) -> IO a
- withB = withAgent 2 agentCfg servers testDB2
+ withB = withAgent 2 agentCfgV7 servers testDB2
testSwitch2Connections :: HasCallStack => InitialAgentServers -> IO ()
testSwitch2Connections servers = do
@@ -3838,9 +3864,9 @@ testSwitch2Connections servers = do
testFullSwitch a bId2 b aId2 14
where
withA :: (AgentClient -> IO a) -> IO a
- withA = withAgent 1 agentCfg servers testDB
+ withA = withAgent 1 agentCfgV7 servers testDB
withB :: (AgentClient -> IO a) -> IO a
- withB = withAgent 2 agentCfg servers testDB2
+ withB = withAgent 2 agentCfgV7 servers testDB2
testSwitch2ConnectionsAbort1 :: HasCallStack => InitialAgentServers -> IO ()
testSwitch2ConnectionsAbort1 servers = do
@@ -3891,9 +3917,9 @@ testSwitch2ConnectionsAbort1 servers = do
testFullSwitch a bId2 b aId2 12
where
withA :: (AgentClient -> IO a) -> IO a
- withA = withAgent 1 agentCfg servers testDB
+ withA = withAgent 1 agentCfgV7 servers testDB
withB :: (AgentClient -> IO a) -> IO a
- withB = withAgent 2 agentCfg servers testDB2
+ withB = withAgent 2 agentCfgV7 servers testDB2
testCreateQueueAuth :: HasCallStack => (Maybe BasicAuth, VersionSMP) -> (Maybe BasicAuth, VersionSMP) -> SndQueueSecured -> AgentMsgId -> IO Int
testCreateQueueAuth clnt1 clnt2 sqSecured baseId = do
diff --git a/tests/AgentTests/NotificationTests.hs b/tests/AgentTests/NotificationTests.hs
index ed6455deb..dbe21d85f 100644
--- a/tests/AgentTests/NotificationTests.hs
+++ b/tests/AgentTests/NotificationTests.hs
@@ -17,7 +17,8 @@ module AgentTests.NotificationTests where
-- import Control.Logger.Simple (LogConfig (..), LogLevel (..), setLogLevel, withGlobalLogging)
import AgentTests.FunctionalAPITests
- ( agentCfgVPrevPQ,
+ ( agentCfgV7,
+ agentCfgVPrevPQ,
createConnection,
exchangeGreetings,
get,
@@ -870,7 +871,7 @@ testNotificationsSMPRestartBatch n ps@(t, ASType qsType _) apns =
testSwitchNotifications :: InitialAgentServers -> APNSMockServer -> IO ()
testSwitchNotifications servers apns =
- withAgentClientsCfgServers2 agentCfg agentCfg servers $ \a b -> runRight_ $ do
+ withAgentClientsCfgServers2 agentCfgV7 agentCfgV7 servers $ \a b -> runRight_ $ do
(aId, bId) <- makeConnection a b
exchangeGreetings a bId b aId
_ <- registerTestToken a "abcd" NMInstant apns
From 06331b78f4c21e009b1c0a7c6cde2d80a39da9fb Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Sun, 16 Aug 2026 18:46:50 +0000
Subject: [PATCH 07/16] test
---
tests/AgentTests/FunctionalAPITests.hs | 34 +++++++++++++++++++++++++-
1 file changed, 33 insertions(+), 1 deletion(-)
diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs
index 502193c83..34c1b4c99 100644
--- a/tests/AgentTests/FunctionalAPITests.hs
+++ b/tests/AgentTests/FunctionalAPITests.hs
@@ -69,7 +69,7 @@ import qualified Data.ByteString.Char8 as B
import Data.Either (isRight)
import Data.Int (Int64)
import Data.List (find, isPrefixOf, isSuffixOf)
-import Data.List.NonEmpty (NonEmpty)
+import Data.List.NonEmpty (NonEmpty (..))
import qualified Data.Map as M
import Data.Maybe (isJust, isNothing)
import qualified Data.Set as S
@@ -530,6 +530,8 @@ functionalAPITests ps = do
describe "Connection switch" $ do
describe "should switch delivery to the new queue with fast rotation" $
testServerMatrix2 ps testFastSwitchConnection
+ it "should switch delivery to the new queue when the old server is down" $
+ testFastSwitchDeadOldServer ps
describe "should switch delivery to the new queue" $
testServerMatrix2 ps testSwitchConnection
describe "should switch to new queue asynchronously" $
@@ -3569,6 +3571,36 @@ fastSwitchComplete a bId b aId = do
phaseRcv a bId SPCompleted [Nothing]
phaseSnd b aId SPCompleted [Nothing]
+-- A's old receive queue is on server1 (stopped after the connection is set up); B's queue and the new queue are on server2.
+-- Fast rotation completes over the live server: B secures the new queue and sends the confirmation and QEND on it,
+-- so the recipient moves to it and removes the old queue without the old server.
+testFastSwitchDeadOldServer :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
+testFastSwitchDeadOldServer ps@(t, ASType qsType _) = do
+ let bServers = initAgentServers {smp = userServers [testSMPServer2]}
+ withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ ->
+ withAgent 1 agentCfg initAgentServers testDB $ \a ->
+ withAgent 2 agentCfg bServers testDB2 $ \b -> do
+ (aId, bId) <- withSmpServerStoreLogOn ps testPort $ \_ -> runRight $ do
+ (aId, bId) <- makeConnection a b
+ exchangeGreetings a bId b aId
+ -- create the rotated queue on the live server
+ liftIO $ setProtocolServers a 1 (noAuthSrvCfg testSMPServer2 :| [])
+ pure (aId, bId)
+ nGet a =##> \case ("", "", DOWN _ cs) -> bId `elem` cs; _ -> False
+ runRight_ $ do
+ _ <- switchConnectionAsync a "" bId
+ switchCompleted a bId QDRcv
+ switchCompleted b aId QDSnd
+ exchangeGreetingsMsgId 6 a bId b aId
+
+-- drains switch and network events until the connection reports SPCompleted in the given direction,
+-- tolerating DOWN/UP and intermediate phases (the old server is stopped mid-rotation)
+switchCompleted :: AgentClient -> ByteString -> QueueDirection -> ExceptT AgentErrorType IO ()
+switchCompleted c connId d =
+ pGet c >>= \case
+ (_, connId', AEvt SAEConn (SWITCH d' SPCompleted _)) | connId' == connId && d' == d -> pure ()
+ _ -> switchCompleted c connId d
+
phaseRcv :: AgentClient -> ByteString -> SwitchPhase -> [Maybe RcvSwitchStatus] -> ExceptT AgentErrorType IO ()
phaseRcv c connId p swchStatuses = phase c connId QDRcv p (\stats -> rcvSwchStatuses' stats `shouldMatchList` swchStatuses)
From 535522158061abce4ccc4b676745f5718b5c4848 Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 07:41:41 +0000
Subject: [PATCH 08/16] refactor
---
src/Simplex/Messaging/Agent.hs | 28 +++++++++----------
src/Simplex/Messaging/Agent/Protocol.hs | 6 ++--
.../Messaging/Agent/Store/AgentStore.hs | 26 ++++++++---------
tests/AgentTests/FunctionalAPITests.hs | 21 +++++++++++---
4 files changed, 47 insertions(+), 34 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index 2984dc6a7..1bda98397 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2291,7 +2291,7 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
void $ setSndSwitchStatus db oldSq $ Just SSSendingQEND
let sq'' = (sq' :: SndQueue) {status = Active, primary = True, dbReplaceQueueId = Nothing}
lift $ submitPendingMsg c sq''
- void $ enqueueMessages c cData (oldSq :| [sq'']) SMP.noMsgFlags $ QEND (oldServer, oldSndId)
+ void $ enqueueMessages c cData [oldSq, sq''] SMP.noMsgFlags $ QEND [(oldServer, oldSndId)]
SomeConn _ conn' <- withStore c (`getConn` connId)
cStats <- connectionStats c conn'
notify $ SWITCH QDSnd SPSecured cStats
@@ -3592,7 +3592,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
-- no action needed for QTEST
-- any message in the new queue will mark it active and trigger deletion of the old queue
QTEST _ -> logServer "<--" c srv rId ("MSG :" <> logSecret' srvMsgId) >> ackDel msgId
- QEND addr -> qDuplexAckDel conn'' "QEND" $ qEndMsg srvMsgId addr
+ QEND addrs -> qDuplexAckDel conn'' "QEND" $ qEndMsg srvMsgId addrs
EREADY _ -> qDuplexAckDel conn'' "EREADY" $ ereadyMsg rcPrev
where
qDuplexAckDel :: Connection c -> String -> (Connection 'CDuplex -> AM ()) -> AM ACKd
@@ -3944,9 +3944,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
swchStatus <-
if connAgentVersion >= rpcAddressSMPAgentVersion
then do
- withStore' c $ \db -> do
- pending <- getPendingSndDeliveries db connId sq
- mapM_ (createSndMsgDelivery db sq2) pending
+ withStore' c $ \db -> copyPendingSndDeliveries db sq sq2
enqueueCommand c "" connId (Just $ qServer sq2) $ AInternalCommand $ ICQSndSecure (snd $ qAddress sq2)
pure SSSecuringQueue
else do
@@ -4007,20 +4005,22 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
_ -> qError "QUSE: switching SndQueue not found in connection"
_ -> qError "QUSE: switched queue address not found in connection"
- -- processed by queue recipient: remove the queue named by QEND
- qEndMsg :: SMP.MsgId -> SndQAddr -> Connection 'CDuplex -> AM ()
- qEndMsg srvMsgId addr (DuplexConnection cData'@ConnData {enableNtfs} rqs sqs) =
- case removeQP (\rq' -> sameQAddress addr (sndAddress rq')) rqs of
- Just (rq'@RcvQueue {server = rmServer, rcvId}, rq'' : rqs') -> do
- logServer "<--" c srv rId $ "MSG :" <> logSecret' srvMsgId <> " " <> logSecret (snd addr)
- withStore' c $ \db -> setRcvQueueDeleted db rq'
- enqueueCommand c "" connId (Just rmServer) $ AInternalCommand $ ICDeleteRcvQueue rcvId
+ -- processed by queue recipient: remove the queues named by QEND
+ qEndMsg :: SMP.MsgId -> NonEmpty SndQAddr -> Connection 'CDuplex -> AM ()
+ qEndMsg srvMsgId addrs (DuplexConnection cData'@ConnData {enableNtfs} rqs sqs) =
+ case L.partition (\rq' -> any (`sameQAddress` sndAddress rq') addrs) rqs of
+ (removed@(_ : _), keptRq : keptRqs) -> do
+ logServer "<--" c srv rId $ "MSG :" <> logSecret' srvMsgId
+ forM_ removed $ \rq'@RcvQueue {server = rmServer, rcvId} -> do
+ withStore' c $ \db -> setRcvQueueDeleted db rq'
+ enqueueCommand c "" connId (Just rmServer) $ AInternalCommand $ ICDeleteRcvQueue rcvId
when enableNtfs $ do
ns <- asks ntfSupervisor
liftIO $ sendNtfSubCommand ns (NSCCreate, [connId])
- let conn' = DuplexConnection cData' (rq'' :| rqs') sqs
+ let conn' = DuplexConnection cData' (keptRq :| keptRqs) sqs
cStats <- connectionStats c conn'
notify $ SWITCH QDRcv SPCompleted cStats
+ -- named queues already removed, or none would remain
_ -> pure ()
qError :: String -> AM a
diff --git a/src/Simplex/Messaging/Agent/Protocol.hs b/src/Simplex/Messaging/Agent/Protocol.hs
index 4d205608a..24edae75b 100644
--- a/src/Simplex/Messaging/Agent/Protocol.hs
+++ b/src/Simplex/Messaging/Agent/Protocol.hs
@@ -1099,8 +1099,8 @@ data AMessage
QUSE (NonEmpty (SndQAddr, Bool))
| -- sent by the sender to test new queues and to complete switching
QTEST (NonEmpty SndQAddr)
- | -- sent by the sender to remove a queue from the connection (fast rotation, v8)
- QEND SndQAddr
+ | -- sent by the sender to remove queues from the connection (fast rotation, v8)
+ QEND (NonEmpty SndQAddr)
| -- ratchet re-synchronization is complete, with last decrypted sender message id (recipient's `last_external_snd_msg_id`)
EREADY AgentMsgId
deriving (Show)
@@ -1172,7 +1172,7 @@ instance Encoding AMessage where
QKEY qs -> smpEncode (QKEY_, qs)
QUSE qs -> smpEncode (QUSE_, qs)
QTEST qs -> smpEncode (QTEST_, qs)
- QEND addr -> smpEncode (QEND_, addr)
+ QEND addrs -> smpEncode (QEND_, addrs)
EREADY lastDecryptedMsgId -> smpEncode (EREADY_, lastDecryptedMsgId)
smpP =
smpP
diff --git a/src/Simplex/Messaging/Agent/Store/AgentStore.hs b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
index c7a07c727..bd6735787 100644
--- a/src/Simplex/Messaging/Agent/Store/AgentStore.hs
+++ b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
@@ -131,7 +131,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
createSndMsg,
updateSndMsgHash,
createSndMsgDelivery,
- getPendingSndDeliveries,
+ copyPendingSndDeliveries,
getSndMsgViaRcpt,
updateSndMsgRcpt,
getPendingQueueMsg,
@@ -1042,18 +1042,18 @@ createSndMsgDelivery :: DB.Connection -> SndQueue -> InternalId -> IO ()
createSndMsgDelivery db SndQueue {connId, dbQueueId} msgId =
DB.execute db "INSERT INTO snd_message_deliveries (conn_id, snd_queue_id, internal_id) VALUES (?, ?, ?)" (connId, dbQueueId, msgId)
-getPendingSndDeliveries :: DB.Connection -> ConnId -> SndQueue -> IO [InternalId]
-getPendingSndDeliveries db connId SndQueue {dbQueueId} =
- map fromOnly
- <$> DB.query
- db
- [sql|
- SELECT internal_id
- FROM snd_message_deliveries
- WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0
- ORDER BY internal_id ASC
- |]
- (connId, dbQueueId)
+-- copies every undelivered (failed = 0) delivery from one snd queue to another, for redundant delivery during fast rotation
+copyPendingSndDeliveries :: DB.Connection -> SndQueue -> SndQueue -> IO ()
+copyPendingSndDeliveries db SndQueue {connId, dbQueueId = fromQueueId} SndQueue {dbQueueId = toQueueId} =
+ DB.execute
+ db
+ [sql|
+ INSERT INTO snd_message_deliveries (conn_id, snd_queue_id, internal_id)
+ SELECT conn_id, ?, internal_id
+ FROM snd_message_deliveries
+ WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0
+ |]
+ (toQueueId, connId, fromQueueId)
getSndMsgViaRcpt :: DB.Connection -> ConnId -> InternalSndId -> IO (Either StoreError SndMsg)
getSndMsgViaRcpt db connId sndMsgId =
diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs
index 34c1b4c99..2432c75df 100644
--- a/tests/AgentTests/FunctionalAPITests.hs
+++ b/tests/AgentTests/FunctionalAPITests.hs
@@ -69,7 +69,7 @@ import qualified Data.ByteString.Char8 as B
import Data.Either (isRight)
import Data.Int (Int64)
import Data.List (find, isPrefixOf, isSuffixOf)
-import Data.List.NonEmpty (NonEmpty (..))
+import Data.List.NonEmpty (NonEmpty)
import qualified Data.Map as M
import Data.Maybe (isJust, isNothing)
import qualified Data.Set as S
@@ -3584,14 +3584,17 @@ testFastSwitchDeadOldServer ps@(t, ASType qsType _) = do
(aId, bId) <- makeConnection a b
exchangeGreetings a bId b aId
-- create the rotated queue on the live server
- liftIO $ setProtocolServers a 1 (noAuthSrvCfg testSMPServer2 :| [])
+ liftIO $ setProtocolServers a 1 [noAuthSrvCfg testSMPServer2]
pure (aId, bId)
nGet a =##> \case ("", "", DOWN _ cs) -> bId `elem` cs; _ -> False
runRight_ $ do
+ -- a message queued while the old server is down must survive the rotation and arrive on the new queue
+ _ <- sendMessage b aId SMP.noMsgFlags "queued while down"
_ <- switchConnectionAsync a "" bId
- switchCompleted a bId QDRcv
+ queuedReceived <- switchCompletedRcvMsg a bId "queued while down"
+ liftIO $ queuedReceived `shouldBe` True
switchCompleted b aId QDSnd
- exchangeGreetingsMsgId 6 a bId b aId
+ exchangeGreetingsMsgId 7 a bId b aId
-- drains switch and network events until the connection reports SPCompleted in the given direction,
-- tolerating DOWN/UP and intermediate phases (the old server is stopped mid-rotation)
@@ -3601,6 +3604,16 @@ switchCompleted c connId d =
(_, connId', AEvt SAEConn (SWITCH d' SPCompleted _)) | connId' == connId && d' == d -> pure ()
_ -> switchCompleted c connId d
+-- like switchCompleted for QDRcv, additionally acking and reporting a message matching the body seen while draining
+switchCompletedRcvMsg :: AgentClient -> ByteString -> MsgBody -> ExceptT AgentErrorType IO Bool
+switchCompletedRcvMsg c connId body = go False
+ where
+ go seen =
+ pGet c >>= \case
+ (_, connId', AEvt SAEConn (SWITCH QDRcv SPCompleted _)) | connId' == connId -> pure seen
+ (_, connId', AEvt SAEConn (Msg' mId _ body')) | connId' == connId && body' == body -> ackMessage c connId' mId Nothing >> go True
+ _ -> go seen
+
phaseRcv :: AgentClient -> ByteString -> SwitchPhase -> [Maybe RcvSwitchStatus] -> ExceptT AgentErrorType IO ()
phaseRcv c connId p swchStatuses = phase c connId QDRcv p (\stats -> rcvSwchStatuses' stats `shouldMatchList` swchStatuses)
From a12b5534f43d6a0c38b2861bd832baa96019635a Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 12:37:45 +0000
Subject: [PATCH 09/16] test switching notificaitons
---
tests/AgentTests/FunctionalAPITests.hs | 1 +
tests/AgentTests/NotificationTests.hs | 17 +++++++++++------
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs
index 2432c75df..259a52227 100644
--- a/tests/AgentTests/FunctionalAPITests.hs
+++ b/tests/AgentTests/FunctionalAPITests.hs
@@ -30,6 +30,7 @@ module AgentTests.FunctionalAPITests
makeConnection,
exchangeGreetings,
switchComplete,
+ fastSwitchComplete,
createConnection,
joinConnection,
sendMessage,
diff --git a/tests/AgentTests/NotificationTests.hs b/tests/AgentTests/NotificationTests.hs
index dbe21d85f..d81c08a41 100644
--- a/tests/AgentTests/NotificationTests.hs
+++ b/tests/AgentTests/NotificationTests.hs
@@ -29,6 +29,7 @@ import AgentTests.FunctionalAPITests
runRight_,
sendMessage,
switchComplete,
+ fastSwitchComplete,
testServerMatrix2,
withAgent,
withAgentClients2,
@@ -165,10 +166,14 @@ notificationTests ps@(t, _) = do
it "should resume batched subscriptions after SMP server is restarted" $
withAPNSMockServer $ \apns ->
withNtfServer t $ testNotificationsSMPRestartBatch 50 ps apns
- describe "should switch notifications to the new queue" $
+ describe "should switch notifications to the new queue (slow rotation)" $
testServerMatrix2 ps $ \servers ->
withAPNSMockServer $ \apns ->
- withNtfServer t $ testSwitchNotifications servers apns
+ withNtfServer t $ testSwitchNotifications agentCfgV7 switchComplete servers apns
+ describe "should switch notifications to the new queue (fast rotation)" $
+ testServerMatrix2 ps $ \servers ->
+ withAPNSMockServer $ \apns ->
+ withNtfServer t $ testSwitchNotifications agentCfg fastSwitchComplete servers apns
it "should keep sending notifications for old token" $
withSmpServer ps $
withAPNSMockServer $ \apns ->
@@ -869,9 +874,9 @@ testNotificationsSMPRestartBatch n ps@(t, ASType qsType _) apns =
killThread t1
pure res
-testSwitchNotifications :: InitialAgentServers -> APNSMockServer -> IO ()
-testSwitchNotifications servers apns =
- withAgentClientsCfgServers2 agentCfgV7 agentCfgV7 servers $ \a b -> runRight_ $ do
+testSwitchNotifications :: AgentConfig -> (AgentClient -> ByteString -> AgentClient -> ByteString -> ExceptT AgentErrorType IO ()) -> InitialAgentServers -> APNSMockServer -> IO ()
+testSwitchNotifications cfg completeSwitch servers apns =
+ withAgentClientsCfgServers2 cfg cfg servers $ \a b -> runRight_ $ do
(aId, bId) <- makeConnection a b
exchangeGreetings a bId b aId
_ <- registerTestToken a "abcd" NMInstant apns
@@ -884,7 +889,7 @@ testSwitchNotifications servers apns =
ackMessage a bId msgId Nothing
testMessage "hello"
_ <- switchConnectionAsync a "" bId
- switchComplete a bId b aId
+ completeSwitch a bId b aId
liftIO $ threadDelay 500000
testMessage "hello again"
From 80b32aed0f3c618aeeb6bf97093bbc71fd9f0a34 Mon Sep 17 00:00:00 2001
From: Evgeny Poberezkin
Date: Mon, 17 Aug 2026 18:18:07 +0100
Subject: [PATCH 10/16] rename
---
src/Simplex/Messaging/Agent/Protocol.hs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/Simplex/Messaging/Agent/Protocol.hs b/src/Simplex/Messaging/Agent/Protocol.hs
index 24edae75b..78630b9cb 100644
--- a/src/Simplex/Messaging/Agent/Protocol.hs
+++ b/src/Simplex/Messaging/Agent/Protocol.hs
@@ -1172,7 +1172,7 @@ instance Encoding AMessage where
QKEY qs -> smpEncode (QKEY_, qs)
QUSE qs -> smpEncode (QUSE_, qs)
QTEST qs -> smpEncode (QTEST_, qs)
- QEND addrs -> smpEncode (QEND_, addrs)
+ QEND qs -> smpEncode (QEND_, qs)
EREADY lastDecryptedMsgId -> smpEncode (EREADY_, lastDecryptedMsgId)
smpP =
smpP
From b01ad76751a0f071b5a4e6d6cc96caeaa04902fc Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 18:28:42 +0000
Subject: [PATCH 11/16] fix, refactor
---
src/Simplex/Messaging/Agent.hs | 10 +++++++---
src/Simplex/Messaging/Agent/Store/AgentStore.hs | 10 ++++++++++
2 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index 1bda98397..3dad3dd57 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2290,7 +2290,11 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
setSndQueuePrimary db connId sq'
void $ setSndSwitchStatus db oldSq $ Just SSSendingQEND
let sq'' = (sq' :: SndQueue) {status = Active, primary = True, dbReplaceQueueId = Nothing}
- lift $ submitPendingMsg c sq''
+ -- R''s worker was stopped while securing, so its accumulated deliveries were not counted in
+ -- msgDeliveryOp; add their count as it starts, since it subtracts one per delivery.
+ pending <- withStore' c $ \db -> countPendingSndDeliveries db sq''
+ atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + pending}
+ lift $ resumeMsgDelivery c sq''
void $ enqueueMessages c cData [oldSq, sq''] SMP.noMsgFlags $ QEND [(oldServer, oldSndId)]
SomeConn _ conn' <- withStore c (`getConn` connId)
cStats <- connectionStats c conn'
@@ -2376,9 +2380,9 @@ enqueueMessageB c reqs = do
pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId))
where
sndDeliverTo :: ConnData -> SndQueue -> Bool
- sndDeliverTo ConnData {connAgentVersion} sq@SndQueue {status, sndSwchStatus} =
+ sndDeliverTo ConnData {connAgentVersion} sq@SndQueue {sndSwchStatus} =
sndSwchStatus /= Just SSSendingQEND
- && (isActiveSndQ sq || (connAgentVersion >= rpcAddressSMPAgentVersion && status == New && isJust (dbReplaceQId sq)))
+ && (isActiveSndQ sq || (connAgentVersion >= rpcAddressSMPAgentVersion && securingSndQueue sq))
storeSentMsg ::
DB.Connection ->
AgentConfig ->
diff --git a/src/Simplex/Messaging/Agent/Store/AgentStore.hs b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
index bd6735787..586e97126 100644
--- a/src/Simplex/Messaging/Agent/Store/AgentStore.hs
+++ b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
@@ -132,6 +132,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
updateSndMsgHash,
createSndMsgDelivery,
copyPendingSndDeliveries,
+ countPendingSndDeliveries,
getSndMsgViaRcpt,
updateSndMsgRcpt,
getPendingQueueMsg,
@@ -1055,6 +1056,15 @@ copyPendingSndDeliveries db SndQueue {connId, dbQueueId = fromQueueId} SndQueue
|]
(toQueueId, connId, fromQueueId)
+countPendingSndDeliveries :: DB.Connection -> SndQueue -> IO Int
+countPendingSndDeliveries db SndQueue {connId, dbQueueId} = do
+ (Only cnt : _) <-
+ DB.query
+ db
+ "SELECT count(1) FROM snd_message_deliveries WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0"
+ (connId, dbQueueId)
+ pure cnt
+
getSndMsgViaRcpt :: DB.Connection -> ConnId -> InternalSndId -> IO (Either StoreError SndMsg)
getSndMsgViaRcpt db connId sndMsgId =
firstRow toSndMsg (SEMsgNotFound "getSndMsgViaRcpt") $
From 8021dc81d2a02573463b544341457300f9c3cc06 Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 21:16:47 +0000
Subject: [PATCH 12/16] fix possible crash
---
src/Simplex/Messaging/Agent.hs | 9 +++++++--
src/Simplex/Messaging/Agent/Store/AgentStore.hs | 14 +++++---------
tests/AgentTests/FunctionalAPITests.hs | 16 ++++++++--------
3 files changed, 20 insertions(+), 19 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index 3dad3dd57..12ce4b294 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2280,7 +2280,10 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
Nothing -> internalErr "ICQSndSecure: queue address not found in connection"
Just sq'@SndQueue {dbReplaceQueueId} ->
case dbReplaceQueueId >>= \replaceQId -> find ((replaceQId ==) . dbQId) sqs of
- Nothing -> pure ()
+ -- R' is already secured; if a crash landed between securing and enqueuing QEND, old still awaits it
+ Nothing ->
+ forM_ (find (\q -> sndSwchStatus q == Just SSSendingQEND) sqs) $ \oldSq ->
+ void $ enqueueMessages c cData [oldSq, sq'] SMP.noMsgFlags $ QEND [qAddress oldSq]
Just oldSq@SndQueue {server = oldServer, sndId = oldSndId} -> do
secureSndQueue c NRMBackground sq'
let confMsg = smpEncode $ AgentConfirmation {agentVersion = connAgentVersion, e2eEncryption_ = Nothing, encConnInfo = ""}
@@ -2292,7 +2295,7 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
let sq'' = (sq' :: SndQueue) {status = Active, primary = True, dbReplaceQueueId = Nothing}
-- R''s worker was stopped while securing, so its accumulated deliveries were not counted in
-- msgDeliveryOp; add their count as it starts, since it subtracts one per delivery.
- pending <- withStore' c $ \db -> countPendingSndDeliveries db sq''
+ pending <- withStore' c $ \db -> countSndQueueDeliveries db sq''
atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + pending}
lift $ resumeMsgDelivery c sq''
void $ enqueueMessages c cData [oldSq, sq''] SMP.noMsgFlags $ QEND [(oldServer, oldSndId)]
@@ -2645,6 +2648,8 @@ runSmpQueueMsgDelivery c@AgentClient {subQ} sq@SndQueue {userId, connId, server,
SomeConn _ conn <- withStore c (`getConn` connId)
case conn of
DuplexConnection cData' rqs sqs ->
+ -- the queue to remove is identified by SSSendingQEND, not the QEND payload (not parsed in the send loop);
+ -- a rotation removes one queue, so a multi-queue QEND would only remove that one here
case removeQP (\sq' -> sndSwchStatus sq' == Just SSSendingQEND) sqs of
Nothing -> pure Nothing
Just (oldSq, sq'' : sqs') -> do
diff --git a/src/Simplex/Messaging/Agent/Store/AgentStore.hs b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
index 586e97126..04fbcf729 100644
--- a/src/Simplex/Messaging/Agent/Store/AgentStore.hs
+++ b/src/Simplex/Messaging/Agent/Store/AgentStore.hs
@@ -132,7 +132,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
updateSndMsgHash,
createSndMsgDelivery,
copyPendingSndDeliveries,
- countPendingSndDeliveries,
+ countSndQueueDeliveries,
getSndMsgViaRcpt,
updateSndMsgRcpt,
getPendingQueueMsg,
@@ -1056,14 +1056,10 @@ copyPendingSndDeliveries db SndQueue {connId, dbQueueId = fromQueueId} SndQueue
|]
(toQueueId, connId, fromQueueId)
-countPendingSndDeliveries :: DB.Connection -> SndQueue -> IO Int
-countPendingSndDeliveries db SndQueue {connId, dbQueueId} = do
- (Only cnt : _) <-
- DB.query
- db
- "SELECT count(1) FROM snd_message_deliveries WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0"
- (connId, dbQueueId)
- pure cnt
+countSndQueueDeliveries :: DB.Connection -> SndQueue -> IO Int
+countSndQueueDeliveries db SndQueue {connId, dbQueueId} =
+ maybeFirstRow' 0 fromOnly $
+ DB.query db "SELECT count(1) FROM snd_message_deliveries WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0" (connId, dbQueueId)
getSndMsgViaRcpt :: DB.Connection -> ConnId -> InternalSndId -> IO (Either StoreError SndMsg)
getSndMsgViaRcpt db connId sndMsgId =
diff --git a/tests/AgentTests/FunctionalAPITests.hs b/tests/AgentTests/FunctionalAPITests.hs
index 259a52227..d831cdea7 100644
--- a/tests/AgentTests/FunctionalAPITests.hs
+++ b/tests/AgentTests/FunctionalAPITests.hs
@@ -3592,22 +3592,22 @@ testFastSwitchDeadOldServer ps@(t, ASType qsType _) = do
-- a message queued while the old server is down must survive the rotation and arrive on the new queue
_ <- sendMessage b aId SMP.noMsgFlags "queued while down"
_ <- switchConnectionAsync a "" bId
- queuedReceived <- switchCompletedRcvMsg a bId "queued while down"
+ queuedReceived <- drainSwitchCompletedRcvMsg a bId "queued while down"
liftIO $ queuedReceived `shouldBe` True
- switchCompleted b aId QDSnd
+ drainSwitchCompleted b aId QDSnd
exchangeGreetingsMsgId 7 a bId b aId
-- drains switch and network events until the connection reports SPCompleted in the given direction,
-- tolerating DOWN/UP and intermediate phases (the old server is stopped mid-rotation)
-switchCompleted :: AgentClient -> ByteString -> QueueDirection -> ExceptT AgentErrorType IO ()
-switchCompleted c connId d =
+drainSwitchCompleted :: AgentClient -> ByteString -> QueueDirection -> ExceptT AgentErrorType IO ()
+drainSwitchCompleted c connId d =
pGet c >>= \case
(_, connId', AEvt SAEConn (SWITCH d' SPCompleted _)) | connId' == connId && d' == d -> pure ()
- _ -> switchCompleted c connId d
+ _ -> drainSwitchCompleted c connId d
--- like switchCompleted for QDRcv, additionally acking and reporting a message matching the body seen while draining
-switchCompletedRcvMsg :: AgentClient -> ByteString -> MsgBody -> ExceptT AgentErrorType IO Bool
-switchCompletedRcvMsg c connId body = go False
+-- like drainSwitchCompleted for QDRcv, additionally acking and reporting a message matching the body seen while draining
+drainSwitchCompletedRcvMsg :: AgentClient -> ByteString -> MsgBody -> ExceptT AgentErrorType IO Bool
+drainSwitchCompletedRcvMsg c connId body = go False
where
go seen =
pGet c >>= \case
From cdce7f2f71916cdffdf8fda07d6dc9cf0c221b65 Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 21:22:18 +0000
Subject: [PATCH 13/16] simplify
---
src/Simplex/Messaging/Agent.hs | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index 12ce4b294..f069f5ebf 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2226,12 +2226,10 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
rq <- withStore c (\db -> getDeletedRcvQueue db connId srv rId)
maxErrs <- asks $ deleteErrorCount . config
tryAllErrors (deleteQueue c NRMBackground rq) >>= \case
- Right () -> withStore' c (`deleteConnRcvQueue` rq)
- Left e
- | temporaryOrHostError e && deleteErrors rq + 1 < maxErrs -> do
- withStore' c (`incRcvDeleteErrors` rq)
- throwE e
- | otherwise -> withStore' c (`deleteConnRcvQueue` rq)
+ Left e | temporaryOrHostError e && deleteErrors rq + 1 < maxErrs -> do
+ withStore' c (`incRcvDeleteErrors` rq)
+ throwE e
+ _ -> withStore' c (`deleteConnRcvQueue` rq)
ICQSecure rId senderKey ->
withServer $ \srv -> tryWithLock "ICQSecure" . withDuplexConn $ \(DuplexConnection cData rqs sqs) ->
case find (sameQueue (srv, rId)) rqs of
From 2499934042674e9f480f4b92fecb38a7eaaec952 Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 21:32:05 +0000
Subject: [PATCH 14/16] refactor
---
src/Simplex/Messaging/Agent.hs | 26 +++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index f069f5ebf..c1a2732b5 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2278,11 +2278,7 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
Nothing -> internalErr "ICQSndSecure: queue address not found in connection"
Just sq'@SndQueue {dbReplaceQueueId} ->
case dbReplaceQueueId >>= \replaceQId -> find ((replaceQId ==) . dbQId) sqs of
- -- R' is already secured; if a crash landed between securing and enqueuing QEND, old still awaits it
- Nothing ->
- forM_ (find (\q -> sndSwchStatus q == Just SSSendingQEND) sqs) $ \oldSq ->
- void $ enqueueMessages c cData [oldSq, sq'] SMP.noMsgFlags $ QEND [qAddress oldSq]
- Just oldSq@SndQueue {server = oldServer, sndId = oldSndId} -> do
+ Just oldSq -> do
secureSndQueue c NRMBackground sq'
let confMsg = smpEncode $ AgentConfirmation {agentVersion = connAgentVersion, e2eEncryption_ = Nothing, encConnInfo = ""}
void $ sendConfirmation c NRMBackground sq' confMsg
@@ -2291,15 +2287,15 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
setSndQueuePrimary db connId sq'
void $ setSndSwitchStatus db oldSq $ Just SSSendingQEND
let sq'' = (sq' :: SndQueue) {status = Active, primary = True, dbReplaceQueueId = Nothing}
- -- R''s worker was stopped while securing, so its accumulated deliveries were not counted in
- -- msgDeliveryOp; add their count as it starts, since it subtracts one per delivery.
- pending <- withStore' c $ \db -> countSndQueueDeliveries db sq''
- atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + pending}
- lift $ resumeMsgDelivery c sq''
- void $ enqueueMessages c cData [oldSq, sq''] SMP.noMsgFlags $ QEND [(oldServer, oldSndId)]
+ resumeSecuredSndDelivery c sq''
+ void $ enqueueMessages c cData [oldSq, sq''] SMP.noMsgFlags $ QEND [qAddress oldSq]
SomeConn _ conn' <- withStore c (`getConn` connId)
cStats <- connectionStats c conn'
notify $ SWITCH QDSnd SPSecured cStats
+ -- R' is already secured; re-enqueue QEND if a crash landed before it, so old (still SSSendingQEND) is removed
+ Nothing ->
+ forM_ (find (\q -> sndSwchStatus q == Just SSSendingQEND) sqs) $ \oldSq ->
+ void $ enqueueMessages c cData [oldSq, sq'] SMP.noMsgFlags $ QEND [qAddress oldSq]
where
ack srv rId srvMsgId =
withStore' c (\db -> getRcvQueue db connId srv rId) >>= \case
@@ -2486,6 +2482,14 @@ submitPendingMsg c sq
atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + 1}
void $ getDeliveryWorker True c sq
+-- starts the worker of a queue whose delivery was held while it was securing: its accumulated deliveries were not
+-- counted in msgDeliveryOp, so add them before the worker starts, as it subtracts one per delivery.
+resumeSecuredSndDelivery :: AgentClient -> SndQueue -> AM ()
+resumeSecuredSndDelivery c sq = do
+ pending <- withStore' c $ \db -> countSndQueueDeliveries db sq
+ atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + pending}
+ lift $ resumeMsgDelivery c sq
+
runSmpQueueMsgDelivery :: AgentClient -> SndQueue -> (Worker, TMVar ()) -> AM ()
runSmpQueueMsgDelivery c@AgentClient {subQ} sq@SndQueue {userId, connId, server, queueMode} (Worker {doWork}, qLock) = do
AgentConfig {messageRetryInterval = ri, messageTimeout, serviceResponseTimeout, helloTimeout, quotaExceededTimeout} <- asks config
From e6621ee2b3a5b258708424430f2a594c1e0bfbb0 Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 21:51:00 +0000
Subject: [PATCH 15/16] simplify
---
src/Simplex/Messaging/Agent.hs | 27 +++++++++------------------
1 file changed, 9 insertions(+), 18 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index c1a2732b5..c64adff8c 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2287,7 +2287,10 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
setSndQueuePrimary db connId sq'
void $ setSndSwitchStatus db oldSq $ Just SSSendingQEND
let sq'' = (sq' :: SndQueue) {status = Active, primary = True, dbReplaceQueueId = Nothing}
- resumeSecuredSndDelivery c sq''
+ -- count R''s deliveries held while securing into msgDeliveryOp before starting its worker (it subtracts one per delivery)
+ pending <- withStore' c $ \db -> countSndQueueDeliveries db sq''
+ atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + pending}
+ lift $ resumeMsgDelivery c sq''
void $ enqueueMessages c cData [oldSq, sq''] SMP.noMsgFlags $ QEND [qAddress oldSq]
SomeConn _ conn' <- withStore c (`getConn` connId)
cStats <- connectionStats c conn'
@@ -2457,12 +2460,10 @@ resumeMsgDelivery :: AgentClient -> SndQueue -> AM' ()
-- hasWork is passed as False to avoid unnecessary write to TMVar:
-- - new worker is always created by "some work to do".
-- - if the worker already exists, there is no need to "push" it again.
-resumeMsgDelivery c sq
- | securingSndQueue sq = pure ()
- | otherwise = void $ getDeliveryWorker False c sq
+resumeMsgDelivery c sq = unless (securingSndQueue sq) $ void $ getDeliveryWorker False c sq
{-# INLINE resumeMsgDelivery #-}
--- a replacement queue holds its worker until it is secured and its confirmation is sent
+-- a replacement queue not yet secured (New, still referencing the queue it replaces)
securingSndQueue :: SndQueue -> Bool
securingSndQueue SndQueue {status, dbReplaceQueueId} = status == New && isJust dbReplaceQueueId
{-# INLINE securingSndQueue #-}
@@ -2476,19 +2477,9 @@ getDeliveryWorker hasWork c sq =
pure (w, retryLock)
submitPendingMsg :: AgentClient -> SndQueue -> AM' ()
-submitPendingMsg c sq
- | securingSndQueue sq = pure ()
- | otherwise = do
- atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + 1}
- void $ getDeliveryWorker True c sq
-
--- starts the worker of a queue whose delivery was held while it was securing: its accumulated deliveries were not
--- counted in msgDeliveryOp, so add them before the worker starts, as it subtracts one per delivery.
-resumeSecuredSndDelivery :: AgentClient -> SndQueue -> AM ()
-resumeSecuredSndDelivery c sq = do
- pending <- withStore' c $ \db -> countSndQueueDeliveries db sq
- atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + pending}
- lift $ resumeMsgDelivery c sq
+submitPendingMsg c sq = unless (securingSndQueue sq) $ do
+ atomically $ modifyTVar' (msgDeliveryOp c) $ \s -> s {opsInProgress = opsInProgress s + 1}
+ void $ getDeliveryWorker True c sq
runSmpQueueMsgDelivery :: AgentClient -> SndQueue -> (Worker, TMVar ()) -> AM ()
runSmpQueueMsgDelivery c@AgentClient {subQ} sq@SndQueue {userId, connId, server, queueMode} (Worker {doWork}, qLock) = do
From 191b5128cc07620cc95a943b0e5062987e18d88d Mon Sep 17 00:00:00 2001
From: "Evgeny @ SimpleX Chat"
<259188159+evgeny-simplex@users.noreply.github.com>
Date: Mon, 17 Aug 2026 22:14:16 +0000
Subject: [PATCH 16/16] refactor
---
src/Simplex/Messaging/Agent.hs | 31 ++++++++++++++--------------
src/Simplex/Messaging/Agent/Store.hs | 4 ++--
2 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/src/Simplex/Messaging/Agent.hs b/src/Simplex/Messaging/Agent.hs
index c64adff8c..63bf61055 100644
--- a/src/Simplex/Messaging/Agent.hs
+++ b/src/Simplex/Messaging/Agent.hs
@@ -2359,8 +2359,11 @@ enqueueMessagesB c reqs = do
enqueueSavedMessageB c $ mapMaybe snd $ rights $ toList reqs'
pure $ fst <$$> reqs'
-isActiveSndQ :: SndQueue -> Bool
-isActiveSndQ SndQueue {status} = status == Secured || status == Active
+-- which tail send queues a fanned-out message is delivered to: active queues, plus a securing replacement (v8), minus a terminating one
+isActiveSndQ :: ConnData -> SndQueue -> Bool
+isActiveSndQ ConnData {connAgentVersion} sq@SndQueue {status, sndSwchStatus} =
+ sndSwchStatus /= Just SSSendingQEND
+ && (status == Secured || status == Active || (connAgentVersion >= rpcAddressSMPAgentVersion && securingSndQueue sq))
{-# INLINE isActiveSndQ #-}
enqueueMessage :: AgentClient -> ConnData -> SndQueue -> MsgFlags -> AMessage -> AM (AgentMsgId, PQEncryption)
@@ -2376,13 +2379,9 @@ enqueueMessageB c reqs = do
mapAccumLM (\ids r -> storeSentMsg db cfg ids r `E.catchAny` \e -> (ids,) <$> handleInternal e) IM.empty reqs
forME reqMids $ \((csqs_, _, _, _), InternalId msgId, pqSecr) -> forM csqs_ $ \(cData, sq :| sqs) -> do
submitPendingMsg c sq
- let sqs' = filter (sndDeliverTo cData) sqs
+ let sqs' = filter (isActiveSndQ cData) sqs
pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId))
where
- sndDeliverTo :: ConnData -> SndQueue -> Bool
- sndDeliverTo ConnData {connAgentVersion} sq@SndQueue {sndSwchStatus} =
- sndSwchStatus /= Just SSSendingQEND
- && (isActiveSndQ sq || (connAgentVersion >= rpcAddressSMPAgentVersion && securingSndQueue sq))
storeSentMsg ::
DB.Connection ->
AgentConfig ->
@@ -2789,9 +2788,9 @@ abortConnectionSwitch' :: AgentClient -> ConnId -> AM ConnectionStats
abortConnectionSwitch' c connId =
withConnLock c connId "abortConnectionSwitch" $
withStore c (`getConn` connId) >>= \case
- SomeConn _ (DuplexConnection cData@ConnData {connAgentVersion} rqs sqs) -> case switchingRQ rqs of
+ SomeConn _ (DuplexConnection cData rqs sqs) -> case switchingRQ rqs of
Just rq
- | canAbortRcvSwitch connAgentVersion rq -> do
+ | canAbortRcvSwitch cData rq -> do
when (ratchetSyncSendProhibited cData) $ throwE $ CMD PROHIBITED "abortConnectionSwitch: send prohibited"
-- multiple queues to which the connections switches were possible when repeating switch was allowed
let (delRqs, keepRqs) = L.partition ((Just (dbQId rq) ==) . dbReplaceQId) rqs
@@ -2820,7 +2819,7 @@ synchronizeRatchet' c connId pqSupport' force = withConnLock c connId "synchroni
AgentConfig {e2eEncryptVRange} <- asks config
g <- asks random
(pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g (maxVersion e2eEncryptVRange) pqSupport'
- enqueueRatchetKeyMsgs c sqs e2eParams
+ enqueueRatchetKeyMsgs c cData' sqs e2eParams
withStore' c $ \db -> do
setConnRatchetSync db connId RSStarted
setRatchetX3dhKeys db connId pks
@@ -3035,9 +3034,9 @@ connectionStats c = \case
subStatus = Nothing
}
rcvQueueInfo :: ConnData -> RcvQueue -> AM RcvQueueInfo
- rcvQueueInfo ConnData {connAgentVersion} rq@RcvQueue {server, status, rcvSwchStatus} = do
+ rcvQueueInfo cData rq@RcvQueue {server, status, rcvSwchStatus} = do
subStatus <- atomically checkQueueSubStatus
- pure $ RcvQueueInfo {rcvServer = server, status, rcvSwitchStatus = rcvSwchStatus, canAbortSwitch = canAbortRcvSwitch connAgentVersion rq, subStatus}
+ pure $ RcvQueueInfo {rcvServer = server, status, rcvSwitchStatus = rcvSwchStatus, canAbortSwitch = canAbortRcvSwitch cData rq, subStatus}
where
checkQueueSubStatus :: STM SubscriptionStatus
checkQueueSubStatus =
@@ -4129,7 +4128,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
sendReplyKey = do
g <- asks random
(pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g e2eVersion pqSupport
- enqueueRatchetKeyMsgs c sqs e2eParams
+ enqueueRatchetKeyMsgs c cData' sqs e2eParams
pure pks
notifyRatchetSyncError = do
let cData'' = cData' {ratchetSyncState = RSRequired} :: ConnData
@@ -4271,10 +4270,10 @@ storeConfirmation c cData@ConnData {connId, pqSupport, connAgentVersion = v} sq
liftIO $ createSndMsg db connId msgData
liftIO $ createSndMsgDelivery db sq internalId
-enqueueRatchetKeyMsgs :: AgentClient -> NonEmpty SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM ()
-enqueueRatchetKeyMsgs c (sq :| sqs) e2eEncryption = do
+enqueueRatchetKeyMsgs :: AgentClient -> ConnData -> NonEmpty SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM ()
+enqueueRatchetKeyMsgs c cData (sq :| sqs) e2eEncryption = do
msgId <- enqueueRatchetKey c sq e2eEncryption
- mapM_ (lift . enqueueSavedMessage c msgId) $ filter isActiveSndQ sqs
+ mapM_ (lift . enqueueSavedMessage c msgId) $ filter (isActiveSndQ cData) sqs
enqueueRatchetKey :: AgentClient -> SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM AgentMsgId
enqueueRatchetKey c sq@SndQueue {connId} e2eEncryption = do
diff --git a/src/Simplex/Messaging/Agent/Store.hs b/src/Simplex/Messaging/Agent/Store.hs
index 1054d1d17..abfae2b59 100644
--- a/src/Simplex/Messaging/Agent/Store.hs
+++ b/src/Simplex/Messaging/Agent/Store.hs
@@ -207,8 +207,8 @@ rcvSMPQueueAddress :: RcvQueue -> SMPQueueAddress
rcvSMPQueueAddress RcvQueue {server, sndId, e2ePrivKey, queueMode} =
SMPQueueAddress server sndId (C.publicKey e2ePrivKey) queueMode
-canAbortRcvSwitch :: VersionSMPA -> RcvQueue -> Bool
-canAbortRcvSwitch connAgentVersion = maybe False canAbort . rcvSwchStatus
+canAbortRcvSwitch :: ConnData -> RcvQueue -> Bool
+canAbortRcvSwitch ConnData {connAgentVersion} = maybe False canAbort . rcvSwchStatus
where
canAbort = \case
RSSwitchStarted -> True