From ea808ff3feac6b326b7a5792859e4c719980e481 Mon Sep 17 00:00:00 2001 From: Philippe Leduc Date: Fri, 18 Sep 2026 19:11:07 +0200 Subject: [PATCH] Fix IPC ownership and recovery races --- ARCHITECTURE.md | 680 ++++++++++++++++++++------------- README.md | 7 +- benchmarks/microbench.cc | 10 +- examples/hello_diagnose.cc | 8 +- examples/hello_zerocopy.cc | 19 +- include/kickmsg/Blackboard.h | 77 ++-- include/kickmsg/Publisher.h | 108 ++++-- include/kickmsg/Region.h | 194 +++------- include/kickmsg/Registry.h | 64 ++-- include/kickmsg/Subscriber.h | 37 +- include/kickmsg/types.h | 227 ++++++----- py_bindings/src/kickmsg_py.cc | 328 +++++----------- python/kickmsg/_native.pyi | 21 +- src/Blackboard.cc | 262 +++++++------ src/Node.cc | 59 ++- src/Publisher.cc | 207 ++++++---- src/Region.cc | 506 ++++++++++++------------ src/Registry.cc | 227 ++++++----- src/Subscriber.cc | 272 ++++--------- src/os/posix/WakeBackends.cc | 22 +- src/os/windows/Futex.cc | 14 +- src/os/windows/WakeBackends.cc | 32 +- src/types.cc | 66 ++-- tests/blackboard_crash_test.cc | 51 ++- tests/crash_test.cc | 16 +- tests/mp_stress_test.cc | 22 +- tests/python/test_zerocopy.py | 4 +- tests/stall_repair_test.cc | 57 +-- tests/stress/big_payload.cc | 48 +-- tests/stress/common.cc | 14 +- tests/stress/gc_recovery.cc | 10 +- tests/stress/live_repair.cc | 10 +- tests/stress/treiber.cc | 8 +- tests/tsan.supp | 36 +- tests/unit/blackboard-t.cc | 97 ++++- tests/unit/node-t.cc | 37 +- tests/unit/publisher-t.cc | 120 +++++- tests/unit/region-t.cc | 568 +++++++++++++++++++-------- tests/unit/registry-t.cc | 661 ++++++++++++++++++++++++++++++-- tests/unit/subscriber-t.cc | 20 +- tests/unit/wait_fd-t.cc | 14 +- 41 files changed, 3147 insertions(+), 2093 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1215cda..deb69f7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -123,20 +123,26 @@ the user's responsibility. The region header is self-describing and forward-compatible. -`version` is 8, and no struct changed to earn it: `Header` and -`SubRingHeader` are byte-identical to version 7. What changed is the -*meaning* of `has_waiter`, which gained `WaiterCarrier(2)` (see Waking on -a descriptor). A version-7 publisher reads any non-zero `has_waiter` as -"parked on a futex", so it would send `futex_wake_all` to a subscriber -waiting on a descriptor and the wake would be lost until its deadline. -The bump exists purely to keep the two interpretations apart, and the -version check is what enforces it. +`version` is 9. `Header` and `SubRingHeader` are unchanged from 8; the +bump covers the entry and slot layouts. `Entry` replaced its +`slot_idx` + `payload_len` pair with the single 64-bit `meta` claim word +(see Entry meta-word encoding), and `SlotHeader` grew from 8 to 16 bytes +to carry `payload_len`. Both drive ring-stride and slot-stride math, so +a version-8 peer would misread every entry and slot. Version 8 itself +changed no struct: it gave `has_waiter` the `WaiterCarrier(2)` value, +which a version-7 publisher would misread as a futex waiter. + +A version mismatch is fatal, never negotiated: `open()`, +`attach_open()` and `create_or_open()` throw `VersionMismatch` at once +(`create_or_open()` does not wait for the region to become "ready"). +One namespace cannot mix kickmsg builds. Upgrading means stopping the +old processes and unlinking their regions and registry. ``` Header (at offset 0) ┌───────────────────────────────────────────────────────────┐ │ magic (atomic) 0x4B49434B4D534721 ("KICKMSG!") │ -│ version 8 │ +│ version 9 │ │ channel_type PubSub | Broadcast │ │ total_size total mmap size in bytes │ │ sub_rings_offset byte offset to first subscriber ring │ @@ -322,27 +328,26 @@ pointer (a `VERSION` bump). ## Subscriber Ring Each ring is a fixed-size circular buffer of `Entry` records. An entry -contains a sequence number, slot index, and payload length -- all atomic. - -``` -Ring[0] -┌──────────────────────────────────────────────────────────┐ -│ state: Live in_flight: 0 write_pos: AtomicU64 = 42 │ -│ │ -│ entries[0..7]: │ -│ ┌─────┬───────────┬──────────┬─────────────┐ │ -│ │ idx │ sequence │ slot_idx │ payload_len │ │ -│ ├─────┼───────────┼──────────┼─────────────┤ │ -│ │ 0 │ 37 │ 5 │ 128 │ │ -│ │ 1 │ 38 │ 12 │ 256 │ │ -│ │ 2 │ 39 │ 0 │ 64 │ │ -│ │ 3 │ 40 │ 7 │ 512 │ │ -│ │ 4 │ 41 │ 2 │ 1024 │ ◄── latest │ -│ │ 5 │ 42 │ 11 │ 128 │ ◄── newest │ -│ │ 6 │ 35 │ 9 │ 256 │ ◄── stale │ -│ │ 7 │ 36 │ 1 │ 64 │ ◄── stale │ -│ └─────┴───────────┴──────────┴─────────────┘ │ -└──────────────────────────────────────────────────────────┘ +is two atomic 64-bit words: a `sequence` (commit barrier and seqlock) +and a `meta` word carrying the entry's claim on a pool slot, tagged +with the position that wrote it. The payload length lives in the slot +itself, not the entry -- see the meta-word encoding below. + +``` ++----------------------------------------------------------------+ +| Ring[0]: state=Live, in_flight=0, write_pos=42 | +| | +| idx sequence meta (tag:40 | slot+1:24) | +| --- -------- ----------------------- | +| 0 37 37 | 6 | +| 1 38 38 | 13 | +| 2 39 39 | 1 | +| 3 40 40 | 8 | +| 4 41 41 | 3 | +| 5 42 42 | 12 <-- newest | +| 6 35 35 | 10 <-- stale | +| 7 36 36 | 2 <-- stale | ++----------------------------------------------------------------+ ``` - **Capacity** must be a power of 2 (index masking: `pos & (cap - 1)`). @@ -491,9 +496,9 @@ bit 62 = repair bit) and a 62-bit value: ``` [tag:2 | value:62] - 00 committed word is pos + 1; slot_idx / payload_len valid + 00 committed word is pos + 1; the meta word names the payload 01 skip marker word carries pos + 1; committed but EMPTY -- - metadata untrustworthy by design + no payload ever arrived at this position 10 locked a publisher is mid-commit at pos (position-tagged lock, unique per position) 11 repair-locked a repairer owns the entry mid-steal @@ -505,12 +510,45 @@ across a full timeout window proves a single holder spanned it: the staleness proof every steal relies on. Stolen entries are committed as **skip markers**, not plain -sequences. The stolen-from publisher's late metadata stores can land -at any time after the steal, so nothing may ever trust `slot_idx` / -`payload_len` under tag 01: subscribers count a skip-marked position -as one lost message without reading its metadata, and the eviction -path never releases a slot through a skip-tagged entry. The victim's -stores are harmless by construction, not by timing. +sequences: no payload was ever committed at that position, so a +subscriber counts a skip-marked position as one lost message. A steal +leaves the meta word alone; the victim cannot damage it, because the +victim's own write is a compare-exchange guarded by the ordering rule +below. + +### Entry meta-word encoding + +`Entry::meta` stores the position tag and slot claim in one atomic word: + +``` +[tag:40 | slot + 1:24] + tag low 40 bits of pos + 1 + slot pool index + 1; zero means no claim +``` + +Each claim owns one slot reference. Replacing or clearing the claim transfers +the duty to release that reference. A publisher may replace only an older +position's claim, using CAS. A delayed publisher therefore cannot overwrite +a newer entry. Tag comparisons require positions less than 2^39 apart. + +Repair keeps the claim even when it changes the sequence to a skip marker. +The next publisher or drainer releases it. A zeroed entry has no claim, and +the 24-bit slot field limits `pool_size` to `MAX_POOL_SIZE`. + +### Validated geometry snapshot + +After creation or validation, each handle copies the header's offsets, +strides, counts, and size limits into local `Geometry`. Pointer arithmetic +uses this copy because peers can still modify the shared header. This +covers `SharedRegion`'s own diagnostic and recovery walks (`diagnose`, +`stats`, `info`, the repair and reclaim primitives) as well as publishers +and subscribers. `info()` re-bounds the live `creator_name_len` against +the validated tail. `sub_ring_at`, `slot_at` and `treiber_pop` take a +`Geometry` (or an explicit pool base and stride), never a `Header`, so the +shared fields have no path into pointer arithmetic. `SampleView` stores +the slot pointer already resolved by its subscriber. + +Counters, `free_top`, and ring/entry atomics remain shared. ### Subscriber join and visibility window @@ -574,8 +612,17 @@ instant it attaches. ## Publish Flow -Any publisher can call `send()` or `allocate()` + `publish()`. Multiple -publishers may race concurrently on the same channel. +Any publisher can call `send()`, or reserve a slot with `allocate()`, +write the payload into it, and `publish(len)`. Multiple publishers may race +concurrently on the same channel. + +`allocate()` returns an `AllocatedSlot` that owns the reservation: +destroying it unpublished returns the slot to the pool, and it refuses +to publish once a later `allocate()` has superseded it. Moving the +`Publisher` invalidates its handles too, and returns the reservation to +the pool at once, since no handle can publish it any more. A caller fills +the slot in place through `data()` (up to `max_size()` bytes), or copies +bytes it already has with `write()`, then commits with `publish(len)`. ``` Publisher @@ -618,8 +665,8 @@ Publisher | | (default 10 ms). | |-- Committed: Proceed to the lock CAS. The previous | | occupant's slot is released AFTER the - | | lock succeeds, from a post-lock read - | | of slot_idx (see below). + | | claim take-over succeeds, from the + | | word it replaced (see below). | '-- Timeout (crash or Records whether ONE position-tagged | stall): lock value spanned the whole wait — | the proof self-repair needs before @@ -658,9 +705,8 @@ Publisher | | succeeds without timeout. The CAS | | backs off if a live writer wins. | | Do NOT release any slot -- the entry - | | was never ours. The stale occupant's - | | unreleased ref is a bounded leak - | | (1 per steal), recoverable by GC. + | | was never ours, and its claim word + | | still names whatever it referenced. | | abandon_delivery(): Count the drop, then release | | dropped_count++ admission -- this ring's in_flight | | state_flight.fetch_sub was incremented at CAS admission and @@ -675,43 +721,36 @@ Publisher | | full timeout. | | excess++, continue | | - | | Lock success -- release previous occupant: - | | If NOT prev_was_skip After locking, we own the entry; the - | | and pos >= capacity: lock-CAS acquire pairs with the - | | read e.slot_idx previous committer's release, so even - | | if in bounds: a commit that landed after our wait - | | release_slot(it) timed out is seen and released here. - | | INVALID (drain marker) fails the - | | bound check: drain_unconsumed already - | | released this ring's reference -- - | | releasing again would double- - | | decrement. A SKIP predecessor is - | | never released: its metadata is - | | untrustworthy by design; the stolen - | | entry's ref is left for GC. Below - | | one wrap there is no predecessor - | | (a zero-initialized slot_idx would - | | read as valid slot 0). + | | Early-out: + | | reload entry.sequence Avoid work if the lock was stolen. + | | if not ours: drop The metadata CAS below guards late writes. | | - | | Theft guard: If sequence != seq_lock(pos), a - | | reload entry.sequence repairer proved our lock stale (we - | | if not ours: drop stalled past commit_timeout) and owns - | | the entry. Storing data now would - | | tear the repaired entry. + | | Take the claim over: + | | my = meta_pack(pos, 3) Compare-exchange, looping while the + | | while meta_precedes observed word precedes our position. + | | (old, pos): Failure means a newer publisher owns + | | CAS meta old -> my the entry: our slot ref is still ours, + | | if not taken: drop, so we count it in `excess` and drop. + | | excess++, continue Only older position tags may be replaced. | | - | | Write entry fields (relaxed, safe because we hold the lock): - | | entry.slot_idx = 3 - | | entry.payload_len = 128 + | | Release the previous occupant: + | | biased = slot of old The word we replaced hands us the + | | if biased != 0: previous occupant's reference. A + | | release_slot(biased-1) drained entry and a never-written one + | | both read as "no slot", so neither + | | needs a special case, and a skip + | | predecessor needs none either: the + | | tag says whose metadata it is. | | | | Phase 2 - CAS commit: | ' CAS entry.sequence CAS, not a blind store: fails only if - | seq_lock(pos) -> 43 a repairer stole the lock after the - | theft guard -- the entry is then a - | committed skip marker, and our late - | slot_idx/payload_len stores landed - | under tag 01, which nothing ever - | trusts: permanently harmless. We - | record a drop (abandon_delivery). + | seq_lock(pos) -> 43 a repairer stole the lock after we + | took the claim over. No `excess` on + | that path -- the entry now holds our + | reference and the next publisher here + | releases it; counting it would + | double-free. We record a drop + | (abandon_delivery). | Release on success: subscribers and | future publishers at this position | see all preceding stores. @@ -747,12 +786,18 @@ Publisher ### Why a two-phase commit? Without the lock, two publishers that CAS `write_pos` to adjacent -positions could interleave their `slot_idx` and `sequence` stores on +positions could interleave their claim and `sequence` stores on overlapping entries (after a ring wrap). The position-tagged lock -prevents this: only one publisher at a time can write an entry's data -fields, and the final CAS commit of the real sequence makes the entry +prevents this: only one publisher at a time drives an entry to a +commit, and the final CAS commit of the real sequence makes the entry visible atomically — or fails, detectably, if the lock was stolen. +The lock alone is not enough, because a publisher can stall while +holding it and have it stolen. That is why the claim word is taken +over by compare-exchange under the position-ordering rule rather than +stored: the lock orders the common case, the claim protocol is what +holds when the lock has been taken away. + Subscribers treat any locked value the same as "not yet committed" and return `nullopt`, so the lock is invisible to them except as a brief delay. @@ -834,51 +879,63 @@ Subscriber X (read_pos_ = 41, local) above consumes. | v -3. Read slot_idx and payload_len from the entry. +3. Read the entry's claim word (acquire). Its tag must name this + position, else count the entry as lost; then take the slot index. + The acquire orders a newer publisher's sequence lock before the + seq2 recheck below, and the tag rejects a claim left by any other + position or by corrupt peer bytes. | - |---- Both modes: refcount pin --------------------------------| - | | - | Both try_receive() and try_receive_view() pin the slot | - | via CAS before reading data. This prevents the publisher | - | from freeing the slot while the subscriber reads it. | - | | - | CAS Slot.refcount: rc -> rc+1 Pin the slot (only if | - | (retry while rc > 0) rc > 0, i.e. slot alive) | - | (if rc == 0: slot freed between seq1 read and | - | between seq1 and now, now. Count as lost.) | - | skip as lost message) | - | | - | seq2 = entry.sequence (acquire) Seqlock validation: if | - | seq2 == seq1? the entry was overwritten | - | -> yes: pin valid after we pinned, the | - | -> no: undo pin, count lost slot_idx may be stale. | - | | - |---- Copy mode: try_receive() --------------------------------| - | | - | memcpy Slot[slot_idx].data -> local recv_buf_ | - | Unpin: refcount.fetch_sub(1) | - | If refcount -> 0: treiber_push(slot) | - | read_pos_++ | - | return SampleRef { recv_buf_, payload_len } | - | | - | Note: SampleRef points into recv_buf_ (subscriber-local | - | buffer). Calling try_receive() again overwrites it. | - | Copy data from SampleRef before the next call. | - | | - |---- Zero-copy mode: try_receive_view() ----------------------| - | | - | read_pos_++ | - | return SampleView { Slot, payload_len } | - | | | - | '--> ~SampleView(): | - | refcount.fetch_sub(1) | - | if refcount -> 0: treiber_push(slot) | - | | - | SampleView holds a direct pointer into shared memory. | - | The refcount pin keeps the slot alive until the view | - | is destroyed. Best for large payloads where memcpy | - | would dominate latency. | - '--------------------------------------------------------------' + |---- Both modes: refcount pin-----------------------------------| + | | + | Both try_receive() and try_receive_view() pin the slot | + | via CAS before reading data. This prevents the publisher | + | from freeing the slot while the subscriber reads it. | + | | + | CAS Slot.refcount: rc -> rc+1 Pin the slot (only if | + | (retry while rc > 0) rc > 0, i.e. slot alive) | + | (if rc == 0: slot freed between seq1 read and | + | between seq1 and now, now. Count as lost.) | + | skip as lost message) | + | | + | seq2 = entry.sequence (acquire) Seqlock validation: if | + | seq2 == seq1? the entry was overwritten | + | -> yes: pin valid after we pinned, the | + | -> no: undo pin, count lost claim may be stale. | + | | + | payload_len = Slot.payload_len Read only once the seqlock | + | bounds-check it, else lost has confirmed the entry | + | still names this slot: the | + | length lives in the slot, | + | so before that point it | + | could belong to whoever | + | recycled it. | + | | + |---- Copy mode: try_receive()-----------------------------------| + | | + | memcpy Slot[slot_idx].data -> local recv_buf_ | + | Unpin: refcount.fetch_sub(1) | + | If refcount -> 0: treiber_push(slot) | + | read_pos_++ | + | return SampleRef { recv_buf_, payload_len } | + | | + | Note: SampleRef points into recv_buf_ (subscriber-local | + | buffer). Calling try_receive() again overwrites it. | + | Copy data from SampleRef before the next call. | + | | + |---- Zero-copy mode: try_receive_view()-------------------------| + | | + | read_pos_++ | + | return SampleView { Slot, payload_len } | + | | | + | '--> ~SampleView(): | + | refcount.fetch_sub(1) | + | if refcount -> 0: treiber_push(slot) | + | | + | SampleView holds a direct pointer into shared memory. | + | The refcount pin keeps the slot alive until the view | + | is destroyed. Best for large payloads where memcpy | + | would dominate latency. | + '----------------------------------------------------------------' ``` @@ -1024,14 +1081,10 @@ On timeout, the publisher: marker (`seq_skip(pos)`: tag 01, word carrying pos + 1) so the next publisher at this position succeeds without paying the timeout. The CAS backs off if a live writer commits first. A stolen-from - publisher that was merely slow detects the theft at its own theft - guard or CAS commit and records a drop instead of corrupting the - entry. Its late metadata stores -- landing at any point after the - steal -- fall under the skip tag, which nothing ever trusts: - subscribers count the position lost without reading - slot_idx/payload_len, and the eviction path never releases a slot - through a skip-tagged entry. The victim's stores are permanently - harmless by construction, not by timing. + publisher that was merely slow records a drop instead of corrupting + the entry: its claim take-over is refused by the position-ordering + rule no matter how late it resumes, and its CAS commit then fails. + The victim's write is harmless by construction, not by timing. 2. Drops delivery for this ring and moves to the next subscriber ring. Every drop path ends in `abandon_delivery()`, which mirrors the success path's Dekker wake (seq_cst fence, `has_waiter` check, @@ -1101,12 +1154,13 @@ and full-window drain: wp = ring.write_pos — now guaranteed final oldest = max(0, wp - capacity) for each entry in [max(oldest, start_pos), wp): - if sequence == pos + 1: — committed and not evicted + if the claim word names a slot: + CAS the claim to "no slot" -- makes this idempotent; a + second pass finds nothing slot.refcount-- if refcount == 0: treiber_push(slot) - entry.slot_idx = INVALID_SLOT (seq_cst) else: - skip (evicted, uncommitted, or locked — falls into Class B) + skip (already drained, or never claimed anything) 4. state = Free (release) — ring available for a new subscriber (timeout path: CAS that preserves the crashed publisher's in_flight) @@ -1134,9 +1188,11 @@ must also be released. `start_pos` is the `write_pos` captured at subscriber construction, ensuring a reused ring slot doesn't double-release entries from a previous subscriber. -After releasing each entry's slot, drain sets `entry.slot_idx` to -`INVALID_SLOT` to prevent a future publisher's eviction from -double-decrementing the refcount. +Drain keys off the **claim word, not the sequence**: the claim is what +owns a reference, and an entry a repairer turned into a skip marker +still holds one. Keying off `sequence == pos + 1` would strand those. +Clearing the claim as part of the release prevents a future +publisher's eviction from double-decrementing the refcount. For `try_receive_view()`, a live `SampleView` holds an extra pin (rc=2: ring ref + view pin). The drain releases the ring ref (rc→1); @@ -1150,11 +1206,14 @@ Only Class B can leak slots. Each publisher crash leaks at most - The slot the crashed publisher allocated (refcount stuck > 0 because the remaining rings were never visited for inline release; or refcount 0 and off the free stack, for a crash before the pre-set) -- The slot referenced by the stolen ring entry, if one existed at the - wrapped position: the steal deliberately never releases it (the - stalled holder may already have batch-released this ring's ref, and - metadata under a skip tag is untrustworthy), so its ring ref leaks - until GC -- at most one per steal + +A steal does **not** add to this budget. It leaves the entry's claim +word intact, so the slot it names stays reachable: the next publisher +at that index releases it on the normal take-over path, and a drain +releases it otherwise. This matters beyond the budget -- a steal can +happen without any process crashing (a publisher merely descheduled +past `commit_timeout`), and clearing the claim there would leak a slot +in an otherwise healthy system. With a typical pool of 256+ slots, the system can tolerate dozens of crashes before running low. Class B leaks can be recovered by the @@ -1193,23 +1252,22 @@ sleep -- the same position-tagged lock value at both instants proves its unique holder exceeded the commit budget. Every steal takes ownership with a CAS to `seq_repair(pos)` before touching the entry, then commits the entry as a skip marker (`seq_skip(pos)`: tag 01, -word carrying pos + 1). The `INVALID_SLOT` / zero-length stores made -under the repair lock are diagnostics only -- nothing ever trusts -metadata under a skip tag. A live publisher that commits first wins -the CAS race and the repairer backs off; a stolen-from publisher that -was merely slow detects the theft (theft guard / CAS commit) and -records a drop. Subscribers count a skip-marked position as one lost -message without reading its metadata; evictions never release a slot -through one. - -Residuals, both bounded: -- The victim's late metadata stores -- landing at any point after the - steal -- fall under the skip tag and are ignored by construction, - so they are harmless; there is no torn-read window. -- The stolen entry's previous slot reference is never released by the - repair (the stalled holder may already have batch-released this - ring's ref; releasing again could double-free). At most one slot - ref leaks per steal, recovered by `reclaim_orphaned_slots()`. +word carrying pos + 1). The claim word is left exactly as it was: it +is the only record of which slot the entry still references. A live +publisher that commits first wins the CAS race and the repairer backs +off; a stolen-from publisher that was merely slow records a drop. +Subscribers count a skip-marked position as one lost message: no +payload was ever committed there. + +Residuals: +- The victim's late write is refused, whenever it resumes: taking the + claim over requires replacing metadata from an older position, and + by then the entry carries its successor's. There is no window in + which a resumed publisher can reach a newer entry. +- No slot reference leaks. The stolen entry keeps its claim, so the + next publisher at that index releases it on the normal take-over + path -- which matters because a steal needs no crash, only a + publisher descheduled past `commit_timeout`. ``` repair_locked_entries(region): @@ -1226,25 +1284,31 @@ repair_locked_entries(region): if entry.sequence == seq: // same holder spanned it steal(entry, pos, seq) -steal(entry, pos, observed): // entry_steal_and_clear +steal(entry, pos, observed): // entry_steal_and_skip CAS entry.sequence: observed -> seq_repair(pos) // live writer wins: back off - entry.slot_idx = INVALID_SLOT // diagnostics only - entry.payload_len = 0 + // entry.meta is deliberately untouched: it is the only record of + // which slot this entry still references, and the victim cannot + // damage it (its own write is a guarded compare-exchange). entry.sequence = seq_skip(pos) // release: committed, empty ``` **`reclaim_orphaned_slots()`** -- requires full quiescence. -Builds the set of slot indices referenced by plain-committed ring -entries (locked and skip-marked entries are excluded -- skip metadata -is untrustworthy by design), walks the free stack to record -membership (exact under the quiescence contract, bounded by -`pool_size` against corrupt `next_free` cycles), then reclaims every -slot that is neither referenced nor on the stack -- regardless of -refcount. The membership walk is what recovers rc == 0 orphans (a -publisher crash between `treiber_pop` and the refcount pre-set, or a -reclaimer killed between its refcount store and its push) that a -refcount-only scan never could. +Counts, per slot, the ring entries whose claim names it -- over every +entry of every ring, whatever the sequence word or write_pos says, since +a locked or skip-marked entry still owns the slot its claim names and a +claim can outlive the tenancy that wrote it. It then walks the free +stack to record membership (exact under the quiescence contract, bounded +by `pool_size` against corrupt `next_free` cycles). Every off-stack slot +with no claim is reclaimed regardless of refcount; every off-stack slot +with claims gets its refcount reset to its claim count. Each claim owns +exactly one reference, so under quiescence that count is the correct +refcount: a leaked extra reference would otherwise pin the slot forever, +and a missing one would free it while an entry still names it. The +membership walk is what recovers rc == 0 orphans (a publisher crash +between `treiber_pop` and the refcount pre-set, or a reclaimer killed +between its refcount store and its push) that a refcount-only scan +never could. NOT safe under live traffic. Requires: - All publishers quiesced (a publisher between refcount pre-set and ring push has rc > 0 but no ring entry yet; one between treiber_pop @@ -1255,19 +1319,22 @@ NOT safe under live traffic. Requires: ``` reclaim_orphaned_slots(region): - referenced = {} + claims[0..pool_size) = 0 for each ring i in [0, max_subs): - for pos in [oldest_live, write_pos): - seq = entries[pos].sequence - if seq is plain-committed (tag 00) and seq >= pos + 1: - referenced.insert(entries[pos].slot_idx) + for idx in [0, sub_ring_capacity): + claim = entries[idx].meta + if claim names a slot: + claims[slot named by claim] += 1 on_stack = {} // exact under quiescence for idx in chain from free_top, bounded by pool_size: on_stack.insert(idx) for idx in [0, pool_size): - if idx not in referenced and idx not in on_stack: + if idx in on_stack: continue + if claims[idx] > 0: + slot[idx].refcount = claims[idx] + else: slot[idx].refcount = 0 treiber_push(free_top, slot[idx], idx) ``` @@ -1369,10 +1436,10 @@ deliberate post-crash action. For a dead *subscriber* prefer `reclaim_dead_rings()`, which is liveness-checked and cannot underflow `in_flight` against a slow publisher. -**`reclaim_orphaned_slots()`** -- walks all rings to build a -referenced-slot set and the free stack for membership, then frees any -slot that is neither referenced nor on the stack, regardless of -refcount. NOT safe under live traffic -- requires all publishers +**`reclaim_orphaned_slots()`** -- counts ring-entry claims per slot and +walks the free stack for membership, then frees any off-stack slot with +no claim and resets every other off-stack slot's refcount to its claim +count. NOT safe under live traffic -- requires all publishers quiesced and no outstanding `SampleView` objects. ### Recommended recovery sequence @@ -1520,6 +1587,13 @@ Time kickmsg/os/ clock_nanosleep nanosleep Que The ABI has been stable since macOS 10.12 and is used internally by libc++ and libdispatch, but Apple has not published a formal stability guarantee. +**Windows limitation.** `WakeByAddressAll` wakes only the calling process. +A blocking `receive()` or Blackboard wait may sleep until timeout when the +writer is in another process. Unread messages can overflow the ring during +that wait. Use a timeout within the ring's buffering budget, or poll. +Cross-process notification requires a different primitive and Windows +multi-process testing. + The core engine (`types.h`, `Region.h`, `Publisher.h`, `Subscriber.h`, `Node.h`) uses only `std::atomic` and these three abstractions -- no platform `#ifdef` leaks into the messaging logic. @@ -1631,7 +1705,8 @@ between the writer and a concurrent new-tenant claim never involves plain non-atomic accesses on the same bytes: - `state` — atomic `Free` / `Claiming` / `Active` / `Reclaiming` -- `generation` — atomic counter bumped on every claim and release (seqlock) +- `generation` -- atomic version; even means settled, odd means a writer + or sweeper holds the row - `pid` — atomic; OS process id of the owner - `pid_starttime` — atomic; opaque OS-specific process start time - `channel_type` — atomic; PubSub / Broadcast @@ -1648,6 +1723,26 @@ A `Node` lazily opens-or-creates the registry on its first (`Free → Claiming`), writes the fields, then release-stores `Active`. The `Node`'s destructor deregisters every slot it claimed. +### Seqlock parity + +Even generations are settled; odd generations mark a write or recovery hold. +Registration makes the generation odd before writing fields, publishes +`Active`, then settles the generation. Snapshots accept a row only if it is +Active and its generation stays even and unchanged across the copy. + +Retirement blocks readers and new registrants before clearing identity: + +``` +CAS Active -> Reclaiming +set generation odd +clear pid and pid_starttime +settle generation to even +CAS Reclaiming -> Free +``` + +`Free` is published last so the next registrant cannot overlap these writes. +Sweeps skip Reclaiming and odd generations, even after a crash. + The key property is **cross-platform parity**: Linux `/dev/shm` is filesystem-visible, but macOS and Windows are not — we can't use `readdir` to list topics there. Routing discovery through a regular @@ -1657,29 +1752,43 @@ all three targets. ### State machine ``` -Free (0) ── CAS ──► Claiming (1) ── release-store ──► Active (2) - ▲ │ - │ │ - ├────────── deregister: store-release ◄──────────────┤ - │ │ - │ sweep_stale: │ - └── CAS(Reclaiming → Free) ◄── CAS(Active | Claiming → Reclaiming) -``` - -Snapshots acquire-load `state` per slot; only `Active` entries are -returned. The `Claiming` state is the publication fence for the -field bytes — a reader observing `Active` is guaranteed to see all -the field writes that happened-before the release-store. - -`Reclaiming` is the exclusive lock held by `sweep_stale` while it -verifies the dead-pid condition and finalizes the slot to `Free`. -It blocks concurrent registrants (they need `Free → Claiming`) and -prevents ABA on the state CAS: without it, a full `dereg + register` -cycle on another CPU could restore the slot to `Active` between the -sweeper's pid check and its CAS, causing the sweeper to stomp a live -tenant's registration. `sweep_stale` releases `Reclaiming` back to -the pre-CAS value if the re-verified pid differs from what it -observed (ABA detected), so the live tenant is restored. +Registration: + +------+ +----------+ +--------+ + | Free | --CAS-> | Claiming | --release-> | Active | + +------+ +----------+ +--------+ + +Retirement: + +--------+ +------------+ +------+ + | Active | --CAS-> | Reclaiming | --CAS-----> | Free | + +--------+ +------------+ +------+ + +Recovery (dead owner, stable even generation): + +--------------------+ + | Active or Claiming | + +--------------------+ + | + | CAS generation: even -> odd + v + +--------------------+ + | Exclusive hold | + +--------------------+ + | + | store state = Reclaiming + v + +------------+ +------+ + | Reclaiming | --CAS-----> | Free | + +------------+ +------+ +``` + +Snapshots acquire-load Active and check for a stable even generation. +Registration's release-store of Active publishes the preceding field writes. + +`sweep_stale` validates the dead owner's identity under a settled +generation, then acquires that exact generation with an even-to-odd CAS. +A changed generation makes acquisition fail without modifying the row. +The odd generation excludes other sweepers before `Reclaiming` is +stored. Reclamation clears the identity, settles the generation, and +publishes `Free` last; there is no rollback to `Active`. ### Role upgrade @@ -1691,26 +1800,14 @@ connect time only — zero hot-path cost. ### Liveness -The registry does not track heartbeats. A crashed process that -never ran its `Node` destructor leaves its entries stuck at -`Active` (or `Claiming`, if it died mid-register) until reclaimed. -Two recovery paths: - -- **Query-time filter**: diagnostic tools probe each entry's pid via - `process_exists()` and hide dead entries from the user without - touching the registry. Non-invasive; safe under live traffic. -- **`Registry::sweep_stale()`**: CAS-resets any `Active` or `Claiming` - slot whose `pid` is dead. Opt-in cleanup for an operator or - supervisor sweep; also called opportunistically from - `register_participant` when the registry is full, so long-running - deployments don't silently drop new registrations as crashed-process - residue accumulates. - -The `Claiming` reclaim branch skips slots where `pid == 0`: that state -is the brief window between the `Free→Claiming` CAS and the -registrant's first field store. Claiming the slot in that window -would stomp a live registrant. Cost: an early crash (before the pid -store) leaks one slot until the region is unlinked. +The registry does not track heartbeats. Diagnostic tools can filter dead +PIDs without changing shared memory. `Registry::sweep_stale()` reclaims dead +owners only when their row has a settled even generation. Registration calls +it when the registry is full. + +Rows with PID zero, odd generations, or Reclaiming state are skipped: their +holder may still be writing. A crash in these states can strand a slot until +the registry is replaced. **PID-reuse mitigation.** `pid_starttime` is captured at register time: `/proc//stat` on Linux, `sysctl(KERN_PROC_PID)` on Darwin, @@ -1731,26 +1828,44 @@ diagnostic nicety, not a correctness dependency. ### Implicit invariants -Load-bearing assumptions for anyone editing the registry: +Registry invariants: - **Field order is ABI.** `sizeof(ParticipantEntry) == 512` and `offsetof(…, _padding) == 368` are statically asserted; any field reorder or resize must update the padding and bump `registry::VERSION`. + So must a protocol change on an unchanged layout: version 4 kept every + offset but made the generation parity-based (odd = held), which a + version-3 writer would violate. +- **A version mismatch is fatal.** Opening a registry stamped with + another `registry::VERSION` throws `VersionMismatch`; there is no + migration and no fallback. The shm name is unversioned on purpose, so + a stale registry left by an older build fails loudly instead of + splitting discovery. Recovery: stop the old processes, then + `Registry::unlink(namespace)`. - **State publication fence.** `state.store(Active, release)` in `register_participant` is the one fence that publishes all field writes that preceded it. `pid` has its own earlier release-store so `sweep_stale` can acquire-load it while state is still `Claiming` (before the Active fence). Any new field that needs to be visible during `Claiming` must use its own release/acquire pair. -- **Generation bump on every mutation.** `generation` is bumped by - `register_participant` *and* by `deregister` *and* by - `sweep_stale`'s reclaim path. A snapshot's seqlock recheck detects - only mutations that bump gen — adding a future write-path that - modifies fields without bumping gen will cause torn reads. -- **`touch_registry` must never throw.** `Node::advertise` and friends - treat registration as best-effort. A failure is logged once per - `Node` (latched via `registry_disabled_`) and subsequent calls - become no-ops. Don't change this contract without also changing +- **Bracket metadata writes with generation changes.** Set an odd generation + before changing fields and settle it to even after writing them. Publish + Active before settling; publish Free after settling. +- **Clear identity before reuse.** Zero pid and start time before publishing + Free, so a new claim cannot be mistaken for its previous owner. +- **Acquire the validated generation.** Read PID and start time under one + stable even generation, check owner death, then CAS that generation to odd. + A stale version fails without touching state; an odd version is already held. +- **Leave held rows alone.** Sweeps skip odd generations and Reclaiming rows. + Their holders may still write metadata, even after clearing PID. A crash + during such a hold can strand one slot. Recovering it requires a separate + ownership protocol or confirmed quiescence. +- **`touch_registry` throws only `VersionMismatch`.** `Node::advertise` + and friends treat registration as best-effort: any other failure is + logged once per `Node` (latched via `registry_disabled_`) and + subsequent calls become no-ops. A version mismatch propagates out of + `advertise` / `subscribe` / etc., because the namespace mixes builds + and cannot work. Don't change this contract without also changing `Node::advertise`'s error handling. - **Role upgrade has a brief visibility gap.** `touch_registry` upgrades Publisher/Subscriber → Both via `deregister` + re-register @@ -1798,6 +1913,13 @@ It is a separate shared-memory object with its own `MAGIC` and its own `blackboard::VERSION`, independent of the channel ABI in `types.h`. Like the registry, it persists beyond any single process. +`blackboard::VERSION` is 2. The layout is unchanged from 1, but the +access protocol is not: payload and key bytes are now accessed as +relaxed atomics (see Read protocol). The race fix only holds +if every peer follows it, and a version-1 peer's plain `memcpy` would +race it again. A mismatch throws `VersionMismatch`, the same fatal +policy as channels and the registry. + ### Layout One region per board at `/{namespace}_bb_{name}`, composed through @@ -1815,10 +1937,10 @@ BlackboardHeader 192 B -- magic, version, capacity, max_value_size, BlackboardEntry[capacity] 384 B each line 0: state, publish, tenancy, owner_pid, owner_starttime, key_hash, declared_at_ns - then: key[128], owner_node[64], padding + then: key (16 atomic u64 = 128 B), owner_node[64], padding value cells capacity * CELLS_PER_KEY cells of value_stride bytes each: BlackboardCell{updated_at_ns, value_len} - followed by the payload + followed by the payload as atomic u64 words ``` `entries_offset`, `values_offset` and the cell stride are **derived, @@ -1829,6 +1951,15 @@ carries only what cannot be computed. The stride is makes every cell cache-line aligned and guarantees no two keys share a line. +The two stored fields, `capacity` and `max_value_size`, are read **once** +at open into a local `blackboard::Geometry` (capacity, value limit, +stride, `values_offset`), and only that copy is validated and kept. The +board and every `Writer` and `Reader` it hands out carry the copy; walks, +`bb_cell_at`, value-size limits and the lock-recovery sweep never +re-read the header, which a peer can rewrite at any time after open. +The creator builds its copy from its own config. This mirrors the +channel's `Geometry` snapshot. + Storing the *configured* `max_value_size` rather than the padded stride matters twice over. It keeps the alignment slack from being handed out as extra payload capacity -- a board configured for 128 B must not @@ -1973,14 +2104,40 @@ refcount-pinned slot there is nothing safe to point at. Pinning would reintroduce the failure mode this design exists to remove -- a crashed reader blocking a writer. -The overtake window is a genuine data race on the payload bytes in the -C11 model: the reader's re-check *detects and discards* the copy, but the -bytes really are touched concurrently. ThreadSanitizer is right to see -it, and `tests/tsan.supp` carries a narrow, documented suppression scoped -to `bb_copy_payload()` -- a `noinline` helper that exists so the -suppression names one frame and leaves `write()` and `read()` themselves -checked. The blackboard stress scenario asserts -that no torn value ever escapes, over hundreds of thousands of reads. +In the overtake window a reader's copy really does overlap a writer's. +Discarding the copy afterwards is not enough in the C++ model: plain +overlapping accesses are a data race, undefined behavior even if the +result is thrown away. So a payload is stored as an array of +`std::atomic` words following its `BlackboardCell` +(`bb_cell_words()`), and both sides move it only as **whole relaxed +words** (`store_words()` / `load_words()`). The writer zero-pads a +partial last word. The cell always has room for it, because the stride +rounds `sizeof(BlackboardCell) + max_value_size` up to a cache line. +The reader copies only `len` bytes out of its last word. Every +concurrent access to a payload word is therefore an atomic access of +the same object type and size. Plain `memcpy` would race, byte atomics +mixed with word atomics would be undefined, and `atomic_ref` over byte +storage would not name a real `uint64_t` object. Being plain +`std::atomic`, this needs no toolchain-specific code. + +Ordering is unchanged, and this is what makes it sound. The writer does: +odd `publish` store, release fence, relaxed payload stores, release fence, +even `publish` store. The reader does: acquire load of `publish`, relaxed +payload loads, acquire fence, reload of `publish`. If any payload load +observes a store from an overtaking write, the reader's acquire fence +synchronizes with that write's first release fence. Its later reload of +`publish` then sees at least that write's odd value, and the overtake +check discards the copy. ThreadSanitizer confirms the accesses are +race-free, with no suppressions left. It does not model +`atomic_thread_fence`, so the ordering argument above is what covers the +fences. The blackboard stress scenario asserts that no torn value ever +escapes, over hundreds of thousands of reads. + +The price is copy bandwidth, because relaxed atomics do not vectorize: +on x86-64, a 1008 B read went from about 13 ns to 33 ns, and 1 MiB +values copy about 1.8x slower. Values of a few words are unaffected. A +true atomic `memcpy` (WG21 P1478) would remove that cost, but it is not +available in C++20. ### Listing the board @@ -2001,10 +2158,14 @@ declared-but-never-written key; `read_all()` has no value to return for one. The unlocked walk copies key bytes that `claim_free_slot()` may be -writing -- the race `bb_key_equals` already carries for -`Reader::resolve()`. `bb_read_key()` is the `noinline` helper that names -that one frame for `tests/tsan.supp`; the tenancy re-check after the copy -discards a torn key. +rewriting under the board lock, and so does `Reader::resolve()`'s key +compare. Keys are stored the same way as payloads: `BlackboardEntry::key` +is `std::atomic[KEY_MAX / 8]`, with the same offset and size as +the old `char[KEY_MAX]`. `bb_store_key()` writes the zero-padded +`KEY_MAX` bytes as whole words, and `bb_load_key()` / `key_equals()` +take a whole-word copy before comparing it or returning it. The tenancy +re-check after the copy discards a torn key. `owner_node` stays plain bytes: it is only written and read +under the board lock. Neither call is atomic across the board: a key claimed or written during the walk may or may not appear. `change_seq()` tells a caller whether the @@ -2326,8 +2487,8 @@ The `lost()` counter lets the application detect overflow and act on it ### Pool exhaustion -When the slot pool is empty, `allocate()` returns `nullptr` and -`send()` returns `-EAGAIN`. If the payload exceeds `max_payload_size`, +When the slot pool is empty, `allocate()` returns an `AllocatedSlot` +whose `valid()` is false, and `send()` returns `-EAGAIN`. If the payload exceeds `max_payload_size`, `send()` returns `-EMSGSIZE`. On success, `send()` returns the number of bytes written. The publisher must handle errors — typically by yielding and retrying on `-EAGAIN`, or failing on `-EMSGSIZE`. @@ -2382,7 +2543,7 @@ The `pool_size` and `sub_ring_capacity` parameters interact: At a publish rate of R Hz, a ring of capacity C gives C/R seconds of tolerance before loss. -- **`pool_size`** must be at least `sub_ring_capacity * max_subscribers`. +- **`pool_size`** must be **more than** `sub_ring_capacity * max_subscribers`. Each active subscriber can hold up to `sub_ring_capacity` slot references (its entire ring window). Pool slots are only freed when **all** subscribers have consumed or evicted them (refcount reaches 0). @@ -2391,8 +2552,15 @@ The `pool_size` and `sub_ring_capacity` parameters interact: the publisher exhausts it and `allocate()` fails even when individual subscribers have room. -**Sizing rule:** `pool_size >= sub_ring_capacity * max_subscribers` -(hard minimum). In practice, add 2x headroom for bursty traffic: + Allocation happens before ring eviction. If every slot is held by a ring, + the next allocation fails and cannot trigger eviction. Receiving does not + help: the ring keeps its reference until overwrite or teardown. For one + subscriber with capacity 4 and a pool of 4, four sends exhaust the pool + even if every message was received. + +**Sizing rule:** `pool_size > sub_ring_capacity * max_subscribers` +(hard minimum: at least one slot must stay allocatable so eviction can +start). In practice, add 2x headroom for bursty traffic: `pool_size = sub_ring_capacity * max_subscribers * 2`. The `sub_ring_capacity` is the primary tuning knob: diff --git a/README.md b/README.md index bd9d0e6..8d43554 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,12 @@ invalid value falls back to `0600` with a warning on stderr. |----------|-------------|-------| | Linux | `shm_open` / `mmap` | `SYS_futex` | | macOS | `shm_open` / `mmap` | `__ulock_wait` / `__ulock_wake` | -| Windows | `CreateFileMapping` / `MapViewOfFile` | `WaitOnAddress` / `WakeByAddressAll` | +| Windows | `CreateFileMapping` / `MapViewOfFile` | `WaitOnAddress` / `WakeByAddressAll` (*) | + +(*) **Windows limitation:** `WakeByAddressAll` wakes only the calling process. +A cross-process `receive()` may wait until timeout; unread messages can overflow +the ring during that wait. Use a timeout within the ring's buffering budget, +or poll. See [ARCHITECTURE.md](ARCHITECTURE.md) (Platform Abstraction). Actively validated on Linux x86-64, Linux ARM64 (Raspberry Pi 4B, 12 h continuous stress), and Darwin ARM64 (Apple Silicon, 12 h continuous stress: 2660 passes, 0 failures, 0 reorders) via `scripts/validate.sh` and `tests/endurance.sh`. diff --git a/benchmarks/microbench.cc b/benchmarks/microbench.cc index d46556e..7ba6f98 100644 --- a/benchmarks/microbench.cc +++ b/benchmarks/microbench.cc @@ -37,16 +37,16 @@ static void BM_TreiberPopPush(benchmark::State& state) SHM_NAME, kickmsg::channel::PubSub, cfg, "bench"); auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); for (auto _ : state) { - uint32_t idx = kickmsg::treiber_pop(hdr->free_top, base, hdr); + uint32_t idx = kickmsg::treiber_pop(header->free_top, base, region.geometry()); benchmark::DoNotOptimize(idx); if (idx != kickmsg::INVALID_SLOT) { - auto* slot = kickmsg::slot_at(base, hdr, idx); - kickmsg::treiber_push(hdr->free_top, slot, idx); + auto* slot = kickmsg::slot_at(base, region.geometry(), idx); + kickmsg::treiber_push(header->free_top, slot, idx); } } @@ -249,7 +249,7 @@ static void BM_CASAdmission(benchmark::State& state) auto region = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg, "bench"); - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); // Set ring to Live so CAS admission succeeds ring->state_flight.store( kickmsg::ring::make_packed(kickmsg::ring::Live), diff --git a/examples/hello_diagnose.cc b/examples/hello_diagnose.cc index cd29d35..08c0be3 100644 --- a/examples/hello_diagnose.cc +++ b/examples/hello_diagnose.cc @@ -56,23 +56,23 @@ int main() std::cout << "\n=== Step 2: Inject faults (simulating publisher crashes) ===\n"; auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); // Fault 1: Lock a ring entry (simulates publisher crash mid-commit) { - auto* ring = kickmsg::sub_ring_at(base, hdr, 0); + auto* ring = kickmsg::sub_ring_at(base, region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // Pretend a publisher claimed pos=write_pos and locked the entry uint64_t wp = ring->write_pos.load(std::memory_order_acquire); ring->write_pos.store(wp + 1, std::memory_order_release); - entries[wp & hdr->sub_ring_mask].sequence.store( + entries[wp & header->sub_ring_mask].sequence.store( kickmsg::seq_lock(wp), std::memory_order_release); std::cout << " Injected: stale lock at ring 0, pos " << wp << "\n"; } // Fault 2: Stuck ring (simulates subscriber teardown timeout after publisher crash) { - auto* ring = kickmsg::sub_ring_at(base, hdr, 1); + auto* ring = kickmsg::sub_ring_at(base, region.geometry(), 1); ring->state_flight.store( kickmsg::ring::make_packed(kickmsg::ring::Free, 1), std::memory_order_release); diff --git a/examples/hello_zerocopy.cc b/examples/hello_zerocopy.cc index 71b5aa2..e1d606c 100644 --- a/examples/hello_zerocopy.cc +++ b/examples/hello_zerocopy.cc @@ -42,16 +42,17 @@ int main() // Publish a few "frames" for (uint32_t i = 0; i < 3; ++i) { - auto [ptr, max_size] = pub.allocate(); - if (ptr == nullptr) + auto slot = pub.allocate(); + if (not slot.valid()) { std::cerr << "Pool exhausted at frame " << i << "\n"; continue; } - ImageHeader hdr{640, 480, 3, i}; - std::memcpy(ptr, &hdr, sizeof(hdr)); - pub.publish(sizeof(hdr)); + // Written straight into shared memory: no staging buffer, no copy. + ImageHeader header{640, 480, 3, i}; + std::memcpy(slot.data(), &header, sizeof(header)); + slot.publish(sizeof(header)); std::cout << "Published frame " << i << " (640x480x3)\n"; } @@ -59,10 +60,10 @@ int main() // Zero-copy receive: view points directly into shared memory while (auto view = sub.try_receive_view()) { - auto const* hdr = static_cast(view->data()); - std::cout << "Received frame " << hdr->frame_id - << " (" << hdr->width << "x" << hdr->height - << "x" << hdr->channels << ")" + auto const* header = static_cast(view->data()); + std::cout << "Received frame " << header->frame_id + << " (" << header->width << "x" << header->height + << "x" << header->channels << ")" << " — zero-copy, " << view->len() << " bytes pinned\n"; // The slot remains pinned while 'view' is alive. diff --git a/include/kickmsg/Blackboard.h b/include/kickmsg/Blackboard.h index 5b65892..d316cfc 100644 --- a/include/kickmsg/Blackboard.h +++ b/include/kickmsg/Blackboard.h @@ -20,7 +20,9 @@ namespace kickmsg { namespace blackboard { - constexpr uint32_t VERSION = 1; + /// 2: payload and key bytes are accessed as relaxed atomics; a version-1 peer's + /// plain copies would race them again. + constexpr uint32_t VERSION = 2; constexpr uint64_t MAGIC = 0x214B4C424B43494BULL; // "KICKBLK!" constexpr std::size_t KEY_MAX = 128; constexpr std::size_t NODE_NAME_MAX = 64; @@ -86,6 +88,16 @@ namespace kickmsg std::string owner_node; bool owner_alive; }; + + /// Validated local copy of the board geometry; offsets and bounds use only this. + /// The shared header stays peer-writable after open. + struct Geometry + { + uint32_t capacity{0}; + std::size_t max_value_size{0}; ///< Value limit; never the padded stride + std::size_t value_stride{0}; + std::size_t values_offset{0}; ///< Byte offset of the first value cell + }; } /// One value cell. Written between the odd and even `publish` stores and @@ -125,7 +137,9 @@ namespace kickmsg std::atomic key_hash; ///< resolve pre-filter only, never an identity proof std::atomic declared_at_ns; uint8_t _pad1[8]; - char key[blackboard::KEY_MAX]; ///< may be unterminated + /// Key text as atomic words: read unlocked while a claim rewrites it. Access only + /// through bb_store_key() / bb_load_key(). May be unterminated. + std::atomic key[blackboard::KEY_MAX / sizeof(uint64_t)]; char owner_node[blackboard::NODE_NAME_MAX]; ///< may be unterminated uint8_t _padding[128]; }; @@ -135,6 +149,8 @@ namespace kickmsg "entry stride must keep every entry cache-line aligned"); static_assert(offsetof(BlackboardEntry, key) == 64, "the guard words must occupy exactly the first cache line"); + static_assert(blackboard::KEY_MAX % sizeof(uint64_t) == 0, + "key storage is whole atomic words"); static_assert(offsetof(BlackboardEntry, _padding) == 256, "BlackboardEntry field offsets must match the expected 256 B prefix"); static_assert(std::is_standard_layout::value, @@ -195,8 +211,15 @@ namespace kickmsg "BlackboardHeader is placed in shared memory via reinterpret_cast"); BlackboardEntry* bb_entry_at(void* base, uint32_t idx); - BlackboardCell* bb_cell_at(void* base, uint32_t idx, uint64_t parity); - uint8_t* bb_cell_payload(BlackboardCell* cell); + BlackboardCell* bb_cell_at(void* base, blackboard::Geometry const& geometry, uint32_t idx, uint64_t parity); + /// Payload words following the cell. Accessed only as whole relaxed words, so a + /// reader overlapping a writer never mixes access sizes. + std::atomic* bb_cell_words(BlackboardCell* cell); + + /// Store `len` <= KEY_MAX key bytes, zero-padded to KEY_MAX, as relaxed words. + void bb_store_key(BlackboardEntry* entry, char const* key, std::size_t len); + /// Relaxed word copy of the stored key, cut at the first NUL or KEY_MAX. + std::string bb_load_key(BlackboardEntry const* entry); uint64_t bb_config_hash(blackboard::Config const& cfg); @@ -292,17 +315,18 @@ namespace kickmsg private: friend class Blackboard; - Writer(void* base, uint32_t entry_idx, uint64_t tenancy, - uint64_t writes, uint64_t owner_pid, std::string key); - - void* base_{nullptr}; - uint32_t entry_idx_{INVALID_SLOT}; - uint64_t tenancy_{0}; - uint64_t writes_{0}; ///< sole owner, so this counter lives in the handle + Writer(void* base, blackboard::Geometry const& geometry, uint32_t entry_idx, + uint64_t tenancy, uint64_t writes, uint64_t owner_pid, std::string key); + + void* base_{nullptr}; + blackboard::Geometry geometry_{}; + uint32_t entry_idx_{INVALID_SLOT}; + uint64_t tenancy_{0}; + uint64_t writes_{0}; ///< sole owner, so this counter lives in the handle /// Declaring process. A Writer inherited across fork() writes and /// releases nothing: the claim stays the parent's. - uint64_t owner_pid_{0}; - std::string key_; + uint64_t owner_pid_{0}; + std::string key_; }; /// Declared read interest in one key. Copyable: it owns nothing. @@ -359,19 +383,20 @@ namespace kickmsg private: friend class Blackboard; - Reader(void* base, std::string key); + Reader(void* base, blackboard::Geometry const& geometry, std::string key); /// Resolves entry_idx_ if it is unset or its tenancy moved. /// Returns false when no active entry holds this key. bool resolve() const; - void* base_{nullptr}; + void* base_{nullptr}; + blackboard::Geometry geometry_{}; /// INVALID_SLOT until the key first materializes: observing a key /// before its writer exists is a supported use. - mutable uint32_t entry_idx_{INVALID_SLOT}; - mutable uint64_t tenancy_{0}; - uint64_t key_hash_{0}; - std::string key_; + mutable uint32_t entry_idx_{INVALID_SLOT}; + mutable uint64_t tenancy_{0}; + uint64_t key_hash_{0}; + std::string key_; }; /// Claim exclusive ownership of `key`. @@ -481,6 +506,7 @@ namespace kickmsg std::string const& name() const { return name_; } uint32_t capacity() const; std::size_t max_value_size() const; + blackboard::Geometry const& geometry() const { return geometry_; } BlackboardHeader* header() { return static_cast(base_); } BlackboardHeader const* header() const { return static_cast(base_); } @@ -492,7 +518,7 @@ namespace kickmsg void init_as_creator(blackboard::Config const& cfg); /// Sweep body, run by a caller that already holds the board lock. - uint32_t sweep_locked(BlackboardHeader* h); + uint32_t sweep_locked(); struct EntryRead { @@ -508,11 +534,12 @@ namespace kickmsg std::error_code read_entry(uint32_t i, EntryRead& out, std::vector* value) const; - SharedMemory shm_; - std::string name_; - std::string owner_name_; - void* base_{nullptr}; - std::size_t size_{0}; + SharedMemory shm_; + std::string name_; + std::string owner_name_; + void* base_{nullptr}; + std::size_t size_{0}; + blackboard::Geometry geometry_{}; }; } diff --git a/include/kickmsg/Publisher.h b/include/kickmsg/Publisher.h index 59ade0f..9722a9e 100644 --- a/include/kickmsg/Publisher.h +++ b/include/kickmsg/Publisher.h @@ -1,19 +1,58 @@ #ifndef KICKMSG_PUBLISHER_H #define KICKMSG_PUBLISHER_H +#include + #include "kickmsg/types.h" #include "kickmsg/Region.h" #include "kickmsg/Waker.h" namespace kickmsg { - /// Reservation returned by Publisher::allocate(): a writable pointer - /// into shared memory plus the maximum number of bytes the caller may - /// write through it. data == nullptr signals pool exhaustion. - struct Allocation + class Publisher; + + /// A reserved pool slot. Destroying an unpublished one returns the slot + /// to the pool. Must not outlive its Publisher: nothing tracks that lifetime. + class AllocatedSlot { - void* data; - std::size_t max_size; + public: + AllocatedSlot() = default; + ~AllocatedSlot(); + + AllocatedSlot(AllocatedSlot&& other) noexcept; + AllocatedSlot& operator=(AllocatedSlot&& other) noexcept; + AllocatedSlot(AllocatedSlot const&) = delete; + AllocatedSlot& operator=(AllocatedSlot const&) = delete; + + /// False once published, or once a later allocate() took the slot back. + bool valid() const; + bool published() const { return published_; } + + /// Writable payload area of max_size() bytes, for filling the slot in place. + /// The pointer dies with the reservation and nothing can revoke a copy of it. + void* data() const { return data_; } + std::size_t max_size() const { return max_size_; } + + /// Copy `len` bytes in. Returns bytes written, 0 if the reservation is gone + /// or `len` exceeds max_size(). + std::size_t write(void const* src, std::size_t len); + + /// Commit the first `len` bytes. Returns rings delivered to; 0 also means a + /// gone reservation, an oversized length, or no subscribers. + std::size_t publish(std::size_t len); + + private: + friend class Publisher; + AllocatedSlot(Publisher& publisher, void* data, std::size_t max_size, uint64_t id) + : publisher_{&publisher}, data_{data}, max_size_{max_size}, id_{id} + { + } + + Publisher* publisher_{nullptr}; + void* data_{nullptr}; + std::size_t max_size_{0}; + uint64_t id_{0}; + bool published_{false}; }; class Publisher @@ -22,7 +61,8 @@ namespace kickmsg Publisher(SharedRegion& region, WakeBackend* backend = nullptr) : base_{region.base()} , header_{region.header()} - , commit_timeout_{microseconds{header_->commit_timeout_us}} + , geometry_{region.geometry()} + , commit_timeout_{microseconds{geometry_.commit_timeout_us}} , pending_slot_{INVALID_SLOT} , wake_backend_{backend} { @@ -33,15 +73,22 @@ namespace kickmsg Publisher(Publisher const&) = delete; Publisher& operator=(Publisher const&) = delete; + /// Moving invalidates every outstanding AllocatedSlot of the source and + /// returns its reserved slot to the pool. Publisher(Publisher&& other) noexcept : base_{other.base_} , header_{other.header_} + , geometry_{other.geometry_} , commit_timeout_{other.commit_timeout_} - , pending_slot_{other.pending_slot_} + , pending_slot_{INVALID_SLOT} + , reservation_{other.reservation_} , dropped_{other.dropped_} , wake_backend_{other.wake_backend_} { - other.pending_slot_ = INVALID_SLOT; + // No handle can reach the slot once its id is invalidated, so free it. + other.release_pending(); + // Advance, never reset: a reused moved-from publisher must not reissue old ids. + ++other.reservation_; } Publisher& operator=(Publisher&& other) noexcept @@ -51,40 +98,38 @@ namespace kickmsg release_pending(); base_ = other.base_; header_ = other.header_; + geometry_ = other.geometry_; commit_timeout_ = other.commit_timeout_; - pending_slot_ = other.pending_slot_; + pending_slot_ = INVALID_SLOT; + // Past both counters, so neither object's older handles can match. + reservation_ = std::max(reservation_, other.reservation_) + 1; dropped_ = other.dropped_; wake_backend_ = other.wake_backend_; - other.pending_slot_ = INVALID_SLOT; + other.release_pending(); + ++other.reservation_; } return *this; } - /// Reserve a slot. Returns {data, max_size}; data is nullptr if - /// the pool is exhausted. - Allocation allocate(); + /// Reserve a slot. The result is invalid if the pool is exhausted. + /// Supersedes any previous reservation and returns its slot to the pool. + AllocatedSlot allocate(); - /// Commit the currently reserved slot, recording `len` as the - /// payload size. - /// - /// Returns the number of rings delivered to. 0 means no pending - /// allocation, oversized `len` (the pending slot is recycled), or - /// zero live subscribers -- indistinguishable by design. - std::size_t publish(std::size_t len); - - /// Allocate, copy, and publish in one call. - /// Returns bytes written on success (NOT a delivery count: a - /// successful send may have reached zero subscribers), -EMSGSIZE - /// if too large, -EAGAIN if pool exhausted. + /// Allocate, copy, and publish. Returns bytes written, even with no subscribers, + /// -EMSGSIZE if too large, or -EAGAIN if the pool is exhausted. int32_t send(void const* data, std::size_t len); /// Number of per-ring delivery drops (CAS lock contention or pool exhaustion). uint64_t dropped() const { return dropped_; } private: - /// Result of waiting for the previous wrap's occupant to commit. - /// stable_lock: one lock value spanned the whole timeout window, - /// proving its (unique) holder stale -- the steal precondition. + friend class AllocatedSlot; + + /// Commit `len` bytes of the pending reservation. + std::size_t publish(std::size_t len); + + /// stable_lock means one lock value persisted for the full timeout, + /// allowing recovery to steal that position. struct CommitWait { uint64_t last_seq; @@ -105,9 +150,14 @@ namespace kickmsg bool wake_ring(SubRingHeader* ring); void* base_; + /// Shared mutable state only; pointer math uses geometry_. Header* header_; + Geometry geometry_; microseconds commit_timeout_; uint32_t pending_slot_; + /// Id of the current reservation; only ever increases, so a stale + /// AllocatedSlot id never matches again. + uint64_t reservation_{0}; uint64_t dropped_{0}; WakeBackend* wake_backend_{nullptr}; }; diff --git a/include/kickmsg/Region.h b/include/kickmsg/Region.h index 6643681..8f03d30 100644 --- a/include/kickmsg/Region.h +++ b/include/kickmsg/Region.h @@ -8,10 +8,7 @@ namespace kickmsg { - /// Runtime snapshot of a single subscriber ring. - /// Values are relaxed/acquire-loaded, so the snapshot is internally - /// consistent per-ring but may race mildly across rings -- fine for a - /// diagnostic view; not intended as a strongly-consistent read. + /// Ring diagnostics. Fields may be read at different times during live traffic. struct RingStats { uint32_t state; ///< ring::State as a raw int (0=Free, 1=Live, 2=Draining, 3=Reclaiming) @@ -21,24 +18,20 @@ namespace kickmsg uint64_t lost_count; ///< Cumulative subscriber losses on this ring }; - /// Aggregate region snapshot returned by SharedRegion::stats(). - /// Safe to call under live traffic: all reads are relaxed/acquire, - /// no writes. + /// Read-only region diagnostics, safe during live traffic. struct RegionStats { std::vector rings; ///< One entry per subscriber-ring slot (length == max_subs) uint64_t total_writes; ///< Max of write_pos across all rings: publish events observed by the channel, monotonic across subscriber churn uint64_t total_drops; ///< Sum of dropped_count across all rings uint64_t total_losses; ///< Sum of lost_count across all rings - uint64_t total_steals; ///< Stale entries stolen (self-repair + repair_locked_entries); each may orphan one slot ref until reclaim_orphaned_slots() + uint64_t total_steals; ///< Stale entries stolen (self-repair + repair_locked_entries) uint64_t live_rings; ///< Number of rings currently Live uint64_t pool_free; ///< Approximate free-slot count (walks Treiber stack -- racy under churn) uint64_t pool_size; ///< Total pool capacity (static) }; - /// Static header metadata returned by SharedRegion::info(). - /// All fields are written once at creation and never mutated, so this - /// read is a plain copy of stable bytes. + /// Header metadata copied from fields set at creation. struct RegionInfo { std::string shm_name; @@ -65,19 +58,17 @@ namespace kickmsg SharedRegion(SharedRegion const&) = delete; SharedRegion& operator=(SharedRegion const&) = delete; - // Hand-written move ops so the moved-from object's base_/size_ - // are reset to a default-constructed state. A defaulted move - // would leave them aliasing the destination's live memory -- - // base() on the moved-from object would silently return a - // dangling-looking-live pointer instead of nullptr. + // Clear base_ and size_ in the moved-from object. SharedRegion(SharedRegion&& other) noexcept : shm_{std::move(other.shm_)} , name_{std::move(other.name_)} , base_{other.base_} , size_{other.size_} + , geometry_{other.geometry_} { other.base_ = nullptr; other.size_ = 0; + other.geometry_ = Geometry{}; } SharedRegion& operator=(SharedRegion&& other) noexcept @@ -88,8 +79,10 @@ namespace kickmsg name_ = std::move(other.name_); base_ = other.base_; size_ = other.size_; + geometry_ = other.geometry_; other.base_ = nullptr; other.size_ = 0; + other.geometry_ = Geometry{}; } return *this; } @@ -104,18 +97,14 @@ namespace kickmsg channel::Config const& cfg, char const* creator_name = ""); - /// Open an existing region. When `expected_identity` and the - /// stamped identity_hash are both nonzero they must match: a - /// mismatch (two logical channels colliding on one shm name) - /// throws instead of silently sharing the region. + /// Open an existing region. If expected_identity and the stored identity are + /// both nonzero, they must match or this throws. Throws VersionMismatch on a + /// region stamped by another kickmsg build. static SharedRegion open(char const* name, uint64_t expected_identity = 0); - /// Create the region if it doesn't exist, otherwise open the - /// existing one. On the open branch, cfg.schema is IGNORED -- - /// schema is orthogonal to channel geometry and doesn't - /// participate in the config-hash mismatch check. Use - /// try_claim_schema() afterwards to publish a descriptor - /// regardless of which side ended up creating the region. + /// Create or open a region. Opening ignores cfg.schema; use try_claim_schema() + /// to publish a descriptor. Schema is separate from geometry validation. + /// Throws VersionMismatch at once on a region stamped by another kickmsg build. static SharedRegion create_or_open(char const* name, channel::Type type, channel::Config const& cfg, char const* creator_name = ""); @@ -159,71 +148,37 @@ namespace kickmsg Header* header() { return static_cast(base_); } Header const* header() const { return static_cast
(base_); } + /// Validated local copy of the geometry; pointer arithmetic uses only this. + Geometry const& geometry() const { return geometry_; } + channel::Type channel_type() const { return header()->channel_type; } /// The shared-memory name this region was created or opened with. /// Empty for a default-constructed SharedRegion (before create/open). std::string const& name() const { return name_; } - /// Read the payload schema descriptor if one has been published. - /// - /// Returns nullopt when the schema slot is still Unset, or while a - /// concurrent claim is mid-write (Claiming). The library never - /// interprets the bytes: callers apply their own mismatch policy - /// against the returned SchemaInfo (identity / layout / version / - /// name / algo tags). + /// Read the published payload schema, or nullopt while Unset or Claiming. + /// The caller decides whether the descriptor is compatible. std::optional schema() const; - /// Atomically publish a schema descriptor to the region. - /// - /// Returns true if this call claimed the slot (Unset -> Claiming -> - /// Set), false if some other claimant got there first -- in which - /// case the caller should read back with schema() and apply its - /// own mismatch policy. When another claim is mid-write, this - /// call briefly yields until the state settles or a small bounded - /// budget is exhausted; if the state is still Claiming at that - /// point (likely a crashed claimant), this call still returns - /// false and the operator should use reset_schema_claim() to - /// recover the wedged slot. - /// - /// Safe under live traffic and across processes; only reachable - /// at connect-time scale (not on the hot path). + /// Publish a schema with Unset -> Claiming -> Set. Returns true on success. + /// If another claim is in progress, wait for a bounded number of yields, + /// then return false. Read schema() to check the published descriptor. + /// Safe during live traffic. A dead claimant requires reset_schema_claim(). bool try_claim_schema(SchemaInfo const& info); - /// Recover a schema slot wedged in the Claiming state by a - /// crashed claimant (CAS'd Unset -> Claiming then died before the - /// release-store of Set). Atomically CASes Claiming -> Unset so a - /// new claim can proceed; returns true if the reset actually - /// happened, false if the state was not Claiming. - /// - /// NOT safe under live traffic. Only call after confirming the - /// crashed claimant is gone: a slow-but-alive writer could still - /// be mid-memcpy into schema_data and would then release-store - /// Set, racing a new claim into torn bytes. Mirrors the safety - /// contract of reset_retired_rings() -- a deliberate post-crash - /// action, not a routine maintenance call. + /// Reset Claiming to Unset after confirming the claimant has stopped. + /// Returns true if reset, false if the state was not Claiming. + /// Unsafe during an active claim: its writer could overwrite a new claim. bool reset_schema_claim(); - /// Read-only health check. Safe under live traffic; does NOT mutate - /// the region. Counts locked entries and ring states, and probes - /// per-ring owner liveness (a bounded number of cheap OS calls, one - /// per occupied ring -- intended for a periodic health timer, not a - /// hot path). + /// Read-only health check, safe during live traffic. Probes each occupied + /// ring's owner; intended for periodic monitoring, not the message path. /// - /// Supervisor policy: - /// - locked_entries > 0: crash residue, call repair_locked_entries() - /// - retired_rings > 0: safe for reset_retired_rings() after - /// confirming the crashed publisher is gone - /// - draining_rings > 0: usually transient (subscriber tearing down), - /// persistent counts may indicate a stuck teardown - /// - dead_rings > 0: a subscriber holding a Live/Draining ring died; - /// call reclaim_dead_rings() to recover the ring slot - /// - live_rings: normal occupancy - /// - schema_stuck: a claimant is in the Claiming state. This is a - /// point-in-time read, so a healthy in-progress try_claim_schema() - /// can transiently set it -- treat it as advisory and act - /// (reset_schema_claim()) only if it persists AND the claimant is - /// confirmed gone. + /// Locked entries can be repaired under live traffic. Retired rings and a + /// stuck schema claim require confirmation that their writers have stopped. + /// Draining and schema_stuck can be transient; dead_rings identifies owners + /// that reclaim_dead_rings() can recover. struct HealthReport { uint32_t locked_entries; ///< Entries holding a position-tagged lock, or committed >1 wrap stale @@ -235,81 +190,45 @@ namespace kickmsg }; HealthReport diagnose(); - /// Repair entries left mid-commit by a crashed publisher, committing - /// them as skip markers so future publishers wrap past. Safe under - /// live traffic: locks are stolen only after a grace re-check proves - /// the holder stale (one commit_timeout, same value), every steal is - /// CAS-owned, and a merely-slow publisher detects the theft as a - /// drop. Blocks one commit_timeout when locked entries exist. - /// Returns the number of entries repaired. + /// Replace stalled commits with skip markers. Safe during live traffic: + /// recheck locks after one commit_timeout and steal by CAS. A resumed + /// publisher detects the stolen lock and drops its commit. + /// Waits one commit_timeout if locks exist; returns entries repaired. std::size_t repair_locked_entries(); - /// Reset retired rings (Free | in_flight>0) so new subscribers can - /// claim them. These rings were left stuck by a subscriber teardown - /// that timed out on a crashed publisher's in_flight. - /// - /// Only safe after confirming the crashed publisher is gone. - /// Unlike repair_locked_entries(), this is a deliberate post-crash - /// action, not a routine maintenance call. - /// Returns the number of rings reset. + /// Reset Free rings with in_flight > 0 so subscribers can reuse them. + /// Only call after confirming the admitted publishers have stopped. + /// Returns the number reset. std::size_t reset_retired_rings(); - /// Reclaim rings whose owner process is provably dead (pid + start - /// time checked against the OS); a slow-but-alive subscriber is - /// never touched. Two-phase like Registry::sweep_stale: single-shot - /// CAS to Reclaiming, death re-verified under that exclusivity, then - /// freed or restored -- safe for concurrent reclaimers; in_flight - /// churn just defers a ring to the next call. in_flight itself is - /// preserved (a mid-commit publisher must still fetch_sub), so a - /// reclaimed ring may land retired for reset_retired_rings(); - /// slot refs are recovered by reclaim_orphaned_slots(). - /// Returns the number of rings reclaimed. + /// Reclaim rings with a dead owner, checked by PID and start time. + /// CAS to Reclaiming, recheck death, then free or restore the ring. + /// Concurrent changes defer a ring to the next call. Preserve in_flight + /// for late publisher decrements; remaining counts require reset_retired_rings(). + /// Undrained entry claims keep their slots until the next publisher at each entry. + /// Returns the number reclaimed. /// - /// Residuals: a subscriber that crashes in the few instructions - /// between winning the claim CAS and recording its pid leaves - /// owner_pid == 0, which this cannot attribute and so will not - /// reclaim. A reclaimer that crashes mid-pass leaves the ring - /// Reclaiming; a later call recovers it (dead owner) or the owner's - /// own teardown does (live owner). + /// A crash before the owner PID is recorded can strand a ring. A later + /// call or owner teardown can recover a ring left at Reclaiming. std::size_t reclaim_dead_rings(); - /// Runtime counter snapshot -- safe under live traffic. - /// - /// Reads the cross-process per-ring counters (`write_pos`, - /// `dropped_count`, `lost_count`) plus ring state and an approximate - /// pool-free count. Intended for external monitoring and the CLI's - /// `stats` / `watch` subcommands. - /// - /// Cheap (no syscalls, no locks, a handful of atomic loads) but not a - /// strongly-consistent view: individual per-ring values are consistent - /// with themselves (sequential loads on one variable), but different - /// rings may be read at slightly different instants. The free-stack - /// walk for `pool_free` is bounded by `pool_size` so it can't loop - /// forever under racing pushes/pops. + /// Read ring counters and an approximate free-slot count under live traffic. + /// Uses no locks or syscalls. Fields may be sampled at different times; + /// the free-stack walk is bounded by pool_size. RegionStats stats() const; /// Static header snapshot -- geometry + creator metadata. All /// fields are written once at creation, so this is a plain copy. RegionInfo info() const; - /// Reclaim orphaned slots (refcount > 0 but not referenced by any ring entry). - /// These are caused by publisher crashes between allocate and publish, or by - /// skipped drain on subscriber teardown timeout. - /// - /// NOT safe under live traffic. Call only when: - /// - all publishers are quiesced (a publisher between refcount pre-set - /// and ring push has rc > 0 with no ring entry yet), AND - /// - no outstanding SampleView exists (a view holds a refcount pin on - /// its slot without any ring entry reference; reclaiming it would free - /// memory still being read). - /// Returns the number of slots reclaimed. + /// Reclaim off-stack slots with no ring claim, and reset every other off-stack + /// slot's refcount to its claim count. + /// Only call with all publishers stopped and no outstanding SampleView: both + /// can hold slots without ring entries. Returns the number reclaimed. std::size_t reclaim_orphaned_slots(); private: - /// Stamp channel geometry, creator metadata, optional schema, and - /// finally MAGIC into an already-mapped region. Shared between - /// create() and create_or_open()'s creator branch so the two paths - /// never diverge on layout or ordering. + /// Initialize the mapped region and publish MAGIC last. void stamp_new_region(channel::Type type, channel::Config const& cfg, char const* creator_name, std::size_t total_size, std::size_t sub_rings_offset, std::size_t pool_offset, @@ -320,6 +239,7 @@ namespace kickmsg std::string name_; void* base_{nullptr}; std::size_t size_{0}; + Geometry geometry_{}; }; } diff --git a/include/kickmsg/Registry.h b/include/kickmsg/Registry.h index 439ac90..a454278 100644 --- a/include/kickmsg/Registry.h +++ b/include/kickmsg/Registry.h @@ -15,11 +15,9 @@ namespace kickmsg { namespace registry { - constexpr uint32_t VERSION = 3; + constexpr uint32_t VERSION = 4; constexpr uint64_t MAGIC = 0x214745524B43494BULL; // "KICKREG!" - // Supports up to ~200-400 topics with a few participants each, - // plus headroom for transient tasks. 4096 × 512 B = 2 MB per - // namespace. + // 4096 entries use 2 MB per namespace. constexpr uint32_t DEFAULT_CAPACITY = 4096; constexpr std::size_t SHM_NAME_MAX = 128; constexpr std::size_t TOPIC_NAME_MAX = 128; @@ -32,14 +30,8 @@ namespace kickmsg Both = 3, ///< Node is both producer and consumer on this channel }; - /// What the channel is used for, from the user-facing API's point - /// of view. channel_type (in types.h) is the low-level ring - /// geometry (PubSub vs Broadcast); Kind distinguishes Mailbox - /// from PubSub even though both share channel::PubSub geometry. - /// Open enum: new kinds are added without a registry::VERSION bump - /// (the field is a uint32_t, so no offset moves). Readers MUST - /// tolerate an unknown value -- never switch exhaustively without a - /// default. + /// Logical channel kind; Mailbox and Pubsub share PubSub ring geometry. + /// Readers must tolerate unknown values. New kinds do not change the layout. enum Kind : uint32_t { Pubsub = 1, @@ -48,9 +40,8 @@ namespace kickmsg Blackboard = 4, }; - /// Only `Active` slots are visible to snapshot readers. - /// `Reclaiming` is the exclusive lock held by `sweep_stale` to - /// prevent ABA on the state CAS. + /// Snapshots include only Active rows with a stable even generation. + /// Reclaiming rows are unavailable to registrants and sweepers. enum SlotState : uint32_t { Free = 0, @@ -60,18 +51,17 @@ namespace kickmsg }; } - /// In-SHM entry, 512 B. Readers go through `Registry::snapshot()`. - /// Scalar fields are atomic so a snapshot reader racing with a new- - /// tenant writer never hits a C++ data race; the seqlock (generation - /// + state) discards torn copies. Do not reorder fields without - /// bumping `registry::VERSION`. + /// Shared-memory entry, 512 bytes. Use Registry::snapshot() to read it. + /// Scalar fields are atomic; generation and state detect changes during a copy. + /// Changing the layout requires a registry::VERSION bump. struct ParticipantEntry { std::atomic state; std::atomic channel_type; std::atomic role; std::atomic kind; - std::atomic generation; ///< seqlock version, bumped on every mutation + /// Even means settled; odd means a writer or sweeper holds the row. + std::atomic generation; std::atomic pid; ///< release/acquire-accessed; inspected while state==Claiming std::atomic pid_starttime; ///< OS-reported start time, or 0 if unavailable std::atomic created_at_ns; @@ -85,6 +75,12 @@ namespace kickmsg static_assert(offsetof(ParticipantEntry, _padding) == 368, "ParticipantEntry field offsets must match expected 368 B prefix"); + /// CAS the validated even generation to odd to acquire the row. + /// Returns false without changes if the version differs or is already odd. + /// The caller must verify owner death before acquisition. + /// Odd rows left by a crash remain unavailable to live recovery. + bool acquire_tenancy(ParticipantEntry& e, uint32_t generation); + /// Plain copyable snapshot of one participant. struct Participant { @@ -99,10 +95,8 @@ namespace kickmsg std::string node_name; }; - /// Topic-centric grouping of registry entries: all participants on - /// one shm_name, split by role (producer / consumer) and by pid - /// liveness (alive / stall). A Role::Both participant appears in - /// both producers and consumers. + /// Participants grouped by shm_name, role, and process liveness. + /// Role::Both appears in both producer and consumer lists. struct TopicSummary { std::string shm_name; @@ -139,13 +133,13 @@ namespace kickmsg Registry& operator=(Registry&&) noexcept = default; /// `capacity` is only used on the create branch; an existing - /// registry keeps its creator's capacity. + /// registry keeps its creator's capacity. Throws VersionMismatch on an + /// existing registry from another kickmsg build. static Registry open_or_create(std::string const& kmsg_namespace, uint32_t capacity = registry::DEFAULT_CAPACITY); - /// Returns nullopt if the region doesn't exist. For read-only - /// tools that must not create a 2 MB SHM as a side effect of - /// inspection. Throws on version mismatch. + /// Open without creating a region. Returns nullopt if absent; throws + /// VersionMismatch on a registry from another kickmsg build. static std::optional try_open(std::string const& kmsg_namespace); static void unlink(std::string const& kmsg_namespace); @@ -159,7 +153,8 @@ namespace kickmsg registry::Role role, std::string const& node_name); - /// Idempotent — `INVALID_SLOT` or already-Free slots are no-ops. + /// Retire a slot still owned by the caller. INVALID_SLOT and non-Active + /// rows are ignored. A stale index can retire a replacement owner. void deregister(uint32_t slot_index); /// Copy of all `Active` entries. Does not filter by process @@ -172,8 +167,12 @@ namespace kickmsg /// Results are sorted by shm_name for stable output. std::vector list_topics() const; - /// CAS-resets `Active` slots whose `pid` no longer exists. - /// Returns the number of slots freed. + /// Reclaim even-generation Active or Claiming rows whose owner is dead. + /// Returns the number freed. Safe during live registration and concurrent sweeps. + /// + /// Odd generations and Reclaiming rows are skipped because their holder may + /// still be writing. A crash in either state can strand a slot until the + /// registry is replaced; live sweeping cannot safely recover it. uint32_t sweep_stale(); std::string const& name() const { return name_; } @@ -192,6 +191,7 @@ namespace kickmsg SharedMemory shm_; std::string name_; + uint32_t capacity_{0}; ///< Validated at open; the shared header copy is peer-writable }; } diff --git a/include/kickmsg/Subscriber.h b/include/kickmsg/Subscriber.h index bd3718c..f378d81 100644 --- a/include/kickmsg/Subscriber.h +++ b/include/kickmsg/Subscriber.h @@ -11,9 +11,7 @@ namespace kickmsg class Subscriber { public: - // Copy-based sample: data is copied into subscriber-local memory. - // Move-only: the internal buffer is reused across try_receive() - // calls, so copies would alias the same memory. + // Sample in a reusable subscriber-local buffer; valid until the next receive. class SampleRef { public: @@ -68,8 +66,8 @@ namespace kickmsg { public: SampleView() - : base_{nullptr} - , header_{nullptr} + : header_{nullptr} + , slot_{nullptr} , slot_idx_{INVALID_SLOT} , len_{0} , ring_pos_{0} @@ -82,8 +80,8 @@ namespace kickmsg SampleView& operator=(SampleView const&) = delete; SampleView(SampleView&& other) noexcept - : base_{other.base_} - , header_{other.header_} + : header_{other.header_} + , slot_{other.slot_} , slot_idx_{other.slot_idx_} , len_{other.len_} , ring_pos_{other.ring_pos_} @@ -96,8 +94,8 @@ namespace kickmsg if (this != &other) { release(); - base_ = other.base_; header_ = other.header_; + slot_ = other.slot_; slot_idx_ = other.slot_idx_; len_ = other.len_; ring_pos_ = other.ring_pos_; @@ -112,7 +110,7 @@ namespace kickmsg { return nullptr; } - return slot_data(slot_at(base_, header_, slot_idx_)); + return slot_data(slot_); } std::size_t len() const { return len_; } @@ -122,9 +120,11 @@ namespace kickmsg private: friend class Subscriber; - SampleView(void* base, Header* hdr, uint32_t slot_idx, uint32_t len, uint64_t ring_pos) - : base_{base} - , header_{hdr} + // Use the slot pointer resolved from the subscriber's validated geometry. + SampleView(Header* header, SlotHeader* slot, uint32_t slot_idx, uint32_t len, + uint64_t ring_pos) + : header_{header} + , slot_{slot} , slot_idx_{slot_idx} , len_{len} , ring_pos_{ring_pos} @@ -135,19 +135,18 @@ namespace kickmsg { if (slot_idx_ != INVALID_SLOT) { - auto* slot = slot_at(base_, header_, slot_idx_); - auto prev = slot->refcount.fetch_sub(1, - std::memory_order_acq_rel); + auto prev = slot_->refcount.fetch_sub(1, + std::memory_order_acq_rel); if (prev == 1) { - treiber_push(header_->free_top, slot, slot_idx_); + treiber_push(header_->free_top, slot_, slot_idx_); } slot_idx_ = INVALID_SLOT; } } - void* base_; - Header* header_; + Header* header_; ///< free_top only + SlotHeader* slot_; uint32_t slot_idx_; uint32_t len_; uint64_t ring_pos_; @@ -223,7 +222,9 @@ namespace kickmsg Wait head_state(SubRingHeader* ring) const; void* base_; + /// Shared mutable state only; pointer math uses geometry_. Header* header_; + Geometry geometry_; uint32_t ring_idx_; uint64_t start_pos_; uint64_t read_pos_; diff --git a/include/kickmsg/types.h b/include/kickmsg/types.h index b1db19e..89f886b 100644 --- a/include/kickmsg/types.h +++ b/include/kickmsg/types.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace kickmsg @@ -20,23 +21,30 @@ namespace kickmsg static_assert(std::atomic::is_always_lock_free, "Kickmsg requires lock-free 32-bit atomics."); + /// A shared-memory object was stamped by an incompatible kickmsg build. + /// Fatal by design: one namespace cannot mix versions. + class VersionMismatch : public std::runtime_error + { + public: + using std::runtime_error::runtime_error; + }; + constexpr uint64_t MAGIC = 0x4B49434B4D534721ULL; // "KICKMSG!" - constexpr uint32_t VERSION = 8; + constexpr uint32_t VERSION = 9; constexpr uint32_t INVALID_SLOT = UINT32_MAX; constexpr std::size_t CACHE_LINE = 64; - // ---- Entry sequence-word encoding ---- - // - // [tag:2 | pos:62] 00 -> committed (word is pos + 1) - // 01 -> skip marker (word carries pos + 1; metadata untrustworthy) - // 10 -> locked by the publisher at `pos` - // 11 -> stolen by a repairer at `pos` + /// Entry::meta stores slot + 1 in 24 bits; zero means no slot. + constexpr uint64_t MAX_POOL_SIZE = (1ULL << 24) - 2; + + // [tag:2 | pos:62] + // 00: committed, pos + 1 + // 01: skip marker, pos + 1; no payload, but the slot claim remains valid + // 10: publisher lock at pos + // 11: repairer lock at pos // - // Lock values are unique (one publisher per position, locks once), so an - // unchanged lock across an interval proves one holder spanned it -- the - // staleness proof repair relies on. Stolen entries commit the skip tag: - // the stolen-from publisher's plain metadata stores can land at any later - // time, so nothing may ever trust slot_idx/payload_len under it. + // Each position has one publisher. An unchanged lock across the grace + // period can be stolen. Readers count skip markers as lost samples. constexpr uint64_t SEQ_LOCK_BIT = 1ULL << 63; constexpr uint64_t SEQ_REPAIR_BIT = 1ULL << 62; @@ -50,39 +58,14 @@ namespace kickmsg constexpr uint64_t seq_skip(uint64_t pos) { return SEQ_REPAIR_BIT | (pos + 1); } constexpr uint64_t seq_pos(uint64_t seq) { return seq & (SEQ_REPAIR_BIT - 1); } - // A healthy commit (memcpy + atomic release-store) finishes in a few - // microseconds; even under moderate CAS contention it stays well under - // a millisecond. 10 ms is therefore ~1000× a normal commit -- enough - // to absorb routine preemption without falsely evicting a live - // publisher, while still recovering from a real crash fast enough to - // avoid stalling subscribers. Applications running under severe - // oversubscription (threads ≫ cores) may want to raise this; hard - // real-time setups may want to lower it. Override via - // channel::Config::commit_timeout. + // Override via channel::Config::commit_timeout. Increase under heavy + // scheduling delays to reduce recovery of slow but live publishers. constexpr microseconds DEFAULT_COMMIT_TIMEOUT = 10ms; - /// Optional payload schema descriptor. - /// - /// The library never interprets any byte of this structure: it stores it - /// in the shared-memory header so that multiple processes (possibly built - /// at different times, from different sources) can agree -- or disagree -- - /// on the payload format carried by the channel. - /// - /// Policy (which fields to fill, how to compute the hashes, what counts as - /// a mismatch) is entirely up to the user. Typical usage: - /// - identity: cryptographic or non-cryptographic hash of a canonical - /// descriptor of the logical type (name + version + field list). - /// - layout: fingerprint of this binary's in-memory layout - /// (e.g. a checksum over (offset, size, kind) tuples per member). - /// Useful to distinguish "wrong type" from "same type, different ABI". - /// - name: human-readable identifier for diagnostics. - /// - version: user-defined version number. - /// - identity_algo / layout_algo: opaque tags that let the user's tooling - /// know which algorithm produced the corresponding bytes (e.g. 1=sha256, - /// 2=fnv128). The library never reads them. - /// - /// Size is fixed at 512 bytes (8 cache lines) to leave generous room for - /// future fields without requiring another layout-version bump. + /// Opaque payload schema, stored in shared memory. Callers choose hashes, + /// algorithm tags, versioning, and compatibility rules. + /// identity names the logical type; layout describes its binary layout. + /// The fixed 512-byte layout includes reserved space for future fields. struct SchemaInfo { std::array identity; ///< Logical fingerprint (user-defined bytes) @@ -99,9 +82,8 @@ namespace kickmsg static_assert(std::is_trivially_copyable::value, "SchemaInfo must be trivially copyable for memcpy into shared memory"); - /// Schema-slot publication state. Drives a small state machine in the - /// header so a claim writes the payload bytes between Claiming and Set, - /// and readers only observe the payload once Set is published. + /// Writers fill schema_data while Claiming and release-store Set. + /// Readers access it only after acquiring Set. namespace schema { enum State : uint32_t @@ -111,18 +93,9 @@ namespace kickmsg Set = 2, ///< Payload is stable and safe to read }; - /// Bitmask describing how two SchemaInfo values differ. - /// - /// Returned by diff(). Zero (Equal) means all checked fields match. - /// The library only compares fields with current semantic meaning -- - /// `flags` and `reserved[]` are deliberately excluded so that - /// forward-compatible additions (a new flag bit, a new field carved - /// from reserved) do NOT retroactively break existing comparisons. - /// - /// The library never decides what counts as a mismatch for the - /// caller: users combine these bits per their own policy (e.g. - /// "Identity mismatch is fatal, Version mismatch triggers a - /// negotiation, Name mismatch is just logged"). + /// Fields that differ between two schemas. Zero means all checked fields match. + /// Flags and reserved bytes are ignored for forward compatibility. + /// The caller decides which differences are acceptable. enum Diff : uint32_t { Equal = 0, @@ -134,9 +107,7 @@ namespace kickmsg LayoutAlgo = 1u << 5, ///< layout_algo tags differ }; - /// Compute a bitwise diff of the semantically-meaningful fields of - /// two schema descriptors. Pure, side-effect free; library does - /// not apply any mismatch policy. + /// Compare schema fields without applying a compatibility policy. uint32_t diff(SchemaInfo const& a, SchemaInfo const& b); } @@ -180,13 +151,6 @@ namespace kickmsg }; } - // ---- Shared-memory layout structures ---- - // - // Convention: atomic fields accessed without explicit memory_order - // (e.g. slot->refcount = 0) are in contexts where ordering is irrelevant - // (quiesced GC, post-join verification, single-threaded init). - // Explicit memory_order at all synchronization points makes them - // visually distinct from incidental reads. /// Shared-memory region header. Written once by the creator, read by all. /// Layout version changes require a VERSION bump. @@ -218,35 +182,24 @@ namespace kickmsg uint16_t creator_name_len; ///< Length of creator name string // creator_name bytes follow immediately after sizeof(Header) - /// Payload schema descriptor -- opt-in, off the hot path. - /// Published via a tiny state machine (Unset -> Claiming -> Set): - /// writers update schema_data while schema_state == Claiming, then - /// release-store Set. Readers acquire-load schema_state and only - /// read schema_data if the state is Set. + /// Write schema_data under Claiming, then release-store Set. + /// Readers acquire Set before copying schema_data. alignas(CACHE_LINE) std::atomic schema_state; alignas(CACHE_LINE) SchemaInfo schema_data; alignas(CACHE_LINE) std::atomic free_top; ///< Treiber free-stack head (tagged: gen|idx) - std::atomic steal_count; ///< Entries stolen from a stale holder (each may orphan one slot ref until GC) + std::atomic steal_count; ///< Entries stolen from a stalled publisher uint64_t identity_hash; ///< Logical-identity fingerprint, written once pre-MAGIC (0 = unstamped); detects shm-name collisions at open }; - // The creator_name tail bytes are written at offset sizeof(Header) in the - // shared-memory mapping. Guaranteeing sizeof(Header) is a multiple of - // CACHE_LINE ensures those bytes start on a fresh cache line and never - // share a line with any atomic field above (schema_state, schema_data, - // free_top). The aliasing of alignas(CACHE_LINE) on several members plus - // struct-level alignment normally produces this automatically, but we - // assert it to catch accidental layout edits. + // The creator name follows Header and must not share a cache line + // with its atomics. static_assert(sizeof(Header) % CACHE_LINE == 0, "Header size must be cache-line multiple to isolate atomic fields " "from the creator_name tail written at offset sizeof(Header)"); - // The magic/version prefix is the cross-build handshake: a build that - // opens a region stamped by a different VERSION must still be able to - // read these two fields at their fixed offsets to reject it. They are - // therefore frozen for ALL future versions -- any edit that moves them - // silently defeats the version-mismatch guard. + // Keep magic and version at fixed offsets across all ABI versions + // so incompatible mappings can be rejected. static_assert(std::is_standard_layout
::value, "Header is placed in shared memory via reinterpret_cast"); static_assert(offsetof(Header, magic) == 0, @@ -254,13 +207,61 @@ namespace kickmsg static_assert(offsetof(Header, version) == 8, "version offset is a permanent ABI contract across all versions"); + // [tag:40 | slot + 1:24]; tag is the low 40 bits of pos + 1. + // Zero in the slot field means no claim. Each claim owns one slot reference. + // Replacing a claim transfers the duty to release its reference. + // Publishers CAS only older position tags, preventing late writes from + // overwriting newer entries. + constexpr uint64_t META_SLOT_BITS = 24; + constexpr uint64_t META_SLOT_MASK = (1ULL << META_SLOT_BITS) - 1; + constexpr uint64_t META_TAG_MASK = (1ULL << 40) - 1; + + constexpr uint64_t meta_tag(uint64_t m) { return m >> META_SLOT_BITS; } + + /// Biased slot field: 0 means the entry names no slot (also the value a + /// freshly zeroed region carries, which must not read as slot 0). + constexpr uint32_t meta_slot_biased(uint64_t m) + { + return static_cast(m & META_SLOT_MASK); + } + + constexpr uint64_t meta_pack(uint64_t pos, uint32_t slot_idx) + { + uint64_t tag = (pos + 1) & META_TAG_MASK; + return (tag << META_SLOT_BITS) + | ((static_cast(slot_idx) + 1) & META_SLOT_MASK); + } + + /// True if m precedes pos under 40-bit serial-number ordering. + /// Requires positions to be less than 2^39 apart. + constexpr bool meta_precedes(uint64_t m, uint64_t pos) + { + uint64_t diff = (meta_tag(m) - ((pos + 1) & META_TAG_MASK)) & META_TAG_MASK; + return diff != 0 and (diff & (1ULL << 39)) != 0; + } + + /// Validated local copy of geometry used for pointer arithmetic. + /// Shared header fields remain writable by peers after validation. + struct Geometry + { + uint64_t sub_rings_offset; + uint64_t sub_ring_stride; + uint64_t sub_ring_capacity; + uint64_t sub_ring_mask; + uint64_t pool_offset; + uint64_t slot_stride; + uint64_t pool_size; + uint64_t slot_data_size; + uint64_t max_subs; + uint64_t commit_timeout_us; + }; + /// Ring entry: one per position in a subscriber ring. /// Packed to guarantee binary layout across compilers. struct Entry { - std::atomic sequence; ///< Commit barrier (pos + 1) and seqlock for data consistency - std::atomic slot_idx; ///< Index into the slot pool (INVALID_SLOT if released by drain) - std::atomic payload_len; ///< Actual payload bytes written to the slot + std::atomic sequence; ///< Commit barrier (pos + 1) and seqlock for data consistency + std::atomic meta; ///< Slot claim; see the meta-word encoding above }; static_assert(sizeof(Entry) == 16 and std::is_standard_layout::value, "Entry layout drives cross-process ring-stride math"); @@ -277,10 +278,7 @@ namespace kickmsg Reclaiming = 3, ///< reclaim_dead_rings() holds the ring exclusively while re-verifying owner death }; - /// Packed [in_flight:30 | state:2] in a single uint32_t. - /// Single-variable atomics eliminate cross-variable ordering concerns: - /// publisher CAS atomically checks state and increments in_flight, - /// so acquire/release is sufficient (no Dekker protocol, no seq_cst). + /// Packed [in_flight:30 | state:2]. One CAS checks Live and admits a publisher. constexpr uint32_t STATE_MASK = 0x3u; constexpr uint32_t IN_FLIGHT_ONE = 0x4u; @@ -296,14 +294,8 @@ namespace kickmsg }; } - /// Per-subscriber ring header in shared memory. - /// state_flight packs ring state and in_flight publisher count into one - /// atomic, enabling single-CAS admission without cross-variable fences. - /// write_pos, has_waiter, dropped_count, lost_count share a cache line: - /// the hot path already owns this line when incrementing write_pos, so - /// the extra fetch_add on a drop/loss path introduces no new cache- - /// coherency traffic. Writers on different rings target different - /// lines (128 B stride), so no cross-ring false sharing either. + /// Per-subscriber ring header. state_flight combines state and publisher + /// admission count. Hot counters share a line; separate rings use distinct lines. struct SubRingHeader { alignas(CACHE_LINE) std::atomic state_flight; ///< Packed [in_flight:30 | state:2] @@ -314,10 +306,7 @@ namespace kickmsg std::atomic dropped_count; ///< Cumulative publisher drops on this ring (all publishers) std::atomic lost_count; ///< Cumulative subscriber losses on this ring (all subscribers) }; - // owner_pid/owner_starttime live in state_flight's cache-line padding, so - // the struct stays 2 lines and the ring-stride math is unchanged. They are - // cold (written once on claim, read only by reclaim_dead_rings), so sharing - // the line with the hot state_flight costs nothing in steady state. + // Owner fields use state_flight's padding without changing the two-line layout. static_assert(sizeof(SubRingHeader) == 2 * CACHE_LINE, "SubRingHeader must stay 2 cache lines -- expanding it past the " "write_pos line padding requires reconsidering ring-stride math in Region.cc"); @@ -328,8 +317,11 @@ namespace kickmsg { std::atomic refcount; ///< Number of ring references + SampleView pins std::atomic next_free; ///< Next slot index in the Treiber free-stack chain + /// Length written before publication and read under a validated slot pin. + std::atomic payload_len; + uint32_t _padding; }; - static_assert(sizeof(SlotHeader) == 8 and std::is_standard_layout::value, + static_assert(sizeof(SlotHeader) == 16 and std::is_standard_layout::value, "SlotHeader layout drives cross-process slot-stride math"); static_assert(std::is_standard_layout::value, "SubRingHeader is placed in shared memory via reinterpret_cast"); @@ -357,30 +349,29 @@ namespace kickmsg constexpr uint32_t tagged_idx(uint64_t tagged) { return static_cast(tagged); } constexpr uint32_t tagged_gen(uint64_t tagged) { return static_cast(tagged >> 32); } - SubRingHeader* sub_ring_at(void* base, Header const* h, uint32_t idx); + SubRingHeader* sub_ring_at(void* base, Geometry const& geometry, uint32_t idx); - /// Forget the ring's owner and any wake it was waiting for. Call this before marking - /// the ring free, never after: once it is free someone else can claim it, and these - /// stores would wipe out what the new owner just wrote. + /// Clear owner and wake mode before publishing Free, while no replacement + /// can claim the ring. void clear_owner(SubRingHeader* ring); Entry* ring_entries(SubRingHeader* ring); - SlotHeader* slot_at(void* base, Header const* h, uint32_t idx); + SlotHeader* slot_at(void* base, Geometry const& geometry, uint32_t idx); SlotHeader* slot_at(void* pool_base, std::size_t slot_stride, uint32_t idx); uint8_t* slot_data(SlotHeader* slot); char* header_creator_name(Header* h); uint64_t compute_config_hash(channel::Type type, channel::Config const& cfg); - /// Take ownership of a ring entry observed at `observed` (a stale lock - /// or a >1-wrap-stale committed value) and commit it as an empty skip - /// marker at position `pos`. Returns false without touching the entry - /// if it changed first -- a live writer beat us; never steal then. - bool entry_steal_and_clear(Entry& e, uint64_t pos, uint64_t observed); + /// CAS a stale observed sequence to a repair lock, then publish a skip at pos. + /// Returns false if the sequence changed. Keeps Entry::meta and its reference + /// for the next publisher or drainer to release. + bool entry_steal_and_skip(Entry& e, uint64_t pos, uint64_t observed); void treiber_push(std::atomic& top, SlotHeader* slot, uint32_t slot_idx); void treiber_push(std::atomic& top, void* pool_base, std::size_t slot_stride, uint32_t slot_idx); - uint32_t treiber_pop(std::atomic& top, void* base, Header const* h); - uint32_t treiber_pop(std::atomic& top, void* pool_base, std::size_t slot_stride); + uint32_t treiber_pop(std::atomic& top, void* base, Geometry const& geometry); + /// Return INVALID_SLOT if a shared free-list index is outside pool_size. + uint32_t treiber_pop(std::atomic& top, void* pool_base, std::size_t slot_stride, uint64_t pool_size); } diff --git a/py_bindings/src/kickmsg_py.cc b/py_bindings/src/kickmsg_py.cc index e8b0034..137ff39 100644 --- a/py_bindings/src/kickmsg_py.cc +++ b/py_bindings/src/kickmsg_py.cc @@ -1,70 +1,9 @@ /// @file kickmsg_py.cc -/// @brief Python bindings for Kickmsg (nanobind-based). +/// Python bindings using nanobind. /// -/// Layout: -/// kickmsg — module -/// ChannelType — enum -/// Config — channel::Config -/// SchemaInfo — payload schema descriptor -/// HealthReport — SharedRegion::diagnose() result -/// RingStats / RegionStats — SharedRegion::stats() result -/// SharedRegion — factory methods + schema/health/repair/stats -/// Publisher — send(bytes) + allocate() → AllocatedSlot -/// AllocatedSlot — writable zero-copy handle + .publish() -/// Subscriber — try_receive / receive (GIL release) / *_view -/// SampleView — read-only zero-copy sample (buffer protocol) -/// BroadcastHandle — NamedTuple-like (pub, sub) -/// Role — registry::Role enum (Publisher/Subscriber/Both) -/// Participant — registry snapshot entry -/// Registry — per-namespace participant discovery -/// Node — high-level topic / broadcast / mailbox -/// BlackboardConfig — blackboard::Config -/// KeyStatus — Blackboard.snapshot() entry -/// ReadOutcome — Blackboard reader result (errno + bytes) -/// BlackboardWriter — declared key owner: .write(bytes) / .release() -/// BlackboardReader — declared read interest: .read() / .owner_alive() -/// Blackboard — key/value state; late readers see current values -/// schema (submodule) -/// Diff — enum (bitmask) -/// diff(a, b) — pure diff function -/// hash (submodule) -/// fnv1a_64(data[, seed]) -/// identity_from_fnv1a(descriptor) -/// -/// Zero-copy contract (lifetime-safe via the Python buffer protocol): -/// -/// slot = pub.allocate() → AllocatedSlot sized to max_payload_size. -/// memoryview(slot) is a writable view into -/// the SHM slot. The memoryview pins the -/// slot, which pins the Publisher, which -/// pins the mmap — so retained memoryviews -/// stay valid (at the mmap level) as long -/// as Python holds them. -/// slot.publish(n) → commits, recording `n` bytes as the -/// payload size. NEW memoryview(slot) -/// after this raises BufferError. -/// Memoryviews obtained BEFORE publish -/// remain pointer-valid but writing -/// through them after publish would -/// corrupt in-flight subscribers — user -/// contract: don't. -/// -/// view = sub.try_receive_view() → SampleView. memoryview(view) is a -/// read-only view into the SHM slot. The -/// memoryview pins the SampleView, which -/// pins the slot's refcount and the mmap. -/// view.release() → drops the pin. NEW memoryview(view) -/// after this raises BufferError. -/// -/// Equivalent context-manager form (preferred for short scopes): -/// with sub.try_receive_view() as view: -/// mv = memoryview(view) -/// ... use mv ... -/// # pin released on block exit, even on exception -/// -/// The pinning is enforced by Py_buffer::obj = self + Py_INCREF inside -/// the buffer-protocol getbuffer slot, so it works with numpy.asarray(), -/// torch.frombuffer(), and any other consumer that respects the protocol. +/// Exported buffers keep wrappers and mappings alive. They do not extend +/// reservation validity: stop using writable views before publish() or the +/// next allocate(). Do not use SampleView buffers after release(). #include #include @@ -94,23 +33,7 @@ using namespace nb::literals; namespace kickmsg { - // Python-only wrapper around a Publisher reservation. Holds the slot - // pointer and max payload size returned by Publisher::allocate(), - // exposes the writable buffer protocol so `memoryview(slot)` points - // directly into the shared-memory slot (zero-copy), and has a - // .publish(n) method that commits `n` bytes via the Publisher. - // - // Lifetime: the Py_buffer obtained through buffer protocol pins this - // AllocatedSlot alive (view->obj = self; Py_INCREF), which in turn - // pins the Publisher (via nb::keep_alive<1, 2> on the constructor), - // which pins the SharedRegion mmap. A memoryview retained past - // `.publish()` stays technically valid as a pointer — but any NEW - // memoryview(slot) after publish is refused with BufferError so - // accidental reuse is caught. - // A blackboard read result plus its bytes. The C++ ReadOutcome carries - // only the length: values are always copied at the Python boundary - // because a writer may overwrite the cell mid-read, so unlike SampleView - // there is nothing safe to expose through the buffer protocol. + // Copy Blackboard values into Python bytes; cells cannot be safely exported. struct PyReadOutcome { std::error_code ec; @@ -119,10 +42,7 @@ namespace kickmsg uint64_t update_count; }; - /// The blackboard reports std::error_code; Python's standard for the same - /// thing is OSError carrying an errno, so failures are raised, not returned. - /// Built by hand because nanobind's builtin_exception set has no OSError, - /// and the (errno, strerror) pair is what populates e.errno for the caller. + /// Build OSError with (errno, message); nanobind has no OSError helper. void raise_if(std::error_code ec, char const* what) { if (ec) @@ -135,26 +55,11 @@ namespace kickmsg } } - struct PyAllocatedSlot - { - Publisher* publisher; - void* ptr; - std::size_t max_size; - bool published; - - PyAllocatedSlot(Publisher& p, void* data, std::size_t cap) - : publisher{&p}, ptr{data}, max_size{cap}, published{false} - { - } - }; } namespace { - // Buffer protocol for Subscriber::SampleView (read-only zero-copy). - // Sets view->obj = self + Py_INCREF so the resulting memoryview pins - // the SampleView alive, which transitively pins the slot refcount - // and the mmap. + // Py_buffer::obj keeps the SampleView and its mapping alive. int sv_getbuffer(PyObject* self, Py_buffer* view, int /*flags*/) noexcept { using SV = kickmsg::Subscriber::SampleView; @@ -185,8 +90,7 @@ namespace void sv_releasebuffer(PyObject* /*self*/, Py_buffer* /*view*/) noexcept { - // Nothing to free: shape/strides borrow from the Py_buffer itself, - // and Py_DECREF(view->obj) is handled by CPython's memoryview. + // CPython decrefs view->obj; shape and strides use the Py_buffer storage. } PyType_Slot sv_slots[] = { @@ -195,14 +99,12 @@ namespace { 0, nullptr } }; - // Buffer protocol for PyAllocatedSlot (writable zero-copy). Refuses - // new buffer requests once .publish() has been called so stale writes - // don't corrupt messages that are already in flight to subscribers. + // Reject new buffer requests after publication or reservation replacement. int as_getbuffer(PyObject* self, Py_buffer* view, int /*flags*/) noexcept { - auto* slot = nb::inst_ptr(nb::handle(self)); + auto* slot = nb::inst_ptr(nb::handle(self)); - if (slot->published) + if (slot->published()) { PyErr_SetString(PyExc_BufferError, "AllocatedSlot has already been published; its buffer is " @@ -211,10 +113,20 @@ namespace return -1; } - view->buf = slot->ptr; + if (not slot->valid()) + { + PyErr_SetString(PyExc_BufferError, + "AllocatedSlot was superseded by a later Publisher.allocate(); " + "its slot is back in the pool and writing through it would " + "corrupt another reservation"); + view->obj = nullptr; + return -1; + } + + view->buf = slot->data(); view->obj = self; Py_INCREF(self); - view->len = static_cast(slot->max_size); + view->len = static_cast(slot->max_size()); view->itemsize = 1; view->readonly = 0; // writable view->ndim = 1; @@ -253,16 +165,11 @@ namespace namespace kickmsg { - // Native module name is `_native`; the outer `kickmsg/__init__.py` does - // `from ._native import *` so user-visible import paths (kickmsg.Publisher, - // kickmsg.Node, …) are unchanged. NB_MODULE(_native, m) { m.doc() = "Kickmsg — lock-free shared-memory IPC (native bindings)"; - // ------------------------------------------------------------------- // Enums & simple types - // ------------------------------------------------------------------- // channel::None is exposed as NoChannel: `ChannelType.None` would be // a syntax error in Python. @@ -296,9 +203,7 @@ namespace kickmsg ", max_payload_size=" + std::to_string(c.max_payload_size) + ")"; }); - // ------------------------------------------------------------------- // SchemaInfo + schema submodule (Diff / diff) - // ------------------------------------------------------------------- nb::class_(m, "SchemaInfo") .def(nb::init<>()) @@ -353,9 +258,7 @@ namespace kickmsg schema_mod.def("diff", &schema::diff, "a"_a, "b"_a, "Return a schema.Diff bitmask of the fields that differ."); - // ------------------------------------------------------------------- // hash submodule - // ------------------------------------------------------------------- auto hash_mod = m.def_submodule("hash", "Optional FNV-1a hash helpers"); hash_mod.attr("FNV1A_64_OFFSET_BASIS") = @@ -375,9 +278,7 @@ namespace kickmsg "Pack a 64-bit FNV-1a of `descriptor` into the leading 8 bytes " "of a 64-byte identity slot, zero-padding the rest."); - // ------------------------------------------------------------------- // HealthReport - // ------------------------------------------------------------------- nb::class_(m, "HealthReport") .def_ro("locked_entries", &SharedRegion::HealthReport::locked_entries) @@ -394,9 +295,7 @@ namespace kickmsg ", schema_stuck=" + (r.schema_stuck ? "True" : "False") + ")"; }); - // ------------------------------------------------------------------- - // RingStats / RegionStats — runtime counter snapshot via stats() - // ------------------------------------------------------------------- + // RingStats / RegionStats nb::class_(m, "RingStats") .def_ro("state", &RingStats::state) @@ -464,9 +363,7 @@ namespace kickmsg ", creator='" + i.creator_name + "')"; }); - // ------------------------------------------------------------------- // SharedRegion - // ------------------------------------------------------------------- nb::class_(m, "SharedRegion") .def_static("create", @@ -510,9 +407,7 @@ namespace kickmsg m.def("unlink_shm", [](std::string const& name) { SharedMemory::unlink(name); }, "name"_a, "Unlink a shared-memory entry by name (no-op if absent)."); - // ------------------------------------------------------------------- - // Registry — per-namespace participant directory - // ------------------------------------------------------------------- + // Registry nb::enum_(m, "Role") .value("Publisher", registry::Publisher) @@ -601,27 +496,13 @@ namespace kickmsg m.def("current_pid", ¤t_pid, "Return the PID of the current process."); - // SampleRef (the C++ byte-copy sample) is not bound directly — - // try_receive() / receive() auto-convert it to `bytes` at the - // Python boundary. Users who want ring-position information - // can use try_receive_view() / receive_view() which return - // SampleView (bound below). - - // ------------------------------------------------------------------- - // SampleView — zero-copy, pins the slot. - // - // Supports the Python buffer protocol: `memoryview(view)` returns - // a read-only memoryview pointing directly at shared memory (no - // copy). The memoryview pins the SampleView alive — so retaining - // a memoryview beyond the SampleView's Python reference keeps the - // slot pinned and the mmap valid until the memoryview is released. - // That makes the zero-copy path lifetime-safe by construction. - // ------------------------------------------------------------------- + // Copy samples become bytes; SampleView also exposes ring position. + + // SampleView + // Read-only exported buffers keep the wrapper alive. release() drops its pin. nb::class_(m, "SampleView", nb::type_slots(sv_slots)) - // __len__ so `len(view)` works; ring_pos / valid as properties - // (no-arg accessors, Pythonic). .def("__len__", [](Subscriber::SampleView const& v) -> std::size_t { return v.len(); }) @@ -630,29 +511,15 @@ namespace kickmsg .def("release", [](Subscriber::SampleView& v) { - // Move-assign a default-constructed view: the old - // state's release() fires via the move-assignment, - // dropping the pin. Subsequent memoryview(view) - // calls fail with BufferError (see sv_getbuffer). + // Release the pin and make future buffer requests fail. v = Subscriber::SampleView{}; }, "Release the slot pin early. Idempotent; after this, any " "NEW memoryview(view) call raises BufferError. Memoryviews " "obtained before .release() remain valid as pointers but " "should not be used (the pin is gone).") - // Context-manager support: `with view:` releases the pin on - // block exit. - // - // __enter__ returns self with reference_internal rv_policy so - // nanobind resolves to the existing Python wrapper rather - // than constructing a second one around the same C++ object - // (which would double-release on exit). - // - // __exit__ uses nb::args to accept the three positional - // arguments Python's `with` statement passes (exc_type, - // exc_value, traceback) — explicit `(nb::object, nb::object, - // nb::object)` triggers a dispatch error in nanobind's - // multi-arg resolution (nb::args sidesteps it). + // Return the existing wrapper from __enter__ to avoid a second owner. + // nb::args accepts the three exception arguments passed to __exit__. .def("__enter__", [](Subscriber::SampleView& v) -> Subscriber::SampleView& { return v; }, @@ -666,64 +533,62 @@ namespace kickmsg ", valid=" + (v.valid() ? "True" : "False") + ")"; }); - // ------------------------------------------------------------------- - // AllocatedSlot — handle returned by Publisher.allocate(). - // - // Supports the writable buffer protocol: `memoryview(slot)` or - // `numpy.asarray(slot)` gets you a zero-copy writable view of the - // reserved shared-memory slot. Fill it in place, then call - // `slot.publish()` to commit. After publish, any NEW - // memoryview(slot) call raises BufferError. - // - // keep_alive<1, 2>: keep the Publisher (arg 2) alive while this - // slot (self, arg 1) is alive — the slot points into the - // Publisher's mmap and must not outlive it. - // ------------------------------------------------------------------- - - nb::class_(m, "AllocatedSlot", + // AllocatedSlot + // The token rejects stale handles; existing writable buffers cannot be revoked. + // Stop using them before publish() or another allocate(). + // keep_alive keeps the publisher mapped while the handle exists. + + nb::class_(m, "AllocatedSlot", nb::type_slots(as_slots)) .def("publish", - [](PyAllocatedSlot& s, std::size_t len) -> std::size_t + [](AllocatedSlot& s, std::size_t len) -> std::size_t { - if (s.published) + if (s.published()) { throw nb::value_error( "AllocatedSlot.publish() called more than once"); } - if (len > s.max_size) + if (not s.valid()) + { + throw nb::value_error( + "AllocatedSlot was superseded by a later " + "Publisher.allocate(); publishing it would commit " + "the newer reservation"); + } + if (len > s.max_size()) { throw nb::value_error( "publish(len) exceeds slot max_size"); } - s.published = true; - return s.publisher->publish(len); + return s.publish(len); }, "len"_a, - "Commit the reserved slot, recording `len` bytes as the " - "payload size. Returns the number of rings the sample was " - "delivered to. After this call, any NEW memoryview(slot) " - "fails with BufferError.") + "Publish len bytes. Returns the number of rings delivered to. " + "Further memoryview(slot) requests raise BufferError.") .def("__len__", - [](PyAllocatedSlot const& s) -> std::size_t { return s.max_size; }) + [](AllocatedSlot const& s) -> std::size_t { return s.max_size(); }) .def_prop_ro("max_size", - [](PyAllocatedSlot const& s) -> std::size_t { return s.max_size; }) - .def_prop_ro("published", - [](PyAllocatedSlot const& s) -> bool { return s.published; }) - .def("__repr__", [](PyAllocatedSlot const& s) + [](AllocatedSlot const& s) -> std::size_t { return s.max_size(); }) + .def_prop_ro("published", &AllocatedSlot::published) + .def_prop_ro("valid", &AllocatedSlot::valid, + "False after publish() or another Publisher.allocate(). " + "Existing buffers cannot be revoked; stop using them before " + "either call.") + .def("__repr__", [](AllocatedSlot const& s) { + char const* published = "False"; + if (s.published()) + { + published = "True"; + } return std::string{"AllocatedSlot(max_size="} + - std::to_string(s.max_size) + - ", published=" + (s.published ? "True" : "False") + ")"; + std::to_string(s.max_size()) + + ", published=" + published + ")"; }); - // ------------------------------------------------------------------- // Publisher - // ------------------------------------------------------------------- - // Publisher holds raw pointers into the SharedRegion's mmap. - // keep_alive<1, 2>: arg 2 (region) must stay alive while arg 1 - // (self) is alive — otherwise the mmap could be unmapped before - // the Publisher's destructor runs, producing a segfault. + // keep_alive<1, 2> keeps the region mapped until the publisher is destroyed. nb::class_(m, "Publisher") .def(nb::init(), "region"_a, nb::keep_alive<1, 2>()) @@ -735,9 +600,7 @@ namespace kickmsg { return static_cast(rc); } - // C++ returns negative errno-style codes; translate to - // Python exceptions so callers don't silently drop - // messages by ignoring a "falsy" negative return. + // Translate negative errno returns into Python exceptions. int err = -rc; if (err == EMSGSIZE) { @@ -761,26 +624,27 @@ namespace kickmsg "the message exceeds max_payload_size, BlockingIOError if " "the slot pool is exhausted, OSError on other failures.") .def("allocate", - [](Publisher& p) -> std::optional + [](Publisher& p) -> std::optional { - auto a = p.allocate(); - if (a.data == nullptr) + auto slot = p.allocate(); + if (not slot.valid()) { return std::nullopt; } - return PyAllocatedSlot{p, a.data, a.max_size}; + return slot; }, - // keep_alive<0, 1>: the returned AllocatedSlot (arg 0) - // must pin the Publisher (arg 1 = self). Memoryviews - // obtained from the slot in turn pin the AllocatedSlot - // (via Py_buffer::obj), so the full chain is - // memoryview → AllocatedSlot → Publisher → SharedRegion. + // Keep the publisher alive while the reservation handle exists. nb::keep_alive<0, 1>(), "Reserve a slot sized to max_payload_size and return an " "AllocatedSlot. Use memoryview(slot) or numpy.asarray(slot) " "to write up to slot.max_size bytes in place (zero-copy), " "then call slot.publish(n) with the actual number of bytes " - "written. Returns None if the pool is exhausted.") + "written. Returns None if the pool is exhausted. Supersedes " + "any previous reservation: an earlier AllocatedSlot becomes " + "invalid and refuses publish() and new memoryview() requests. " + "Buffers already exported from it cannot be revoked: writing " + "through one after this call corrupts whichever reservation " + "now holds the slot.") .def_prop_ro("dropped", &Publisher::dropped, "Per-ring delivery drops (CAS contention or pool exhaustion).") .def("__repr__", [](Publisher const& p) @@ -789,12 +653,9 @@ namespace kickmsg std::to_string(p.dropped()) + ")"; }); - // ------------------------------------------------------------------- // Subscriber - // ------------------------------------------------------------------- - // Same lifetime rule as Publisher — region's mmap must outlive - // the Subscriber. + // Keep the region alive while the subscriber exists. nb::class_(m, "Subscriber") .def(nb::init(), "region"_a, nb::keep_alive<1, 2>()) @@ -831,9 +692,7 @@ namespace kickmsg "timeout"_a, "Blocking receive with timeout (timedelta). Releases the GIL while " "waiting. Returns bytes on success, None on timeout.") - // keep_alive<0, 1>: the returned SampleView (arg 0) must keep - // the Subscriber (arg 1 = self) alive — the view dereferences - // mmap pointers owned transitively by self on destruction. + // Keep the subscriber and its mapping alive while the view exists. .def("try_receive_view", [](Subscriber& s) -> std::optional { return s.try_receive_view(); }, @@ -845,13 +704,7 @@ namespace kickmsg [](Subscriber& s, nanoseconds timeout) -> std::optional { - // Scope the GIL release tightly around the blocking - // wait, matching receive() above. The C++ return value - // is pure C++ (no Python state), so strictly speaking - // the GIL only needs to be released for the futex - // wait itself — but keeping the scope explicit avoids - // any future-footgun if the Python-conversion path - // ever touches CPython state before reacquisition. + // Reacquire the GIL before converting the C++ result to Python. std::optional result; { nb::gil_scoped_release release; @@ -887,13 +740,9 @@ namespace kickmsg return nb::bytes(reinterpret_cast(sample->data()), sample->len()); }); - // ------------------------------------------------------------------- // BroadcastHandle - // ------------------------------------------------------------------- - // BroadcastHandle — Publisher/Subscriber are move-only, so the - // fields are exposed read-only. The handle itself is the owner; - // callers use .pub / .sub as references. + // Move-only fields are exposed by reference; the handle owns them. nb::class_(m, "BroadcastHandle") .def_prop_ro("pub", [](BroadcastHandle& h) -> Publisher& { return h.pub; }, @@ -906,16 +755,9 @@ namespace kickmsg return std::string{"BroadcastHandle(pub=Publisher, sub=Subscriber)"}; }); - // ------------------------------------------------------------------- // Node - // ------------------------------------------------------------------- - - // Node-returned Publisher/Subscriber/BroadcastHandle all point - // into SharedRegion objects stored inside the Node itself. - // keep_alive<0, 1>: the return value (0) pins the Node (1 = self). - // ------------------------------------------------------------------- - // Blackboard - // ------------------------------------------------------------------- + + // Returned handles keep the Node and its regions alive. nb::class_(m, "BlackboardConfig") .def(nb::init<>()) diff --git a/python/kickmsg/_native.pyi b/python/kickmsg/_native.pyi index c7a1e8a..e5cd7ec 100644 --- a/python/kickmsg/_native.pyi +++ b/python/kickmsg/_native.pyi @@ -380,9 +380,10 @@ class AllocatedSlot: Release the buffer object that exposes the underlying memory of the object. """ - def publish(self) -> int: + def publish(self, len: int) -> int: """ - Commit the reserved slot. Returns the number of rings the sample was delivered to. After this call, any NEW memoryview(slot) fails with BufferError. + Publish len bytes. Returns the number of rings delivered to. + Further memoryview(slot) requests raise BufferError. """ def __len__(self) -> int: ... @@ -390,6 +391,13 @@ class AllocatedSlot: @property def published(self) -> bool: ... + @property + def valid(self) -> bool: + """False after publish() or another Publisher.allocate(). + + Existing buffers cannot be revoked; stop using them before either call. + """ + def __repr__(self) -> str: ... class Publisher: @@ -400,9 +408,14 @@ class Publisher: Copy `data` into a slot and publish (atomic convenience). Returns the number of bytes written. Raises ValueError if the message exceeds max_payload_size, BlockingIOError if the slot pool is exhausted, OSError on other failures. """ - def allocate(self, len: int) -> AllocatedSlot | None: + def allocate(self) -> AllocatedSlot | None: """ - Reserve a slot of `len` bytes and return an AllocatedSlot. Use memoryview(slot) or numpy.asarray(slot) to fill it in place (zero-copy), then call slot.publish(). Returns None if the pool is exhausted. + Reserve a slot, or return None if the pool is exhausted. + Fill it through memoryview(slot) or numpy.asarray(slot), then publish(len). + Supersedes the previous reservation: that AllocatedSlot refuses publish() + and new memoryview() requests. Buffers already exported from it cannot be + revoked; writing through one after this call corrupts whichever + reservation now holds the slot. """ @property diff --git a/src/Blackboard.cc b/src/Blackboard.cc index d366168..7fe6bc3 100644 --- a/src/Blackboard.cc +++ b/src/Blackboard.cc @@ -1,5 +1,6 @@ #include "kickmsg/Blackboard.h" +#include #include #include @@ -10,9 +11,6 @@ #include "kickmsg/os/Time.h" -#define KICKMSG_BB_NOINLINE __attribute__((noinline)) - - namespace kickmsg { namespace @@ -33,30 +31,58 @@ namespace kickmsg } } - /// Copy a value payload. A reader overtaken by CELLS_PER_KEY writes - /// races the writer's copy here and discards the result; the payload - /// race lives in this function alone, and tests/tsan.supp names it. - KICKMSG_BB_NOINLINE void bb_copy_payload(void* dst, void const* src, std::size_t len) + // Payload and key bytes are read without the board lock while a writer may be + // rewriting them, so both live in std::atomic words and are only ever + // moved as whole relaxed words: an overlap the publish/tenancy re-check later + // discards is then not a data race, and no two accesses differ in size. The + // fences around each copy are what order it. + static_assert(sizeof(BlackboardCell) % sizeof(uint64_t) == 0 and CACHE_LINE % sizeof(uint64_t) == 0, + "a zero-padded last payload word must stay inside its cell"); + + void store_words(std::atomic* shared, void const* src, std::size_t len) { - std::memcpy(dst, src, len); + auto const* in = static_cast(src); + std::size_t full = len / sizeof(uint64_t); + for (std::size_t w = 0; w < full; ++w) + { + uint64_t word; + std::memcpy(&word, in + w * sizeof(uint64_t), sizeof(word)); + shared[w].store(word, std::memory_order_relaxed); + } + std::size_t rest = len % sizeof(uint64_t); + if (rest != 0) + { + uint64_t word = 0; + std::memcpy(&word, in + full * sizeof(uint64_t), rest); + shared[full].store(word, std::memory_order_relaxed); + } } - /// Compare a stored key against \p key. Reader::resolve() runs this - /// unlocked against claim_free_slot()'s copy_field, the board's second - /// suppressed race; declare() runs it under the board lock. - KICKMSG_BB_NOINLINE bool bb_key_equals(char const* stored, char const* key, - std::size_t key_len) + void load_words(void* dst, std::atomic const* shared, std::size_t len) { - return ::strnlen(stored, blackboard::KEY_MAX) == key_len - and std::memcmp(stored, key, key_len) == 0; + auto* out = static_cast(dst); + std::size_t full = len / sizeof(uint64_t); + for (std::size_t w = 0; w < full; ++w) + { + uint64_t word = shared[w].load(std::memory_order_relaxed); + std::memcpy(out + w * sizeof(uint64_t), &word, sizeof(word)); + } + std::size_t rest = len % sizeof(uint64_t); + if (rest != 0) + { + uint64_t word = shared[full].load(std::memory_order_relaxed); + std::memcpy(out + full * sizeof(uint64_t), &word, rest); + } } - /// Runs unlocked against claim_free_slot()'s copy_field, the race - /// bb_key_equals also carries; the caller's tenancy re-check discards a - /// torn copy. - KICKMSG_BB_NOINLINE std::string bb_read_key(char const* stored, std::size_t size) + /// Compare a stored key against \p key. Runs unlocked in Reader::resolve(); + /// a torn copy costs a retry once the tenancy re-check sees the entry move. + bool key_equals(BlackboardEntry const* entry, char const* key, std::size_t key_len) { - return std::string(stored, ::strnlen(stored, size)); + char copy[blackboard::KEY_MAX]; + load_words(copy, entry->key, sizeof(copy)); + return ::strnlen(copy, sizeof(copy)) == key_len + and std::memcmp(copy, key, key_len) == 0; } std::size_t stride_for(std::size_t max_value_size) @@ -75,14 +101,15 @@ namespace kickmsg /// The value limit is the creator's configured size, never the padded /// stride: handing out the alignment slack would let one peer write /// more than its correctly-sized readers can hold. - std::size_t value_capacity(BlackboardHeader const* header) + blackboard::Geometry make_geometry(uint32_t capacity, std::size_t max_value_size) { - return static_cast(header->max_value_size); - } - - std::size_t value_stride(BlackboardHeader const* header) - { - return stride_for(static_cast(header->max_value_size)); + blackboard::Geometry geometry; + geometry.capacity = capacity; + geometry.max_value_size = max_value_size; + geometry.value_stride = stride_for(max_value_size); + geometry.values_offset = sizeof(BlackboardHeader) + + static_cast(capacity) * sizeof(BlackboardEntry); + return geometry; } void copy_field(char* dst, std::size_t dst_size, char const* src) @@ -156,7 +183,7 @@ namespace kickmsg return make_token(pid, live.starttime) != token; } - void repair_board(void* base, BlackboardHeader* header); + void repair_board(void* base, uint32_t capacity); /// Returns false if the wait ran out, which means a live holder. /// `budget` bounds yields and `limit` bounds wall clock; zero disables @@ -164,8 +191,8 @@ namespace kickmsg /// /// An abandoned lock transfers directly from the dead holder's token /// to ours, never through zero, so nothing can slip in mid-repair. - bool board_lock(void* base, BlackboardHeader* header, uint64_t my_token, - int budget, nanoseconds limit) + bool board_lock(void* base, BlackboardHeader* header, uint32_t capacity, + uint64_t my_token, int budget, nanoseconds limit) { nanoseconds start = monotonic_ns(); for (int attempt = 0; budget == 0 or attempt < budget; ++attempt) @@ -202,7 +229,7 @@ namespace kickmsg expected, my_token, std::memory_order_acq_rel, std::memory_order_relaxed)) { - repair_board(base, header); + repair_board(base, capacity); return true; } } @@ -226,15 +253,15 @@ namespace kickmsg /// The token is always the caller's own, so the guard derives it /// rather than taking one -- a parameter that can only be passed /// one way is a hazard, not a knob. - BoardGuard(void* base, BlackboardHeader* header, int budget) - : h_{header} - , held_{board_lock(base, header, self_token(), budget, nanoseconds::zero())} + BoardGuard(void* base, uint32_t capacity, int budget) + : h_{static_cast(base)} + , held_{board_lock(base, h_, capacity, self_token(), budget, nanoseconds::zero())} { } - BoardGuard(void* base, BlackboardHeader* header, nanoseconds limit) - : h_{header} - , held_{board_lock(base, header, self_token(), 0, limit)} + BoardGuard(void* base, uint32_t capacity, nanoseconds limit) + : h_{static_cast(base)} + , held_{board_lock(base, h_, capacity, self_token(), 0, limit)} { } @@ -298,9 +325,9 @@ namespace kickmsg return false; } - void repair_board(void* base, BlackboardHeader* header) + void repair_board(void* base, uint32_t capacity) { - for (uint32_t i = 0; i < header->capacity; ++i) + for (uint32_t i = 0; i < capacity; ++i) { normalize_entry(bb_entry_at(base, i)); } @@ -325,7 +352,7 @@ namespace kickmsg bool key_matches(BlackboardEntry const* entry, char const* key, std::size_t key_len) { - return bb_key_equals(entry->key, key, key_len); + return key_equals(entry, key, key_len); } /// Fingerprint of the RAW (namespace, name) pair, each component @@ -381,20 +408,31 @@ namespace kickmsg return reinterpret_cast(bytes + static_cast(idx) * sizeof(BlackboardEntry)); } - BlackboardCell* bb_cell_at(void* base, uint32_t idx, uint64_t parity) + BlackboardCell* bb_cell_at(void* base, blackboard::Geometry const& geometry, uint32_t idx, uint64_t parity) { - auto const* header = static_cast(base); - std::size_t values = sizeof(BlackboardHeader) - + static_cast(header->capacity) * sizeof(BlackboardEntry); std::size_t cell = (static_cast(idx) * blackboard::CELLS_PER_KEY + static_cast(parity & CELL_MASK)) - * value_stride(header); - return reinterpret_cast(static_cast(base) + values + cell); + * geometry.value_stride; + return reinterpret_cast(static_cast(base) + geometry.values_offset + cell); } - uint8_t* bb_cell_payload(BlackboardCell* cell) + std::atomic* bb_cell_words(BlackboardCell* cell) { - return reinterpret_cast(cell) + sizeof(BlackboardCell); + return reinterpret_cast*>(reinterpret_cast(cell) + sizeof(BlackboardCell)); + } + + void bb_store_key(BlackboardEntry* entry, char const* key, std::size_t len) + { + char copy[blackboard::KEY_MAX] = {}; + std::memcpy(copy, key, std::min(len, sizeof(copy))); + store_words(entry->key, copy, sizeof(copy)); + } + + std::string bb_load_key(BlackboardEntry const* entry) + { + char copy[blackboard::KEY_MAX]; + load_words(copy, entry->key, sizeof(copy)); + return std::string(copy, ::strnlen(copy, sizeof(copy))); } uint64_t bb_config_hash(blackboard::Config const& cfg) @@ -412,9 +450,11 @@ namespace kickmsg , owner_name_{std::move(other.owner_name_)} , base_{other.base_} , size_{other.size_} + , geometry_{other.geometry_} { - other.base_ = nullptr; - other.size_ = 0; + other.base_ = nullptr; + other.size_ = 0; + other.geometry_ = blackboard::Geometry{}; } Blackboard& Blackboard::operator=(Blackboard&& other) noexcept @@ -426,8 +466,10 @@ namespace kickmsg owner_name_ = std::move(other.owner_name_); base_ = other.base_; size_ = other.size_; - other.base_ = nullptr; - other.size_ = 0; + geometry_ = other.geometry_; + other.base_ = nullptr; + other.size_ = 0; + other.geometry_ = blackboard::Geometry{}; } return *this; } @@ -453,6 +495,7 @@ namespace kickmsg header()->identity_hash = cfg.identity; header()->creator_pid = current_pid(); header()->created_at_ns = static_cast(since_epoch().count()); + geometry_ = make_geometry(cfg.capacity, cfg.max_value_size); // MAGIC published last -- openers spin on it with acquire. header()->magic.store(blackboard::MAGIC, std::memory_order_release); @@ -476,10 +519,15 @@ namespace kickmsg { if (header->version != blackboard::VERSION) { - throw std::runtime_error("Blackboard version mismatch on " + shm); + throw VersionMismatch("Blackboard version mismatch on " + shm + + ": stamped by an incompatible kickmsg build; stop its users and unlink it"); } - if (header->total_size < sizeof(BlackboardHeader) - or header->total_size > mapping.size()) + // Each field is read once and only the copies are checked and kept: + // a peer can rewrite the header during or after validation. + uint64_t const total_size = header->total_size; + uint32_t const capacity = header->capacity; + uint64_t const max_value_size = header->max_value_size; + if (total_size < sizeof(BlackboardHeader) or total_size > mapping.size()) { throw std::runtime_error("Blackboard total_size invalid on " + shm); } @@ -487,26 +535,25 @@ namespace kickmsg // pointer computation; a corrupt value would send // snapshot()/read() off the mapping. Bound both // (division-based, so no intermediate can overflow). - std::size_t after_header = static_cast(header->total_size) + std::size_t after_header = static_cast(total_size) - sizeof(BlackboardHeader); - if (header->capacity == 0 or header->capacity > blackboard::MAX_CAPACITY - or header->capacity > after_header / sizeof(BlackboardEntry)) + if (capacity == 0 or capacity > blackboard::MAX_CAPACITY + or capacity > after_header / sizeof(BlackboardEntry)) { throw std::runtime_error("Blackboard capacity exceeds segment on " + shm); } // Bounding max_value_size (not the derived stride) is what // keeps the accepted range identical on both sides: a board // created at exactly MAX_VALUE_SIZE must stay openable. - if (header->max_value_size == 0 - or header->max_value_size > blackboard::MAX_VALUE_SIZE) + if (max_value_size == 0 or max_value_size > blackboard::MAX_VALUE_SIZE) { throw std::runtime_error("Blackboard max_value_size invalid on " + shm); } - std::size_t entries_bytes = static_cast(header->capacity) - * sizeof(BlackboardEntry); - std::size_t after_entries = after_header - entries_bytes; - if (header->capacity > after_entries - / (blackboard::CELLS_PER_KEY * value_stride(header))) + blackboard::Geometry const geometry = + make_geometry(capacity, static_cast(max_value_size)); + std::size_t after_entries = static_cast(total_size) - geometry.values_offset; + if (capacity > after_entries + / (blackboard::CELLS_PER_KEY * geometry.value_stride)) { throw std::runtime_error("Blackboard value area exceeds segment on " + shm); } @@ -523,8 +570,9 @@ namespace kickmsg Blackboard out; out.name_ = shm; out.base_ = mapping.address(); - out.size_ = mapping.size(); - out.shm_ = std::move(mapping); + out.size_ = mapping.size(); + out.geometry_ = geometry; + out.shm_ = std::move(mapping); return out; } } @@ -612,13 +660,13 @@ namespace kickmsg uint32_t Blackboard::capacity() const { require_open(base_); - return header()->capacity; + return geometry_.capacity; } std::size_t Blackboard::max_value_size() const { require_open(base_); - return value_capacity(header()); + return geometry_.max_value_size; } uint64_t Blackboard::change_seq() const @@ -646,7 +694,7 @@ namespace kickmsg throw std::invalid_argument("Blackboard key exceeds KEY_MAX"); } - uint32_t capacity = header()->capacity; + uint32_t capacity = geometry_.capacity; uint64_t key_hash = key_fingerprint(key, key_len); SelfIdentity self = self_identity(); @@ -657,7 +705,7 @@ namespace kickmsg // key" spans the whole board, so scanning for the key and claiming a // slot must be one indivisible step -- otherwise two claimants can each // scan, each see nothing, and each commit. - BoardGuard guard{base_, header(), 4096}; + BoardGuard guard{base_, geometry_.capacity, 4096}; if (not guard) { throw std::runtime_error("Blackboard is busy: could not take the board lock"); @@ -697,7 +745,7 @@ namespace kickmsg uint64_t writes = entry->publish.load(std::memory_order_acquire) >> 1; notify_change(header()); - return Writer(base_, i, tenancy, writes, my_pid, std::string(key, key_len)); + return Writer(base_, geometry_, i, tenancy, writes, my_pid, std::string(key, key_len)); } // Pass 2: under the board lock, pass 1 finding nothing is a proof. @@ -717,7 +765,7 @@ namespace kickmsg // holder returns the entry to Free. entry->state.store(blackboard::Claiming, std::memory_order_release); entry->publish.store(0, std::memory_order_relaxed); - copy_field(entry->key, sizeof(entry->key), key); + bb_store_key(entry, key, key_len); copy_field(entry->owner_node, sizeof(entry->owner_node), owner_node); entry->declared_at_ns.store( static_cast(monotonic_ns().count()), @@ -735,7 +783,7 @@ namespace kickmsg }; uint32_t claimed = claim_free_slot(); - if (claimed == INVALID_SLOT and sweep_locked(header()) > 0) + if (claimed == INVALID_SLOT and sweep_locked() > 0) { // Crash residue can be sitting on the last free slots. claimed = claim_free_slot(); @@ -745,7 +793,7 @@ namespace kickmsg auto* entry = bb_entry_at(base_, claimed); uint64_t tenancy = entry->tenancy.load(std::memory_order_acquire); notify_change(header()); - return Writer(base_, claimed, tenancy, 0, my_pid, std::string(key, key_len)); + return Writer(base_, geometry_, claimed, tenancy, 0, my_pid, std::string(key, key_len)); } throw std::runtime_error("Blackboard is at capacity"); @@ -763,14 +811,15 @@ namespace kickmsg { throw std::invalid_argument("Blackboard key exceeds KEY_MAX"); } - return Reader(base_, std::string(key, key_len)); + return Reader(base_, geometry_, std::string(key, key_len)); } // ---- Writer ---------------------------------------------------------- - Blackboard::Writer::Writer(void* base, uint32_t entry_idx, uint64_t tenancy, - uint64_t writes, uint64_t owner_pid, std::string key) + Blackboard::Writer::Writer(void* base, blackboard::Geometry const& geometry, uint32_t entry_idx, + uint64_t tenancy, uint64_t writes, uint64_t owner_pid, std::string key) : base_{base} + , geometry_{geometry} , entry_idx_{entry_idx} , tenancy_{tenancy} , writes_{writes} @@ -786,6 +835,7 @@ namespace kickmsg Blackboard::Writer::Writer(Writer&& other) noexcept : base_{other.base_} + , geometry_{other.geometry_} , entry_idx_{other.entry_idx_} , tenancy_{other.tenancy_} , writes_{other.writes_} @@ -802,6 +852,7 @@ namespace kickmsg { release(); base_ = other.base_; + geometry_ = other.geometry_; entry_idx_ = other.entry_idx_; tenancy_ = other.tenancy_; writes_ = other.writes_; @@ -826,11 +877,11 @@ namespace kickmsg return std::make_error_code(std::errc::operation_not_permitted); } auto* header = static_cast(base_); - if (entry_idx_ >= header->capacity) + if (entry_idx_ >= geometry_.capacity) { return std::make_error_code(std::errc::bad_file_descriptor); } - if (len > value_capacity(header)) + if (len > geometry_.max_value_size) { return std::make_error_code(std::errc::message_size); } @@ -847,7 +898,7 @@ namespace kickmsg } uint64_t writes = writes_ + 1; - auto* cell = bb_cell_at(base_, entry_idx_, writes); + auto* cell = bb_cell_at(base_, geometry_, entry_idx_, writes); entry->publish.store(2 * writes - 1, std::memory_order_relaxed); // Keeps the write-in-progress store above the payload stores. A @@ -856,7 +907,7 @@ namespace kickmsg if (len > 0) { - bb_copy_payload(bb_cell_payload(cell), data, len); + store_words(bb_cell_words(cell), data, len); } // relaxed: covered by the release fence below and validated by the // reader's publish re-check. @@ -889,10 +940,10 @@ namespace kickmsg std::error_code ec = std::make_error_code(std::errc::bad_file_descriptor); auto* header = static_cast(base_); - if (entry_idx_ < header->capacity) + if (entry_idx_ < geometry_.capacity) { ec = std::make_error_code(std::errc::device_or_resource_busy); - BoardGuard guard{base_, header, RELEASE_LOCK_WAIT}; + BoardGuard guard{base_, geometry_.capacity, RELEASE_LOCK_WAIT}; if (guard) { ec = std::make_error_code(std::errc::state_not_recoverable); @@ -920,8 +971,9 @@ namespace kickmsg // ---- Reader ---------------------------------------------------------- - Blackboard::Reader::Reader(void* base, std::string key) + Blackboard::Reader::Reader(void* base, blackboard::Geometry const& geometry, std::string key) : base_{base} + , geometry_{geometry} , key_hash_{key_fingerprint(key.data(), key.size())} , key_{std::move(key)} { @@ -933,9 +985,7 @@ namespace kickmsg { return false; } - auto const* header = static_cast(base_); - - if (entry_idx_ < header->capacity) + if (entry_idx_ < geometry_.capacity) { auto* entry = bb_entry_at(base_, entry_idx_); if (entry->state.load(std::memory_order_acquire) == blackboard::Active @@ -945,7 +995,7 @@ namespace kickmsg } } - for (uint32_t i = 0; i < header->capacity; ++i) + for (uint32_t i = 0; i < geometry_.capacity; ++i) { auto* entry = bb_entry_at(base_, i); if (entry->state.load(std::memory_order_acquire) != blackboard::Active) @@ -961,7 +1011,7 @@ namespace kickmsg uint64_t tenancy = entry->tenancy.load(std::memory_order_acquire); // key_hash is only a pre-filter: it cannot survive a collision, // so the bytes must actually match. - if (not bb_key_equals(entry->key, key_.data(), key_.size())) + if (not key_equals(entry, key_.data(), key_.size())) { continue; } @@ -982,8 +1032,7 @@ namespace kickmsg { return result; } - auto* header = static_cast(base_); - std::size_t limit = value_capacity(header); + std::size_t limit = geometry_.max_value_size; for (int retry = 0; retry < blackboard::READ_RETRY_BUDGET; ++retry) { @@ -1006,7 +1055,7 @@ namespace kickmsg return result; } - auto* cell = bb_cell_at(base_, entry_idx_, writes); + auto* cell = bb_cell_at(base_, geometry_, entry_idx_, writes); // relaxed: ordered by the acquire load of publish above and // validated by the re-check below. std::size_t len = cell->value_len.load(std::memory_order_relaxed); @@ -1018,7 +1067,7 @@ namespace kickmsg bool fits = len <= capacity; if (fits and len > 0) { - bb_copy_payload(out, bb_cell_payload(cell), len); + load_words(out, bb_cell_words(cell), len); } uint64_t stamp = cell->updated_at_ns.load(std::memory_order_relaxed); @@ -1133,7 +1182,7 @@ namespace kickmsg std::vector Blackboard::snapshot() const { require_open(base_); - uint32_t capacity = header()->capacity; + uint32_t capacity = geometry_.capacity; std::vector out; std::vector starttimes; @@ -1145,8 +1194,7 @@ namespace kickmsg // Serialized: a takeover rewrites owner_node with no seqlock over // those bytes, so an unlocked listing can return torn text. { - BoardGuard guard{const_cast(base_), - const_cast(header()), 1024}; + BoardGuard guard{const_cast(base_), capacity, 1024}; if (not guard) { throw std::runtime_error("Blackboard is busy: could not take the board lock"); @@ -1195,7 +1243,7 @@ namespace kickmsg std::vector* value) const { BlackboardEntry* entry = bb_entry_at(base_, i); - std::size_t const limit = value_capacity(header()); + std::size_t const limit = geometry_.max_value_size; for (int retry = 0; retry < blackboard::READ_RETRY_BUDGET; ++retry) { @@ -1205,14 +1253,14 @@ namespace kickmsg } uint64_t tenancy_before = entry->tenancy.load(std::memory_order_acquire); - out = EntryRead{bb_read_key(entry->key, sizeof(entry->key)), 0, 0, 0}; + out = EntryRead{bb_load_key(entry), 0, 0, 0}; uint64_t publish_before = entry->publish.load(std::memory_order_acquire); uint64_t writes = publish_before >> 1; out.update_count = writes; if (writes > 0) { - auto* cell = bb_cell_at(base_, i, writes); + auto* cell = bb_cell_at(base_, geometry_, i, writes); // relaxed: ordered by the acquire load of publish above and // validated by the re-check below. std::size_t len = cell->value_len.load(std::memory_order_relaxed); @@ -1228,7 +1276,7 @@ namespace kickmsg value->resize(len); if (len > 0) { - bb_copy_payload(value->data(), bb_cell_payload(cell), len); + load_words(value->data(), bb_cell_words(cell), len); } } } @@ -1262,7 +1310,7 @@ namespace kickmsg std::vector Blackboard::keys(std::string_view prefix) const { require_open(base_); - uint32_t const capacity = header()->capacity; + uint32_t const capacity = geometry_.capacity; std::vector out; EntryRead entry; @@ -1286,7 +1334,7 @@ namespace kickmsg Blackboard::read_all(std::string_view prefix) const { require_open(base_); - uint32_t const capacity = header()->capacity; + uint32_t const capacity = geometry_.capacity; std::unordered_map> out; EntryRead entry; @@ -1306,10 +1354,10 @@ namespace kickmsg return out; } - uint32_t Blackboard::sweep_locked(BlackboardHeader* header) + uint32_t Blackboard::sweep_locked() { uint32_t reclaimed = 0; - for (uint32_t i = 0; i < header->capacity; ++i) + for (uint32_t i = 0; i < geometry_.capacity; ++i) { auto* entry = bb_entry_at(base_, i); if (normalize_entry(entry)) @@ -1340,13 +1388,13 @@ namespace kickmsg { require_open(base_); - BoardGuard guard{base_, header(), 4096}; + BoardGuard guard{base_, geometry_.capacity, 4096}; if (not guard) { return 0; } - uint32_t reclaimed = sweep_locked(header()); + uint32_t reclaimed = sweep_locked(); if (reclaimed > 0) { notify_change(header()); diff --git a/src/Node.cc b/src/Node.cc index b886371..8e75ec4 100644 --- a/src/Node.cc +++ b/src/Node.cc @@ -67,6 +67,26 @@ namespace kickmsg h = hash::fnv1a_64(s, h); return hash::fnv1a_64(s.size(), h); } + + /// Cache hits must validate identity and config because sanitized names can collide. + void check_cached_identity(SharedRegion const& r, std::string const& shm_name, + uint64_t expected_identity) + { + uint64_t stamped = r.header()->identity_hash; + if (expected_identity != 0 and stamped != 0 and stamped != expected_identity) + { + throw std::runtime_error(std::string{"Identity mismatch on existing region (shm name collision): "} + shm_name); + } + } + + void check_cached_config(SharedRegion const& r, std::string const& shm_name, + channel::Type type, channel::Config const& cfg) + { + if (r.header()->config_hash != compute_config_hash(type, cfg)) + { + throw std::runtime_error(std::string{"Config mismatch on existing region: "} + shm_name); + } + } } void Node::touch_registry(std::string const& shm_name, @@ -100,11 +120,8 @@ namespace kickmsg { if (it->second.role != role and it->second.role != registry::Both) { - // Upgrade to Both via dereg + re-register; brief - // visibility gap during the swap is acceptable since - // the registry is diagnostic-only. On fill-failure - // of the Both re-register, fall back to re-registering - // the original role to keep at least partial discovery. + // Role changes use deregister/register. If the registry fills, + // try to restore the previous role. reg.deregister(it->second.slot_index); uint32_t slot = reg.register_participant( shm_name, topic_name, channel_type, kind, @@ -138,6 +155,11 @@ namespace kickmsg } registry_slots_[shm_name] = RegistrySlot{slot, role}; } + catch (VersionMismatch const&) + { + // Not best-effort: a namespace mixing kickmsg versions cannot work. + throw; + } catch (std::exception const& e) { // Latch to avoid stderr spam on a Node that brings up many topics. @@ -154,11 +176,11 @@ namespace kickmsg { auto shm_name = make_topic_name(topic); auto topic_path = with_leading_slash(topic); - // Guard the create: a second advertise() would re-run create() - // (unlink + fresh object), orphaning the live segment for existing - // Publishers and remote peers. + // Reuse cached regions; create() would replace the shared object. if (auto* r = find_region(shm_name)) { + check_cached_identity(*r, shm_name, make_topic_identity(topic)); + check_cached_config(*r, shm_name, channel::PubSub, cfg); touch_registry(shm_name, topic_path, channel::PubSub, registry::Pubsub, registry::Publisher); return Publisher(*r, backend); @@ -179,6 +201,7 @@ namespace kickmsg auto topic_path = with_leading_slash(topic); if (auto* r = find_region(shm_name)) { + check_cached_identity(*r, shm_name, make_topic_identity(topic)); touch_registry(shm_name, topic_path, channel::PubSub, registry::Pubsub, registry::Subscriber); return Subscriber(*r); @@ -202,6 +225,8 @@ namespace kickmsg { if (auto* r = find_region(shm_name)) { + check_cached_identity(*r, shm_name, cfg.identity); + check_cached_config(*r, shm_name, channel_type, cfg); touch_registry(shm_name, topic_path, channel_type, kind, role); return Handle(*r, std::forward(args)...); } @@ -239,6 +264,8 @@ namespace kickmsg auto topic_path = with_leading_slash(channel); if (auto* r = find_region(shm_name)) { + check_cached_identity(*r, shm_name, make_broadcast_identity(channel)); + check_cached_config(*r, shm_name, channel::Broadcast, cfg); touch_registry(shm_name, topic_path, channel::Broadcast, registry::Broadcast, registry::Both); return BroadcastHandle{Publisher{*r, backend}, Subscriber{*r}}; @@ -261,10 +288,11 @@ namespace kickmsg mbx_cfg.identity = make_mailbox_identity(name_.c_str(), tag); auto shm_name = make_mailbox_name(name_.c_str(), tag); auto topic_path = mailbox_topic(name_.c_str(), tag); - // Guard the create (see advertise); the duplicate claim then fails - // loudly in the Subscriber ctor instead of splitting the mailbox. + // Reuse the region so a duplicate subscriber fails without replacing it. if (auto* r = find_region(shm_name)) { + check_cached_identity(*r, shm_name, mbx_cfg.identity); + check_cached_config(*r, shm_name, channel::PubSub, mbx_cfg); touch_registry(shm_name, topic_path, channel::PubSub, registry::Mailbox, registry::Subscriber); return Subscriber(*r); @@ -285,6 +313,7 @@ namespace kickmsg auto topic_path = mailbox_topic(owner_node, tag); if (auto* r = find_region(shm_name)) { + check_cached_identity(*r, shm_name, make_mailbox_identity(owner_node, tag)); touch_registry(shm_name, topic_path, channel::PubSub, registry::Mailbox, registry::Publisher); return Publisher(*r, backend); @@ -326,9 +355,7 @@ namespace kickmsg Blackboard& Node::blackboard(char const* name, blackboard::Config const& cfg) { - // Keyed by the LOGICAL name: "a:b" and "a b" sanitize to one shm - // path, so keying by that path would let the second call hit the - // cache and bypass the identity check. + // Use the logical name as the key; distinct names can sanitize to one path. std::string logical = name; auto path = with_leading_slash(name); @@ -413,10 +440,8 @@ namespace kickmsg std::string Node::make_topic_name(char const* topic) const { - // namespace_ is pre-sanitized in the ctor; topic is user-supplied on - // each call and may be a ROS-style "/a/b/c" path. compose_shm_name - // handles the platform shm-name limit (hash on macOS, readable on - // Linux, throw on overflow). + // namespace_ is already sanitized; topic may contain path separators. + // compose_shm_name applies platform name limits. return compose_shm_name(namespace_, sanitize_shm_component(topic, "topic")); } diff --git a/src/Publisher.cc b/src/Publisher.cc index 52480cf..bb99e88 100644 --- a/src/Publisher.cc +++ b/src/Publisher.cc @@ -26,36 +26,99 @@ namespace kickmsg if (pending_slot_ != INVALID_SLOT) { // Return the uncommitted slot to the free-stack. - auto* slot = slot_at(base_, header_, pending_slot_); + auto* slot = slot_at(base_, geometry_, pending_slot_); treiber_push(header_->free_top, slot, pending_slot_); pending_slot_ = INVALID_SLOT; } } - Allocation Publisher::allocate() + AllocatedSlot::~AllocatedSlot() + { + if (valid()) + { + publisher_->release_pending(); + } + } + + AllocatedSlot::AllocatedSlot(AllocatedSlot&& other) noexcept + : publisher_{other.publisher_} + , data_{other.data_} + , max_size_{other.max_size_} + , id_{other.id_} + , published_{other.published_} + { + other.publisher_ = nullptr; + } + + AllocatedSlot& AllocatedSlot::operator=(AllocatedSlot&& other) noexcept + { + if (this != &other) + { + if (valid()) + { + publisher_->release_pending(); + } + publisher_ = other.publisher_; + data_ = other.data_; + max_size_ = other.max_size_; + id_ = other.id_; + published_ = other.published_; + other.publisher_ = nullptr; + } + return *this; + } + + bool AllocatedSlot::valid() const + { + return publisher_ != nullptr and not published_ + and publisher_->reservation_ == id_; + } + + std::size_t AllocatedSlot::write(void const* src, std::size_t len) + { + if (not valid() or len > max_size_) + { + return 0; + } + std::memcpy(data_, src, len); + return len; + } + + std::size_t AllocatedSlot::publish(std::size_t len) + { + if (not valid()) + { + return 0; + } + // Consumed either way: an oversized length recycles the slot. + published_ = true; + return publisher_->publish(len); + } + + AllocatedSlot Publisher::allocate() { // Release any previously allocated but unpublished slot. release_pending(); - uint32_t slot_idx = treiber_pop(header_->free_top, base_, header_); + // Bump even if the pop fails: the previous slot is already back in the pool. + ++reservation_; + + uint32_t slot_idx = treiber_pop(header_->free_top, base_, geometry_); if (slot_idx == INVALID_SLOT) { - return Allocation{nullptr, 0}; + return AllocatedSlot{}; } pending_slot_ = slot_idx; - auto* slot = slot_at(base_, header_, slot_idx); - return Allocation{slot_data(slot), header_->slot_data_size}; + auto* slot = slot_at(base_, geometry_, slot_idx); + return AllocatedSlot{*this, slot_data(slot), geometry_.slot_data_size, reservation_}; } std::size_t Publisher::publish(std::size_t len) { - // Oversized len would otherwise be truncated by the uint32_t store - // into payload_len -- possibly to a small VALID length, bypassing - // the subscriber's bound check and delivering a silently wrong - // length. Recycle the pending slot and report zero deliveries. - if (len > header_->slot_data_size) + // Reject oversized lengths before narrowing to uint32_t. + if (len > geometry_.slot_data_size) { release_pending(); return 0; @@ -68,27 +131,29 @@ namespace kickmsg uint32_t slot_idx = pending_slot_; pending_slot_ = INVALID_SLOT; - auto* slot = slot_at(base_, header_, slot_idx); - uint64_t capacity = header_->sub_ring_capacity; + auto* slot = slot_at(base_, geometry_, slot_idx); + uint64_t capacity = geometry_.sub_ring_capacity; + + // Relaxed: nobody can reach this slot until a commit below publishes + // it, and every commit is a release-CAS that carries this store. + slot->payload_len.store(static_cast(len), std::memory_order_relaxed); // Pre-set refcount to max_subs before publishing to any ring, // so a fast eviction on ring[k] cannot free the slot before // we finish publishing to ring[k+1]. - slot->refcount.store(static_cast(header_->max_subs), + slot->refcount.store(static_cast(geometry_.max_subs), std::memory_order_release); std::size_t delivered = 0; uint32_t excess = 0; bool carrier = false; - for (uint32_t i = 0; i < header_->max_subs; ++i) + for (uint32_t i = 0; i < geometry_.max_subs; ++i) { - auto* ring = sub_ring_at(base_, header_, i); + auto* ring = sub_ring_at(base_, geometry_, i); - // Relaxed pre-check: skip obviously non-Live rings without - // any RMW atomic. Stale reads are safe: - // - Sees Free, actually Live: miss one delivery (acceptable). - // - Sees Live, actually Draining: CAS catches it below. + // Relaxed pre-check: a stale Free may miss one delivery; + // a stale Live is checked by the admission CAS. uint32_t snapshot = ring->state_flight.load(std::memory_order_relaxed); if (ring::get_state(snapshot) != ring::Live) { @@ -96,9 +161,7 @@ namespace kickmsg continue; } - // CAS admission: atomically verify state==Live and increment - // in_flight. All ordering is on a single variable, so - // acquire/release is sufficient (no seq_cst needed). + // Check Live and increment in_flight in one acquire-release CAS. uint32_t old = snapshot; bool admitted = false; while (true) @@ -126,12 +189,9 @@ namespace kickmsg // Admitted: in_flight incremented, state is Live. - // Claim a position in this ring. fetch_add is unconditional: - // no CAS retry loop, O(1) under contention, and compiles to - // a single LDADDAL on AArch64 with LSE atomics. uint64_t pos = ring->write_pos.fetch_add(1, std::memory_order_acq_rel); - uint64_t idx = pos & header_->sub_ring_mask; + uint64_t idx = pos & geometry_.sub_ring_mask; auto* entries = ring_entries(ring); auto& e = entries[idx]; @@ -157,16 +217,13 @@ namespace kickmsg wait = wait_for_commit(e, prev_seq, commit_timeout_); } - // Two-phase commit: CAS to our lock, write data, CAS-commit. - // A repairer's theft makes both CASes fail instead of being - // blind-stored over. + // Lock and commit by CAS so a repairer can revoke this position. uint64_t const lock_val = seq_lock(pos); uint64_t observed = 0; if (pos >= capacity) { observed = wait.last_seq; } - bool prev_was_skip = false; bool locked = false; for (int attempt = 0; attempt < 64; ++attempt) { @@ -178,7 +235,6 @@ namespace kickmsg if (e.sequence.compare_exchange_weak(expected, lock_val, std::memory_order_acquire, std::memory_order_relaxed)) { - prev_was_skip = seq_is_skip(observed); locked = true; break; } @@ -201,40 +257,51 @@ namespace kickmsg continue; } - // Release the previous occupant's slot from the post-lock read - // (sees even a commit that landed after our wait timed out; a - // drain's INVALID marker fails the bound check). Never for a - // skip predecessor (untrustworthy metadata), never below one - // wrap (zero-init slot_idx would read as valid slot 0). - if (pos >= capacity and not prev_was_skip) + // This early check avoids work; the metadata CAS below guards late writes. + if (e.sequence.load(std::memory_order_acquire) != lock_val) { - uint32_t prev_slot = e.slot_idx.load(std::memory_order_acquire); - if (prev_slot < header_->pool_size) + carrier |= abandon_delivery(ring); + ++excess; + continue; + } + + // Replace only an older position's claim, so a late writer cannot + // overwrite a newer entry. + uint64_t const my_meta = meta_pack(pos, slot_idx); + uint64_t old_meta = e.meta.load(std::memory_order_acquire); + bool taken = false; + while (meta_precedes(old_meta, pos)) + { + if (e.meta.compare_exchange_weak(old_meta, my_meta, + std::memory_order_acq_rel, std::memory_order_acquire)) { - release_slot(prev_slot); + taken = true; + break; } } - - // Theft guard: a repairer may have stolen our lock during a - // stall; storing data now would tear the repaired entry. - if (e.sequence.load(std::memory_order_acquire) != lock_val) + if (not taken) { + // A newer publisher owns this entry: our slot reference is + // still ours to drop. carrier |= abandon_delivery(ring); ++excess; continue; } - e.slot_idx.store(slot_idx, std::memory_order_relaxed); - e.payload_len.store(static_cast(len), std::memory_order_relaxed); + // The entry now owns our reference. We must release its previous claim. + uint32_t prev_biased = meta_slot_biased(old_meta); + if (prev_biased != 0) + { + release_slot(prev_biased - 1); + } - // CAS-commit: fails only on theft after the guard above. - // Release on success publishes the data stores. + // CAS-commit. Release on success publishes the data stores. uint64_t expected_lock = lock_val; if (not e.sequence.compare_exchange_strong(expected_lock, pos + 1, std::memory_order_release, std::memory_order_relaxed)) { + // The entry owns our reference even if commit fails. Do not release it twice. carrier |= abandon_delivery(ring); - ++excess; continue; } @@ -242,21 +309,15 @@ namespace kickmsg ring->state_flight.fetch_sub(ring::IN_FLIGHT_ONE, std::memory_order_release); - // seq_cst fence orders the write_pos fetch_add before the - // has_waiter load: without it a weakly-ordered CPU can read - // has_waiter == 0 stale and skip the wake to a subscriber already - // parked in futex_wait (a lost wakeup until its timeout). Pairs - // with the subscriber's fence. x86's locked RMW already fences, - // which is why this never surfaced on x86. + // Pair with the subscriber's fence: publish write_pos before checking + // has_waiter, so either the subscriber sees the position or we send a wake. std::atomic_thread_fence(std::memory_order_seq_cst); carrier |= wake_ring(ring); ++delivered; } - // Batch release excess refs for all non-delivered rings. - // Safe because: Free rings have no drain to race with, and - // Draining rings where CAS failed never admitted us (in_flight - // was never incremented), so their drain doesn't depend on us. + // Release references that were not transferred to ring entries. + // Any ring admission for these deliveries has already been released. if (excess > 0) { uint32_t prev = slot->refcount.fetch_sub(excess, @@ -278,19 +339,19 @@ namespace kickmsg int32_t Publisher::send(void const* data, std::size_t len) { - if (len > header_->slot_data_size) + if (len > geometry_.slot_data_size) { return -EMSGSIZE; } - auto a = allocate(); - if (a.data == nullptr) + auto slot = allocate(); + if (not slot.valid()) { return -EAGAIN; } - std::memcpy(a.data, data, len); - publish(len); + slot.write(data, len); + slot.publish(len); return static_cast(len); } @@ -298,10 +359,16 @@ namespace kickmsg microseconds timeout) { constexpr int CHECK_INTERVAL = 1024; - nanoseconds start = kickmsg::monotonic_ns(); + // Avoid a clock read when the predecessor is already committed. uint64_t first = e.sequence.load(std::memory_order_acquire); - uint64_t seq = first; + if (not seq_is_locked(first) and seq_pos(first) >= expected_seq) + { + return CommitWait{first, false}; + } + + nanoseconds start = kickmsg::monotonic_ns(); + uint64_t seq = first; int i = 0; while (true) { @@ -355,7 +422,7 @@ namespace kickmsg { return; // at most one wrap behind: normal contention residue } - if (entry_steal_and_clear(e, pos, seq)) + if (entry_steal_and_skip(e, pos, seq)) { header_->steal_count.fetch_add(1, std::memory_order_relaxed); } @@ -365,11 +432,11 @@ namespace kickmsg { // idx is read from a ring entry a peer wrote; a crashed or hostile // publisher can leave it out of range (this also covers INVALID_SLOT). - if (idx >= header_->pool_size) + if (idx >= geometry_.pool_size) { return; } - auto* s = slot_at(base_, header_, idx); + auto* s = slot_at(base_, geometry_, idx); uint32_t prev = s->refcount.fetch_sub(1, std::memory_order_acq_rel); if (prev == 1) { diff --git a/src/Region.cc b/src/Region.cc index c2fff1e..e022d7f 100644 --- a/src/Region.cc +++ b/src/Region.cc @@ -23,6 +23,11 @@ namespace kickmsg { throw std::runtime_error("pool_size must be > 0"); } + // Slot indices ride in Entry::meta's 24-bit field; see types.h. + if (cfg.pool_size > MAX_POOL_SIZE) + { + throw std::runtime_error("pool_size exceeds MAX_POOL_SIZE"); + } if (cfg.max_subscribers == 0) { throw std::runtime_error("max_subscribers must be > 0"); @@ -58,17 +63,24 @@ namespace kickmsg throw std::runtime_error("creator_name exceeds 65535 bytes"); } + // Check additions and alignment before computing strides, which must not wrap. + if (cfg.sub_ring_capacity > (SIZE_MAX - sizeof(SubRingHeader) - CACHE_LINE) / sizeof(Entry)) + { + throw std::runtime_error("Config too large: ring stride overflows"); + } + if (cfg.max_payload_size > SIZE_MAX - sizeof(SlotHeader) - CACHE_LINE) + { + throw std::runtime_error("Config too large: slot stride overflows"); + } + RegionLayout layout; layout.creator_len = static_cast(name_len); layout.header_size = align_up(sizeof(Header) + layout.creator_len, CACHE_LINE); - layout.ring_stride = align_up( - sizeof(SubRingHeader) + cfg.sub_ring_capacity * sizeof(Entry), CACHE_LINE); + layout.ring_stride = align_up(sizeof(SubRingHeader) + cfg.sub_ring_capacity * sizeof(Entry), CACHE_LINE); layout.slot_stride = align_up(sizeof(SlotHeader) + cfg.max_payload_size, CACHE_LINE); layout.sub_rings_offset = layout.header_size; - // Overflow guards: a cfg with huge counts must not wrap total_size - // into a small value that maps a tiny region while publishers and - // subscribers stride off the end. + // Reject sizes that overflow the mapped region length. if (cfg.max_subscribers > (SIZE_MAX - layout.sub_rings_offset) / layout.ring_stride) { throw std::runtime_error("Config too large: subscriber rings overflow"); @@ -91,6 +103,18 @@ namespace kickmsg { std::memset(base(), 0, total_size); + // Built from the computed layout, never read back from shared memory. + geometry_.sub_rings_offset = sub_rings_offset; + geometry_.sub_ring_stride = ring_stride; + geometry_.sub_ring_capacity = cfg.sub_ring_capacity; + geometry_.sub_ring_mask = cfg.sub_ring_capacity - 1; + geometry_.pool_offset = pool_offset; + geometry_.slot_stride = slot_stride; + geometry_.pool_size = cfg.pool_size; + geometry_.slot_data_size = cfg.max_payload_size; + geometry_.max_subs = cfg.max_subscribers; + geometry_.commit_timeout_us = static_cast(cfg.commit_timeout.count()); + auto* h = header(); h->version = VERSION; h->channel_type = type; @@ -111,14 +135,8 @@ namespace kickmsg h->creator_name_len = creator_len; std::memcpy(header_creator_name(h), creator_name, creator_len); - // Optional payload schema: publish directly before the magic store. - // No claim state machine needed at creation because (a) we are the - // only writer -- no concurrent claimant can race -- and (b) the - // release-store of MAGIC below carries all preceding writes, - // including the memcpy into schema_data and this relaxed store of - // schema_state, across to any reader that acquire-loads MAGIC. - // The relaxed is therefore correct; do NOT "fix" it to release in - // isolation -- MAGIC is the sole publication fence for this region. + // The creator is the only writer. The release-store of MAGIC publishes + // the schema bytes and relaxed schema_state store. if (cfg.schema.has_value()) { std::memcpy(&h->schema_data, &*cfg.schema, sizeof(SchemaInfo)); @@ -131,14 +149,14 @@ namespace kickmsg for (uint32_t i = 0; i < cfg.pool_size; ++i) { - auto* slot = slot_at(base(), h, i); + auto* slot = slot_at(base(), geometry_, i); slot->refcount = 0; treiber_push(h->free_top, slot, i); } for (uint32_t i = 0; i < cfg.max_subscribers; ++i) { - auto* ring = sub_ring_at(base(), h, i); + auto* ring = sub_ring_at(base(), geometry_, i); ring->state_flight = ring::make_packed(ring::Free); ring->write_pos = 0; ring->dropped_count = 0; @@ -154,139 +172,161 @@ namespace kickmsg namespace { - /// Validate that an already-attached Header has internally - /// consistent geometry. - /// - /// Reject a Header whose geometry fields are not self-consistent. - /// attach_open() trusts caller-supplied bytes, and every offset / - /// stride / count / length below drives later pointer math in - /// Publisher, Subscriber, info() and the repair paths -- junk here - /// means wild pointers. A region kickmsg itself stamped always - /// passes; only corrupt or hostile input fails. open() runs it too - /// as defense in depth. Caller has already checked magic, version, - /// and size >= total_size. - void validate_header_geometry(Header const* h) - { - // channel::None carries no ring geometry and is never stamped by - // create(); rejecting it here is what makes that guarantee hold - // against a corrupt or hostile peer region as well. - if (h->channel_type != channel::PubSub - and h->channel_type != channel::Broadcast) + /// Header fields validated together, each read from shared memory exactly once. + struct HeaderSnapshot + { + Geometry geometry; + channel::Type channel_type; + uint64_t total_size; + uint16_t creator_name_len; + }; + + /// Validate all geometry used for pointer arithmetic. + void validate_snapshot(HeaderSnapshot const& snapshot, std::size_t size) + { + Geometry const& geometry = snapshot.geometry; + if (size < snapshot.total_size) + { + throw std::runtime_error("Buffer smaller than embedded region total_size"); + } + // Only PubSub and Broadcast have ring geometry. + if (snapshot.channel_type != channel::PubSub and + snapshot.channel_type != channel::Broadcast) { throw std::runtime_error("Header geometry: unsupported channel type"); } - if (h->total_size < sizeof(Header)) + if (snapshot.total_size < sizeof(Header)) { throw std::runtime_error( "Header geometry: total_size smaller than Header"); } - // No zero counts or strides -- divide-by-zero protection for - // the bound checks below depends on these, and stamp_new_region - // never produces a zero here. - if (h->max_subs == 0 or h->pool_size == 0 - or h->slot_data_size == 0 or h->sub_ring_capacity == 0 - or h->slot_stride == 0 or h->sub_ring_stride == 0) + // Nonzero counts and strides are required by the divisions below. + if (geometry.max_subs == 0 or + geometry.pool_size == 0 or + geometry.slot_data_size == 0 or + geometry.sub_ring_capacity == 0 or + geometry.slot_stride == 0 or + geometry.sub_ring_stride == 0) { throw std::runtime_error( "Header geometry: zero-cardinality field"); } - if (not is_power_of_two(h->sub_ring_capacity)) + // Slot indices must stay representable in Entry::meta, or a peer + // could name a slot the claim word cannot round-trip. + if (geometry.pool_size > MAX_POOL_SIZE) { - throw std::runtime_error( - "Header geometry: sub_ring_capacity not a power of 2"); + throw std::runtime_error("Header geometry: pool_size exceeds MAX_POOL_SIZE"); } - if (h->sub_ring_mask != h->sub_ring_capacity - 1) + if (not is_power_of_two(geometry.sub_ring_capacity)) { - throw std::runtime_error( - "Header geometry: sub_ring_mask inconsistent with capacity"); + throw std::runtime_error("Header geometry: sub_ring_capacity not a power of 2"); + } + if (geometry.sub_ring_mask != geometry.sub_ring_capacity - 1) + { + throw std::runtime_error("Header geometry: sub_ring_mask inconsistent with capacity"); } // Sub-rings span [sub_rings_offset, pool_offset); pool spans // [pool_offset, total_size). - if (h->sub_rings_offset < sizeof(Header) - or h->sub_rings_offset >= h->pool_offset - or h->pool_offset >= h->total_size) + if (geometry.sub_rings_offset < sizeof(Header) or + geometry.sub_rings_offset >= geometry.pool_offset or + geometry.pool_offset >= snapshot.total_size) { - throw std::runtime_error( - "Header geometry: ring/pool offsets out of range"); + throw std::runtime_error("Header geometry: ring/pool offsets out of range"); } // creator_name tail lives in [sizeof(Header), sub_rings_offset); // bound it there so info() can't read into the ring/pool area. - if (h->creator_name_len > h->sub_rings_offset - sizeof(Header)) + if (snapshot.creator_name_len > geometry.sub_rings_offset - sizeof(Header)) { - throw std::runtime_error( - "Header geometry: creator_name_len exceeds tail"); + throw std::runtime_error("Header geometry: creator_name_len exceeds tail"); + } + + // Shared atomics require cache-line-aligned offsets and strides. + if ((geometry.sub_rings_offset % CACHE_LINE) != 0 or + (geometry.pool_offset % CACHE_LINE) != 0 or + (geometry.sub_ring_stride % CACHE_LINE) != 0 or + (geometry.slot_stride % CACHE_LINE) != 0) + { + throw std::runtime_error("Header geometry: offset or stride not cache-line aligned"); } // Bound sub_ring_capacity by total_size before multiplying so // the min_ring_stride product can't overflow on a junk value. - if (h->sub_ring_capacity > h->total_size / sizeof(Entry)) + if (geometry.sub_ring_capacity > snapshot.total_size / sizeof(Entry)) { - throw std::runtime_error( - "Header geometry: sub_ring_capacity exceeds region"); + throw std::runtime_error("Header geometry: sub_ring_capacity exceeds region"); } - std::size_t const min_ring_stride = - sizeof(SubRingHeader) + h->sub_ring_capacity * sizeof(Entry); - if (h->sub_ring_stride < min_ring_stride) + std::size_t const min_ring_stride = sizeof(SubRingHeader) + geometry.sub_ring_capacity * sizeof(Entry); + if (geometry.sub_ring_stride < min_ring_stride) { - throw std::runtime_error( - "Header geometry: sub_ring_stride too small"); + throw std::runtime_error("Header geometry: sub_ring_stride too small"); } - std::size_t const min_slot_stride = - sizeof(SlotHeader) + h->slot_data_size; - if (h->slot_stride < min_slot_stride) + // Bound the payload size before adding to it, to prevent overflow. + if (geometry.slot_data_size > snapshot.total_size - sizeof(SlotHeader)) { - throw std::runtime_error( - "Header geometry: slot_stride too small"); + throw std::runtime_error("Header geometry: slot_data_size exceeds region"); + } + std::size_t const min_slot_stride = sizeof(SlotHeader) + geometry.slot_data_size; + if (geometry.slot_stride < min_slot_stride) + { + throw std::runtime_error("Header geometry: slot_stride too small"); } // max_subs * sub_ring_stride must fit in the rings region. // Division-based bound avoids mul-overflow on a junk max_subs. - std::size_t const rings_space = h->pool_offset - h->sub_rings_offset; - if (h->max_subs > rings_space / h->sub_ring_stride) + std::size_t const rings_space = geometry.pool_offset - geometry.sub_rings_offset; + if (geometry.max_subs > rings_space / geometry.sub_ring_stride) { - throw std::runtime_error( - "Header geometry: subscriber rings overflow pool_offset"); + throw std::runtime_error("Header geometry: subscriber rings overflow pool_offset"); } // pool_size * slot_stride must fit in the pool region. - std::size_t const pool_space = h->total_size - h->pool_offset; - if (h->pool_size > pool_space / h->slot_stride) + std::size_t const pool_space = snapshot.total_size - geometry.pool_offset; + if (geometry.pool_size > pool_space / geometry.slot_stride) { - throw std::runtime_error( - "Header geometry: slot pool overflow total_size"); + throw std::runtime_error("Header geometry: slot pool overflow total_size"); } } - // Validate an already-mapped region: throws on a buffer too small - // to even hold a Header, bad magic, bad version, buffer too small - // for the embedded total_size, or geometry fields that would make - // downstream pointer math wild. - void validate_opened(void* address, std::size_t size) + // Validate a peer-written header and return the geometry pointer math may use. + // Every field is copied once and only the copy is checked, so a peer rewriting + // the header mid-validation cannot slip an unchecked value past it. + Geometry validate_opened(void* address, std::size_t size) { if (size < sizeof(Header)) { - throw std::runtime_error( - "Buffer smaller than region Header"); + throw std::runtime_error("Buffer smaller than region Header"); } - auto* h = static_cast(address); + auto const* h = static_cast
(address); if (h->magic.load(std::memory_order_acquire) != MAGIC) { throw std::runtime_error("Invalid shared memory (bad magic)"); } if (h->version != VERSION) { - throw std::runtime_error("Version mismatch"); - } - if (size < h->total_size) - { - throw std::runtime_error( - "Buffer smaller than embedded region total_size"); - } - validate_header_geometry(h); + throw VersionMismatch("Region version mismatch"); + } + + HeaderSnapshot snapshot{}; + snapshot.geometry.sub_rings_offset = h->sub_rings_offset; + snapshot.geometry.sub_ring_stride = h->sub_ring_stride; + snapshot.geometry.sub_ring_capacity = h->sub_ring_capacity; + snapshot.geometry.sub_ring_mask = h->sub_ring_mask; + snapshot.geometry.pool_offset = h->pool_offset; + snapshot.geometry.slot_stride = h->slot_stride; + snapshot.geometry.pool_size = h->pool_size; + snapshot.geometry.slot_data_size = h->slot_data_size; + snapshot.geometry.max_subs = h->max_subs; + snapshot.geometry.commit_timeout_us = h->commit_timeout_us; + snapshot.channel_type = h->channel_type; + snapshot.total_size = h->total_size; + snapshot.creator_name_len = h->creator_name_len; + + validate_snapshot(snapshot, size); + return snapshot.geometry; } // True if the ring's recorded owner is provably gone. owner_pid == 0 @@ -348,7 +388,7 @@ namespace kickmsg region.base_ = address; region.size_ = size; region.name_ = label; - validate_opened(region.base_, region.size_); + region.geometry_ = validate_opened(region.base_, region.size_); return region; } @@ -378,13 +418,11 @@ namespace kickmsg region.shm_.open(name); region.base_ = region.shm_.address(); region.size_ = region.shm_.size(); - validate_opened(region.base_, region.size_); + region.geometry_ = validate_opened(region.base_, region.size_); uint64_t stamped = region.header()->identity_hash; if (expected_identity != 0 and stamped != 0 and stamped != expected_identity) { - throw std::runtime_error( - std::string{"Identity mismatch on existing region (shm name collision): "} - + name); + throw std::runtime_error(std::string{"Identity mismatch on existing region (shm name collision): "} + name); } return region; } @@ -396,13 +434,8 @@ namespace kickmsg validate_config(type, cfg); RegionLayout layout = compute_layout(cfg, creator_name); - // Try to be the creator. On success, try_create leaves the - // SharedMemory fully mapped -- we stamp the header directly rather - // than closing and re-entering SharedMemory::create, which would - // require either O_TRUNC (rejected on Darwin) or shm_unlink + - // recreate (introduces a tiny race window where a concurrent - // caller could see the name missing or point to a different - // object than the one they initially observed). + // Initialize the mapping returned by try_create; reopening could race + // with another creator or require truncating the shared object. SharedRegion region; region.name_ = name; if (region.shm_.try_create(name, layout.total_size)) @@ -422,59 +455,49 @@ namespace kickmsg for (int i = 0; i < 200; ++i) { SharedMemory shm; - if (shm.try_open(name)) + // A creator sizes the object before stamping it; smaller means it is not ready yet. + if (shm.try_open(name) and shm.size() >= sizeof(Header)) { auto* h = static_cast(shm.address()); - if (h->magic.load(std::memory_order_acquire) == MAGIC - and h->version == VERSION) + if (h->magic.load(std::memory_order_acquire) == MAGIC and + h->version != VERSION) + { + throw VersionMismatch(std::string{"Region version mismatch on "} + name + + ": stamped by an incompatible kickmsg build; stop its users and unlink it"); + } + if (h->magic.load(std::memory_order_acquire) == MAGIC) { if (h->config_hash != expected_hash) { - throw std::runtime_error( - std::string{"Config mismatch on existing region: "} + name); + throw std::runtime_error(std::string{"Config mismatch on existing region: "} + name); } if (cfg.identity != 0 and h->identity_hash != 0 and h->identity_hash != cfg.identity) { - throw std::runtime_error( - std::string{"Identity mismatch on existing region " - "(shm name collision): "} + name); + throw std::runtime_error(std::string{"Identity mismatch on existing region (shm name collision): "} + name); } SharedRegion region; region.name_ = name; region.shm_ = std::move(shm); region.base_ = region.shm_.address(); region.size_ = region.shm_.size(); - // config_hash covers the cfg fields but NOT total_size, - // offsets, or strides -- validate the geometry like - // open()/attach_open() so a corrupt or partially-stamped - // creator can't hand us junk that later pointer math trusts. - validate_opened(region.base_, region.size_); + // The config hash does not cover offsets, strides, or total_size. + region.geometry_ = validate_opened(region.base_, region.size_); return region; } - // SHM exists but magic/version not ready yet -- creator - // is still mid-init. Close and retry. + // SHM exists but magic not ready yet -- creator is still mid-init. } // try_open returned false (ENOENT) or magic not ready -> retry. kickmsg::sleep(10ms); } - throw std::runtime_error( - std::string{"Timed out waiting for region init: "} + name); + throw std::runtime_error(std::string{"Timed out waiting for region init: "} + name); } void SharedRegion::unlink() { - // Release the OS-level name backing this region. Existing - // mappings -- this process and every peer that already opened - // the region -- keep working until their last reference drops; - // only the region's discoverability by name is affected. Any - // holder, creator or opener, may call this. Future open-by- - // name behaviour is OS-dependent and intentionally left to the - // backend. - // - // Skipped for injected regions (shm_ never opened): the caller - // owns the memory; kickmsg has no OS-level name to release. + // Unlinking removes the name; existing mappings remain valid. + // Injected regions have no OS name to remove. if (shm_.is_open() and not name_.empty()) { SharedMemory::unlink(name_); @@ -487,18 +510,16 @@ namespace kickmsg auto* h = header(); HealthReport report{}; - // Schema slot wedged at Claiming: crashed claimant that CAS'd but - // never reached Set. Mirrors the operator-surface pattern of - // retired_rings/locked_entries -- reset_schema_claim() recovers it. + // Claiming may be transient or left by a crashed schema writer. report.schema_stuck = (h->schema_state.load(std::memory_order_acquire) == schema::Claiming); - for (uint64_t i = 0; i < h->max_subs; ++i) + for (uint64_t i = 0; i < geometry_.max_subs; ++i) { - auto* ring = sub_ring_at(b, h, static_cast(i)); + auto* ring = sub_ring_at(b, geometry_, static_cast(i)); auto* entries = ring_entries(ring); uint64_t wp = ring->write_pos.load(std::memory_order_acquire); - uint64_t cap = h->sub_ring_capacity; + uint64_t cap = geometry_.sub_ring_capacity; uint64_t start = 0; if (wp > cap) @@ -507,7 +528,7 @@ namespace kickmsg } for (uint64_t pos = start; pos < wp; ++pos) { - auto& e = entries[pos & h->sub_ring_mask]; + auto& e = entries[pos & geometry_.sub_ring_mask]; uint64_t seq = e.sequence.load(std::memory_order_acquire); // Case A: explicitly locked, never committed. @@ -536,11 +557,7 @@ namespace kickmsg ++report.draining_rings; } - // A Live/Draining/Reclaiming ring whose owner process is gone - // is an orphan no other count surfaces (a dead Live ring - // otherwise reads as healthy; Reclaiming is the residue of a - // reclaimer that crashed mid-pass). reclaim_dead_rings() - // recovers all three. + // Count dead ring owners separately from ring state. if ((state == ring::Live or state == ring::Draining or state == ring::Reclaiming) and ring_owner_dead(ring)) @@ -566,12 +583,12 @@ namespace kickmsg }; std::vector candidates; - for (uint64_t i = 0; i < h->max_subs; ++i) + for (uint64_t i = 0; i < geometry_.max_subs; ++i) { - auto* ring = sub_ring_at(b, h, static_cast(i)); + auto* ring = sub_ring_at(b, geometry_, static_cast(i)); auto* entries = ring_entries(ring); uint64_t wp = ring->write_pos.load(std::memory_order_acquire); - uint64_t cap = h->sub_ring_capacity; + uint64_t cap = geometry_.sub_ring_capacity; uint64_t start = 0; if (wp > cap) @@ -580,7 +597,7 @@ namespace kickmsg } for (uint64_t pos = start; pos < wp; ++pos) { - auto& e = entries[pos & h->sub_ring_mask]; + auto& e = entries[pos & geometry_.sub_ring_mask]; uint64_t seq = e.sequence.load(std::memory_order_acquire); uint64_t expected = pos + 1; @@ -594,7 +611,7 @@ namespace kickmsg { // Case B: committed >1 wrap behind (claimant crashed // before its lock CAS). - if (entry_steal_and_clear(e, pos, seq)) + if (entry_steal_and_skip(e, pos, seq)) { h->steal_count.fetch_add(1, std::memory_order_relaxed); ++repaired; @@ -610,7 +627,7 @@ namespace kickmsg // Grace pass: an unchanged lock value across a full commit_timeout // proves its (unique) holder exceeded the commit budget. - kickmsg::sleep(microseconds{h->commit_timeout_us}); + kickmsg::sleep(microseconds{geometry_.commit_timeout_us}); for (auto const& c : candidates) { @@ -618,7 +635,7 @@ namespace kickmsg { continue; } - if (entry_steal_and_clear(*c.entry, c.pos, c.seq)) + if (entry_steal_and_skip(*c.entry, c.pos, c.seq)) { h->steal_count.fetch_add(1, std::memory_order_relaxed); ++repaired; @@ -631,16 +648,15 @@ namespace kickmsg std::size_t SharedRegion::reset_retired_rings() { auto* b = base(); - auto* h = header(); std::size_t reset = 0; - for (uint64_t i = 0; i < h->max_subs; ++i) + for (uint64_t i = 0; i < geometry_.max_subs; ++i) { - auto* ring = sub_ring_at(b, h, static_cast(i)); + auto* ring = sub_ring_at(b, geometry_, static_cast(i)); uint32_t packed = ring->state_flight.load(std::memory_order_acquire); - if (ring::get_state(packed) == ring::Free - and ring::get_in_flight(packed) > 0) + if (ring::get_state(packed) == ring::Free and + ring::get_in_flight(packed) > 0) { // Already Free but unclaimable while in_flight > 0: the store below is // the hand-off, so the retraction still goes first. @@ -657,19 +673,19 @@ namespace kickmsg std::size_t SharedRegion::reclaim_dead_rings() { auto* b = base(); - auto* h = header(); std::size_t reclaimed = 0; - for (uint64_t i = 0; i < h->max_subs; ++i) + for (uint64_t i = 0; i < geometry_.max_subs; ++i) { - auto* ring = sub_ring_at(b, h, static_cast(i)); + auto* ring = sub_ring_at(b, geometry_, static_cast(i)); uint32_t packed = ring->state_flight.load(std::memory_order_acquire); ring::State state = ring::get_state(packed); // Reclaiming residue (reclaimer crashed mid-pass) is only // recoverable here. - if (state != ring::Live and state != ring::Draining - and state != ring::Reclaiming) + if (state != ring::Live and + state != ring::Draining and + state != ring::Reclaiming) { continue; } @@ -678,11 +694,8 @@ namespace kickmsg continue; } - // Two-phase, mirroring Registry::sweep_stale: a naive CAS retry - // is value-ABA-prone (ring freed and re-claimed between checks - // would be stomped). Single-shot CAS to Reclaiming, re-verify - // death under that exclusivity; in_flight churn just defers the - // ring to the next pass. + // Use one CAS attempt, then recheck owner death. A retry could acquire + // a replacement owner's ring; in_flight changes defer this pass. uint32_t fresh = ring->state_flight.load(std::memory_order_acquire); if (ring::get_state(fresh) != state) { @@ -697,9 +710,7 @@ namespace kickmsg if (ring_owner_dead(ring)) { - // Still held Reclaiming, so still ours. The CAS below PRESERVES - // in_flight: zeroing it underflows into the state bits on a late - // fetch_sub. + // Preserve in_flight so a late publisher decrement cannot underflow state. clear_owner(ring); uint32_t old = claim; @@ -746,9 +757,7 @@ namespace kickmsg } SchemaInfo out; std::memcpy(&out, &h->schema_data, sizeof(SchemaInfo)); - // name is a C string consumers stream with operator<<; a hostile or - // corrupt region may leave it unterminated. Force a terminator so a - // reader can't run off the array. + // Ensure the copied schema name is terminated before consumers read it. out.name[sizeof(out.name) - 1] = '\0'; return out; } @@ -758,15 +767,9 @@ namespace kickmsg auto* h = header(); uint32_t expected = schema::Unset; - // Acq_rel on success: acquire so any prior claim's Set is visible on - // retry paths; release so our pre-CAS zeroing (none here) is ordered - // before subsequent writes to schema_data (still fine: Claiming is - // only visible once CAS wins, and the payload write happens-before - // the Set release-store below). - if (h->schema_state.compare_exchange_strong( - expected, schema::Claiming, - std::memory_order_acq_rel, - std::memory_order_acquire)) + // Acquire observes prior claims; the later Set release-store publishes data. + if (h->schema_state.compare_exchange_strong(expected, schema::Claiming, + std::memory_order_acq_rel, std::memory_order_acquire)) { std::memcpy(&h->schema_data, &info, sizeof(SchemaInfo)); // Release: pairs with the acquire in schema() so a reader that @@ -775,18 +778,8 @@ namespace kickmsg return true; } - // Someone else won the claim. If they're mid-write, wait briefly - // for the state to settle at Set so a follow-up schema() read is - // meaningful -- but bound the wait: a claimant that crashed between - // CAS->Claiming and store->Set leaves the slot wedged. Operators - // recover such a wedge with reset_schema_claim(), and diagnose() - // surfaces it via HealthReport::schema_stuck. - // - // MAX_YIELDS is chosen empirically: a memcpy of SchemaInfo (512 B) - // plus a release-store completes in well under a microsecond on - // any target platform, so 1024 yields gives the legitimate winner - // several orders of magnitude more than it needs while keeping the - // worst-case wait on a crashed claimant imperceptible to callers. + // Wait briefly for an active schema writer, but bound the wait if it died. + // A confirmed dead claimant can be cleared with reset_schema_claim(). constexpr int MAX_YIELDS = 1024; for (int i = 0; i < MAX_YIELDS and expected == schema::Claiming; ++i) { @@ -798,18 +791,11 @@ namespace kickmsg bool SharedRegion::reset_schema_claim() { - // Force a wedged Claiming state back to Unset so a new claim can - // proceed. Analogous to reset_retired_rings(): a deliberate - // post-crash action, NOT safe under live traffic. Only call after - // confirming the original claimant is gone; otherwise a slow-but- - // alive writer could finish its memcpy into schema_data and then - // release-store Set, while a new claimant is concurrently using - // the slot -- producing torn bytes. + // Only reset after the original claimant has stopped; a live writer could + // otherwise publish data while a new claimant overwrites it. uint32_t expected = schema::Claiming; - return header()->schema_state.compare_exchange_strong( - expected, schema::Unset, - std::memory_order_acq_rel, - std::memory_order_relaxed); + return header()->schema_state.compare_exchange_strong(expected, schema::Unset, + std::memory_order_acq_rel, std::memory_order_relaxed); } RegionStats SharedRegion::stats() const @@ -818,16 +804,16 @@ namespace kickmsg auto const* h = header(); RegionStats out{}; - out.pool_size = h->pool_size; + out.pool_size = geometry_.pool_size; out.total_steals = h->steal_count.load(std::memory_order_relaxed); - out.rings.reserve(h->max_subs); + out.rings.reserve(geometry_.max_subs); - for (uint64_t i = 0; i < h->max_subs; ++i) + for (uint64_t i = 0; i < geometry_.max_subs; ++i) { - // sub_ring_at needs a non-const base*/header*, but the operation + // sub_ring_at needs a non-const base*, but the operation // is read-only -- const_cast is safe here. auto* ring = sub_ring_at(const_cast(b), - h, static_cast(i)); + geometry_, static_cast(i)); uint32_t packed = ring->state_flight.load(std::memory_order_acquire); RingStats rs{}; @@ -841,11 +827,8 @@ namespace kickmsg { ++out.live_rings; } - // Max across ALL rings: a Free ring's write_pos is frozen at - // whatever value it had when its last subscriber left, so it's - // a valid past observation. Using max (not sum) matches the - // "publish events observed by the channel" semantic and stays - // monotonic across subscriber churn. + // Include Free rings: their frozen write_pos is a valid past observation. + // The maximum stays monotonic across subscriber changes. if (rs.write_pos > out.total_writes) { out.total_writes = rs.write_pos; @@ -856,20 +839,16 @@ namespace kickmsg out.rings.push_back(rs); } - // Approximate free-slot count: walk the Treiber stack from the head, - // bounded by pool_size so a concurrent push/pop storm can't fool us - // into an unbounded loop. Under churn we can undercount (a slot - // being popped mid-walk) or overcount (a slot's next_free pointing - // to a just-pushed node we've already counted) -- acceptable for a - // diagnostic view. + // Bound the free-stack walk by pool_size. Concurrent changes can cause + // an overcount or undercount. uint64_t top = h->free_top.load(std::memory_order_acquire); uint32_t idx = tagged_idx(top); uint64_t count = 0; - uint64_t const limit = h->pool_size; + uint64_t const limit = geometry_.pool_size; while (idx != INVALID_SLOT and count < limit) { - if (idx >= h->pool_size) break; - auto* slot = slot_at(const_cast(b), h, idx); + if (idx >= geometry_.pool_size) break; + auto* slot = slot_at(const_cast(b), geometry_, idx); idx = slot->next_free.load(std::memory_order_relaxed); ++count; } @@ -887,17 +866,24 @@ namespace kickmsg out.version = h->version; out.config_hash = h->config_hash; out.total_size = h->total_size; - out.max_subs = h->max_subs; - out.sub_ring_capacity = h->sub_ring_capacity; - out.pool_size = h->pool_size; - out.max_payload_size = h->slot_data_size; - out.commit_timeout_us = h->commit_timeout_us; + out.max_subs = geometry_.max_subs; + out.sub_ring_capacity = geometry_.sub_ring_capacity; + out.pool_size = geometry_.pool_size; + out.max_payload_size = geometry_.slot_data_size; + out.commit_timeout_us = geometry_.commit_timeout_us; out.creator_pid = h->creator_pid; out.created_at_ns = h->created_at_ns; // Creator name tail: bytes written at offset sizeof(Header). + // Re-bound the live length: a peer can rewrite it after validation. auto const* tail = static_cast(base()) + sizeof(Header); - out.creator_name.assign(tail, h->creator_name_len); + std::size_t name_len = h->creator_name_len; + std::size_t tail_len = geometry_.sub_rings_offset - sizeof(Header); + if (name_len > tail_len) + { + name_len = tail_len; + } + out.creator_name.assign(tail, name_len); return out; } @@ -906,65 +892,57 @@ namespace kickmsg auto* b = base(); auto* h = header(); - // Build a set of all slot indices referenced by committed ring entries. - std::vector referenced(h->pool_size, false); + // Each claim owns exactly one reference, so under quiescence a slot's + // refcount must equal the number of entries claiming it. + std::vector claims(geometry_.pool_size, 0); - for (uint64_t i = 0; i < h->max_subs; ++i) + for (uint64_t i = 0; i < geometry_.max_subs; ++i) { - auto* ring = sub_ring_at(b, h, static_cast(i)); - auto* entries = ring_entries(ring); - uint64_t wp = ring->write_pos.load(std::memory_order_acquire); - uint64_t cap = h->sub_ring_capacity; + auto* ring = sub_ring_at(b, geometry_, static_cast(i)); + auto* entries = ring_entries(ring); - uint64_t start = 0; - if (wp > cap) - { - start = wp - cap; - } - for (uint64_t pos = start; pos < wp; ++pos) + // Every entry, not just the write_pos window: a stranded claim can + // outlive the tenancy that wrote it. Locked and skip-marked entries + // still own the slots named by their claims. + for (uint64_t idx = 0; idx < geometry_.sub_ring_capacity; ++idx) { - auto& e = entries[pos & h->sub_ring_mask]; - uint64_t seq = e.sequence.load(std::memory_order_acquire); - - // Skip uncommitted, locked, and skip-marker entries (the - // latter carry untrustworthy metadata by design). - if (not seq_is_locked(seq) and not seq_is_skip(seq) - and seq >= pos + 1) + uint32_t biased = meta_slot_biased(entries[idx].meta.load(std::memory_order_acquire)); + if (biased != 0 and biased - 1 < geometry_.pool_size) { - uint32_t idx = e.slot_idx.load(std::memory_order_acquire); - if (idx < h->pool_size) - { - referenced[idx] = true; - } + ++claims[biased - 1]; } } } - // Free-stack membership (exact under the quiescence contract) - // recovers rc == 0 orphans a refcount-only scan never could; - // bounded against corrupt next_free cycles. - std::vector on_stack(h->pool_size, false); + // Under quiescence, free-stack membership also identifies rc == 0 orphans. + // Bound the walk to handle corrupt cycles. + std::vector on_stack(geometry_.pool_size, false); uint64_t walked = 0; uint32_t idx32 = tagged_idx(h->free_top.load(std::memory_order_acquire)); - while (idx32 != INVALID_SLOT and idx32 < h->pool_size - and walked < h->pool_size) + while (idx32 != INVALID_SLOT and idx32 < geometry_.pool_size + and walked < geometry_.pool_size) { on_stack[idx32] = true; - idx32 = slot_at(b, h, idx32)->next_free.load(std::memory_order_relaxed); + idx32 = slot_at(b, geometry_, idx32)->next_free.load(std::memory_order_relaxed); ++walked; } - // Reclaim slots that are neither ring-referenced nor on the free - // stack, regardless of refcount. + // Free unclaimed slots that are off the stack, and reset a claimed + // slot's refcount to its claim count so a leaked reference cannot pin it. std::size_t reclaimed = 0; - for (uint64_t idx = 0; idx < h->pool_size; ++idx) + for (uint64_t idx = 0; idx < geometry_.pool_size; ++idx) { - if (referenced[idx] or on_stack[idx]) + if (on_stack[idx]) { continue; } - auto* slot = slot_at(b, h, static_cast(idx)); + auto* slot = slot_at(b, geometry_, static_cast(idx)); + if (claims[idx] != 0) + { + slot->refcount.store(claims[idx], std::memory_order_release); + continue; + } slot->refcount.store(0, std::memory_order_release); treiber_push(h->free_top, slot, static_cast(idx)); ++reclaimed; diff --git a/src/Registry.cc b/src/Registry.cc index 280f4f5..8587a41 100644 --- a/src/Registry.cc +++ b/src/Registry.cc @@ -10,6 +10,36 @@ namespace kickmsg { + namespace + { + /// Mark the row as being written. The following release fence orders field stores. + void open_generation(ParticipantEntry& e) + { + uint32_t g = e.generation.load(std::memory_order_relaxed); + e.generation.store((g + 1) | 1u, std::memory_order_relaxed); + } + + /// Mark the row as settled. The caller publishes preceding writes with a fence. + void settle_generation(ParticipantEntry& e) + { + uint32_t g = e.generation.load(std::memory_order_relaxed); + e.generation.store((g + 2) & ~1u, std::memory_order_relaxed); + } + } + + bool acquire_tenancy(ParticipantEntry& e, uint32_t generation) + { + // An even-to-odd CAS holds the row until its owner settles it. + if ((generation & 1u) != 0) + { + return false; + } + uint32_t expected = generation; + return e.generation.compare_exchange_strong( + expected, generation + 1, + std::memory_order_acq_rel, std::memory_order_relaxed); + } + std::size_t Registry::region_size(uint32_t capacity) { return sizeof(RegistryHeader) @@ -18,9 +48,7 @@ namespace kickmsg std::string Registry::make_shm_name(std::string const& kmsg_namespace) { - return compose_shm_name( - sanitize_shm_component(kmsg_namespace, "namespace"), - "registry"); + return compose_shm_name(sanitize_shm_component(kmsg_namespace, "namespace"), "registry"); } RegistryHeader* Registry::header() @@ -47,7 +75,7 @@ namespace kickmsg uint32_t Registry::capacity() const { - return header()->capacity; + return capacity_; } void Registry::init_as_creator(uint32_t capacity) @@ -57,6 +85,7 @@ namespace kickmsg auto* h = header(); h->version = registry::VERSION; h->capacity = capacity; + capacity_ = capacity; // MAGIC published last -- readers spin on it with acquire. h->magic.store(registry::MAGIC, std::memory_order_release); @@ -67,30 +96,30 @@ namespace kickmsg for (int i = 0; i < 200; ++i) { SharedMemory shm; - if (shm.try_open(name)) + // A creator sizes the object before stamping it; smaller means it is not ready yet. + if (shm.try_open(name) and shm.size() >= sizeof(RegistryHeader)) { auto const* h = static_cast(shm.address()); if (h->magic.load(std::memory_order_acquire) == registry::MAGIC) { if (h->version != registry::VERSION) { - throw std::runtime_error( - "Registry version mismatch on " + name); + throw VersionMismatch("Registry version mismatch on " + name + + ": stamped by an incompatible kickmsg build; stop its users and " + "remove it with Registry::unlink()"); } - // capacity is read from shared memory and drives every - // entries[0..capacity) walk; a corrupt value would send - // snapshot()/sweep_stale() off the mapping. Bound it - // (overflow-safe) against the actual segment size. + // Bound the entry array by the mapped size. Read once: walks use only + // this validated copy, since a peer can rewrite the header later. + uint32_t cap = h->capacity; std::size_t avail = shm.size() - sizeof(RegistryHeader); - if (shm.size() < sizeof(RegistryHeader) - or h->capacity > avail / sizeof(ParticipantEntry)) + if (cap > avail / sizeof(ParticipantEntry)) { - throw std::runtime_error( - "Registry capacity exceeds segment on " + name); + throw std::runtime_error("Registry capacity exceeds segment on " + name); } Registry out; - out.name_ = name; - out.shm_ = std::move(shm); + out.name_ = name; + out.shm_ = std::move(shm); + out.capacity_ = cap; return out; } } @@ -152,9 +181,8 @@ namespace kickmsg { auto try_claim = [&]() -> uint32_t { - auto* h = header(); auto* es = entries(); - uint32_t cap = h->capacity; + uint32_t cap = capacity_; auto copy_field = [](char* dst, std::size_t dst_size, std::string const& src) @@ -172,22 +200,20 @@ namespace kickmsg for (uint32_t i = 0; i < cap; ++i) { uint32_t expected = registry::Free; - if (not es[i].state.compare_exchange_strong( - expected, registry::Claiming, - std::memory_order_acq_rel, - std::memory_order_relaxed)) + if (not es[i].state.compare_exchange_strong(expected, registry::Claiming, + std::memory_order_acq_rel, std::memory_order_relaxed)) { continue; } - // pid_starttime must be written before pid's release-store - // so a sweeper's acquire-load of pid sees a matching - // starttime. + // Mark the row odd before changing fields that a snapshot may still read. + open_generation(es[i]); + std::atomic_thread_fence(std::memory_order_release); + + // The release-store of pid also publishes pid_starttime. es[i].pid_starttime.store(my_starttime, std::memory_order_relaxed); es[i].pid.store(my_pid, std::memory_order_release); - es[i].generation.fetch_add(1, std::memory_order_relaxed); - es[i].channel_type.store(static_cast(channel_type), std::memory_order_relaxed); es[i].role.store(static_cast(role), @@ -202,6 +228,10 @@ namespace kickmsg std::memset(es[i]._padding, 0, sizeof(es[i]._padding)); es[i].state.store(registry::Active, std::memory_order_release); + + // Publish an even generation only after all fields and state are set. + std::atomic_thread_fence(std::memory_order_release); + settle_generation(es[i]); return i; } return INVALID_SLOT; @@ -212,8 +242,7 @@ namespace kickmsg { return slot; } - // Registry full -- sweep dead-pid residue and retry. Bounded to - // avoid livelock when many registrants race on a full registry. + // Bound retries when concurrent registrations compete for freed rows. for (int attempt = 0; attempt < 3; ++attempt) { if (sweep_stale() == 0) @@ -235,25 +264,41 @@ namespace kickmsg { return; } - auto* h = header(); - auto* es = entries(); - if (slot_index >= h->capacity) + auto* es = entries(); + if (slot_index >= capacity_) { return; } - // Fields are intentionally not zeroed: a concurrent snapshot may - // still be reading them, and partial zeroing before state=Free - // would create torn reads that the seqlock can't catch. The - // next claim overwrites every field. - es[slot_index].generation.fetch_add(1, std::memory_order_relaxed); - es[slot_index].state.store(registry::Free, std::memory_order_release); + auto& e = es[slot_index]; + + // Keep descriptive fields for concurrent readers. Clear identity under + // Reclaiming and publish Free last, after all metadata writes. + uint32_t expected = registry::Active; + if (not e.state.compare_exchange_strong(expected, registry::Reclaiming, + std::memory_order_acq_rel, std::memory_order_relaxed)) + { + return; + } + + // Order the state change before clearing identity. + std::atomic_thread_fence(std::memory_order_release); + open_generation(e); + e.pid.store(0, std::memory_order_relaxed); + e.pid_starttime.store(0, std::memory_order_relaxed); + settle_generation(e); + + // Sweeps skip our Reclaiming hold; publish Free only after all writes. + std::atomic_thread_fence(std::memory_order_release); + uint32_t retiring = registry::Reclaiming; + e.state.compare_exchange_strong(retiring, registry::Free, + std::memory_order_release, + std::memory_order_relaxed); } std::vector Registry::snapshot() const { - auto const* h = header(); auto const* es = entries(); - uint32_t cap = h->capacity; + uint32_t cap = capacity_; std::vector out; out.reserve(cap); @@ -265,6 +310,10 @@ namespace kickmsg continue; } uint32_t g1 = es[i].generation.load(std::memory_order_acquire); + if ((g1 & 1u) != 0) + { + continue; + } Participant p{}; p.pid = es[i].pid.load(std::memory_order_relaxed); @@ -273,20 +322,11 @@ namespace kickmsg p.channel_type = es[i].channel_type.load(std::memory_order_relaxed); p.role = es[i].role.load(std::memory_order_relaxed); p.kind = es[i].kind.load(std::memory_order_relaxed); - p.shm_name.assign( - es[i].shm_name, - ::strnlen(es[i].shm_name, sizeof(es[i].shm_name))); - p.topic_name.assign( - es[i].topic_name, - ::strnlen(es[i].topic_name, sizeof(es[i].topic_name))); - p.node_name.assign( - es[i].node_name, - ::strnlen(es[i].node_name, sizeof(es[i].node_name))); - - // Seqlock recheck. The fence is load-bearing: an acquire load - // only orders later accesses, so without it the relaxed field - // reads could be satisfied after g2/s2 (cf. read_seqretry's - // smp_rmb). + p.shm_name.assign (es[i].shm_name, ::strnlen(es[i].shm_name, sizeof(es[i].shm_name))); + p.topic_name.assign(es[i].topic_name, ::strnlen(es[i].topic_name, sizeof(es[i].topic_name))); + p.node_name.assign (es[i].node_name, ::strnlen(es[i].node_name, sizeof(es[i].node_name))); + + // Keep field reads before the generation and state recheck. std::atomic_thread_fence(std::memory_order_acquire); uint32_t g2 = es[i].generation.load(std::memory_order_acquire); uint32_t s2 = es[i].state.load(std::memory_order_acquire); @@ -319,10 +359,8 @@ namespace kickmsg } bool alive = process_exists(p.pid); - bool is_pub = (p.role == registry::Publisher - or p.role == registry::Both); - bool is_sub = (p.role == registry::Subscriber - or p.role == registry::Both); + bool is_pub = (p.role == registry::Publisher or p.role == registry::Both); + bool is_sub = (p.role == registry::Subscriber or p.role == registry::Both); if (is_pub) { @@ -362,69 +400,66 @@ namespace kickmsg uint32_t Registry::sweep_stale() { - auto* h = header(); auto* es = entries(); - uint32_t cap = h->capacity; + uint32_t cap = capacity_; uint32_t freed = 0; for (uint32_t i = 0; i < cap; ++i) { uint32_t s = es[i].state.load(std::memory_order_acquire); + // A Reclaiming owner may still be writing, even with pid == 0. + // Sweeps leave these rows alone; a crash can strand the slot. if (s != registry::Active and s != registry::Claiming) { continue; } - // Acquire syncs with register_participant's release-store of - // pid, so we see a valid pid even while state==Claiming. - uint64_t pid = es[i].pid.load(std::memory_order_acquire); - if (pid == 0) + // Validate that pid and start time belong to the same generation. + // The acquire-load of pid pairs with registration's release-store. + uint32_t g1 = es[i].generation.load(std::memory_order_acquire); + if ((g1 & 1u) != 0) { - // Registrant hasn't stored its pid yet -- reclaiming here - // would race with its pending field writes. + // An odd row may belong to a writer or another sweeper. continue; } + uint64_t pid = es[i].pid.load(std::memory_order_acquire); uint64_t stored_start = es[i].pid_starttime.load( std::memory_order_relaxed); - if (not owner_is_dead(pid, stored_start)) + std::atomic_thread_fence(std::memory_order_acquire); + if (es[i].generation.load(std::memory_order_acquire) != g1) { - continue; + continue; // spliced across a tenancy change } - // Phase 1: CAS to Reclaiming to block concurrent registrants - // and close the ABA window on a direct state->Free CAS. - uint32_t expected = s; - if (not es[i].state.compare_exchange_strong( - expected, registry::Reclaiming, - std::memory_order_acq_rel, - std::memory_order_relaxed)) + if (pid == 0) + { + // The claimant has not published its identity yet. + continue; + } + if (not owner_is_dead(pid, stored_start)) { continue; } - // Re-verify under our exclusive hold. A full dereg+register - // cycle could have slipped in between our initial pid read - // and the CAS above. - uint64_t post_pid = es[i].pid.load(std::memory_order_acquire); - uint64_t post_start = es[i].pid_starttime.load( - std::memory_order_relaxed); - if (post_pid != pid or post_start != stored_start or - not owner_is_dead(post_pid, post_start)) + // Acquire only the even generation whose owner was checked. + // A changed or already-held generation must fail without touching state. + if (not acquire_tenancy(es[i], g1)) { - // Restore via CAS, not blind store: a live tenant may - // have legitimately dereg'd (state==Free) and a blind - // store of `s` would resurrect the slot. - uint32_t reclaiming_expected = registry::Reclaiming; - es[i].state.compare_exchange_strong( - reclaiming_expected, s, - std::memory_order_release, - std::memory_order_relaxed); continue; } - // Phase 2: finalize. Fields are left untouched -- same - // reasoning as deregister(). - es[i].generation.fetch_add(1, std::memory_order_relaxed); - es[i].state.store(registry::Free, std::memory_order_release); + es[i].state.store(registry::Reclaiming, std::memory_order_release); + + // The generation is already odd. Clear identity before settling it; + // publish Free last so another registrant cannot start during these writes. + std::atomic_thread_fence(std::memory_order_release); + es[i].pid.store(0, std::memory_order_relaxed); + es[i].pid_starttime.store(0, std::memory_order_relaxed); + settle_generation(es[i]); + + std::atomic_thread_fence(std::memory_order_release); + uint32_t reclaiming = registry::Reclaiming; + es[i].state.compare_exchange_strong(reclaiming, registry::Free, + std::memory_order_release, std::memory_order_relaxed); ++freed; } return freed; diff --git a/src/Subscriber.cc b/src/Subscriber.cc index e81532c..996bc0f 100644 --- a/src/Subscriber.cc +++ b/src/Subscriber.cc @@ -17,45 +17,35 @@ namespace kickmsg Subscriber::Subscriber(SharedRegion& region) : base_{region.base()} , header_{region.header()} + , geometry_{region.geometry()} , ring_idx_{UINT32_MAX} , start_pos_{0} , read_pos_{0} , lost_{0} { - recv_buf_.resize(header_->slot_data_size); + recv_buf_.resize(geometry_.slot_data_size); - for (uint32_t i = 0; i < header_->max_subs; ++i) + for (uint32_t i = 0; i < geometry_.max_subs; ++i) { - auto* ring = sub_ring_at(base_, header_, i); - // Requires Free | in_flight=0. A ring stuck at Free | in_flight>0 - // (from a crashed publisher) stays retired until the operator - // calls reset_retired_rings(). We do NOT force-reset stale - // in_flight: the packed layout means a late fetch_sub from a - // slow publisher would underflow into the state bits. + auto* ring = sub_ring_at(base_, geometry_, i); + // Require Free with in_flight == 0. Resetting a live count could let + // a late decrement underflow into the state bits. uint32_t expected = ring::make_packed(ring::Free); - // Capture write_pos BEFORE setting Live. Once Live, publishers - // can immediately commit via fetch_add, racing with our read. - // Reading first ensures start_pos_ <= any position a publisher - // can claim after seeing Live. + // Read the drain floor before Live allows publishers to advance write_pos. uint64_t wp = ring->write_pos.load(std::memory_order_acquire); if (ring->state_flight.compare_exchange_strong(expected, ring::make_packed(ring::Live), std::memory_order_acq_rel)) { - // Record owner liveness so reclaim_dead_rings() can recover - // this ring if we crash without releasing it. starttime - // first; owner_pid (release) last, so a sweeper that reads a - // non-zero pid also sees the matching starttime. owner_pid - // stays 0 until here, so a sweep racing the claim sees 0 and - // skips (treats it as a claim in progress). + // Store starttime before releasing owner_pid, so recovery sees a matching + // identity. Until then, pid == 0 prevents recovery. uint64_t pid = current_pid(); ring->owner_starttime.store(process_starttime(pid), std::memory_order_relaxed); ring->owner_pid.store(pid, std::memory_order_release); ring_idx_ = i; - // Pre-CAS wp can be stale (no HB edge to the previous - // tenant): keep it as the drain floor, but consume from the - // freshest value or we'd replay the previous tenancy. + // Keep the earlier position as the drain floor; consume from the latest + // position to avoid replaying a previous subscriber's entries. uint64_t wp2 = ring->write_pos.load(std::memory_order_acquire); start_pos_ = wp; read_pos_ = wp; @@ -80,7 +70,7 @@ namespace kickmsg return; } - auto* ring = sub_ring_at(base_, header_, ring_idx_); + auto* ring = sub_ring_at(base_, geometry_, ring_idx_); // Transition Live -> Draining, preserving in_flight count. uint32_t old = ring->state_flight.load(std::memory_order_acquire); @@ -96,17 +86,15 @@ namespace kickmsg // Wait for all admitted publishers to finish. bool quiesced = true; - microseconds deadline{header_->commit_timeout_us}; + microseconds deadline{geometry_.commit_timeout_us}; nanoseconds start = kickmsg::monotonic_ns(); while (ring::get_in_flight( ring->state_flight.load(std::memory_order_acquire)) > 0) { if (kickmsg::elapsed_time(start) >= deadline) { - // Publisher likely crashed. Do NOT force in_flight to 0: - // a slow-but-alive publisher may still be mid-commit. - // Skip drain to avoid racing with it. Leaked slot refs - // are recoverable by GC (reclaim_orphaned_slots). + // A live publisher may still be writing. Skip draining on timeout; + // orphan recovery can release the remaining references after quiescence. quiesced = false; ++drain_timeouts_; break; @@ -153,6 +141,7 @@ namespace kickmsg Subscriber::Subscriber(Subscriber&& other) noexcept : base_{other.base_} , header_{other.header_} + , geometry_{other.geometry_} , ring_idx_{other.ring_idx_} , start_pos_{other.start_pos_} , read_pos_{other.read_pos_} @@ -173,6 +162,7 @@ namespace kickmsg base_ = other.base_; header_ = other.header_; + geometry_ = other.geometry_; ring_idx_ = other.ring_idx_; start_pos_ = other.start_pos_; read_pos_ = other.read_pos_; @@ -204,12 +194,12 @@ namespace kickmsg { return Wait::Armed; } - if (wp - read_pos_ > header_->sub_ring_capacity) + if (wp - read_pos_ > geometry_.sub_ring_capacity) { // Overrun: try_receive resynchronises and returns a sample. return Wait::Ready; } - auto& e = ring_entries(ring)[read_pos_ & header_->sub_ring_mask]; + auto& e = ring_entries(ring)[read_pos_ & geometry_.sub_ring_mask]; uint64_t seq = e.sequence.load(std::memory_order_acquire); // Same test try_receive gives up on: a lock at this position, or an // entry still holding an older generation. Everything else (commit, @@ -227,7 +217,7 @@ namespace kickmsg { return Wait::Armed; } - return head_state(sub_ring_at(base_, header_, ring_idx_)); + return head_state(sub_ring_at(base_, geometry_, ring_idx_)); } Subscriber::Wait Subscriber::arm_wait() @@ -236,14 +226,10 @@ namespace kickmsg { return Wait::Armed; } - auto* ring = sub_ring_at(base_, header_, ring_idx_); - - // Sampled BEFORE head_state decides the ring is empty. A publish landing between - // that decision and this load would otherwise already be in cur, the re-read below - // would match, and the ring would wait on a wake the publisher never sent: it read - // has_waiter before the store below and saw WaiterNone. receive() survives the same - // ordering only because futex_wait re-checks the word inside the kernel; poll on a - // descriptor has no such re-check, so this is the only guard. + auto* ring = sub_ring_at(base_, geometry_, ring_idx_); + + // Sample before checking the head to detect a publish during waiter setup. + // Descriptor polling does not recheck write_pos as futex_wait does. uint64_t cur = ring->write_pos.load(std::memory_order_relaxed); Wait state = head_state(ring); @@ -278,137 +264,20 @@ namespace kickmsg } // Relaxed: a publisher reading the mode just before this can still signal, // leaving one stale wake for the Waker's owner to drain. - auto* ring = sub_ring_at(base_, header_, ring_idx_); + auto* ring = sub_ring_at(base_, geometry_, ring_idx_); ring->has_waiter.store(ring::WaiterNone, std::memory_order_relaxed); } std::optional Subscriber::try_receive() { - // Moved-from Subscriber: ring_idx_ is the UINT32_MAX sentinel, so - // sub_ring_at would compute a wild pointer. - if (ring_idx_ == UINT32_MAX) + // Copy while the SampleView pins the slot. + auto view = try_receive_view(); + if (not view) { return std::nullopt; } - auto* ring = sub_ring_at(base_, header_, ring_idx_); - - for (int retries = 0; retries < 64; ++retries) - { - uint64_t wp = ring->write_pos.load(std::memory_order_acquire); - if (wp <= read_pos_) - { - return std::nullopt; - } - - uint64_t capacity = header_->sub_ring_capacity; - if (wp - read_pos_ > capacity) - { - uint64_t skipped = (wp - read_pos_) - capacity; - lost_ += skipped; - ring->lost_count.fetch_add(skipped, std::memory_order_relaxed); - read_pos_ = wp - capacity; - } - - uint64_t idx = read_pos_ & header_->sub_ring_mask; - auto* entries = ring_entries(ring); - auto& e = entries[idx]; - - // Acquire: ensures we see the slot_idx/payload_len written - // by the publisher before the sequence commit. - uint64_t seq1 = e.sequence.load(std::memory_order_acquire); - if (seq1 != read_pos_ + 1) - { - if (seq_is_skip(seq1) and seq_pos(seq1) == read_pos_ + 1) - { - // Skip marker: metadata untrustworthy by design. - ++lost_; - ring->lost_count.fetch_add(1, std::memory_order_relaxed); - ++read_pos_; - continue; - } - if (seq_is_locked(seq1) or seq_pos(seq1) < read_pos_ + 1) - { - // Publisher is mid-commit (position-tagged lock) or has - // not committed yet. Come back later. - return std::nullopt; - } - // Entry was overwritten (seq > expected): advance and retry. - ++lost_; - ring->lost_count.fetch_add(1, std::memory_order_relaxed); - ++read_pos_; - continue; - } - - uint32_t slot_idx = e.slot_idx.load(std::memory_order_relaxed); - uint32_t payload_len = e.payload_len.load(std::memory_order_relaxed); - - if (slot_idx >= header_->pool_size or payload_len > header_->slot_data_size) - { - ++lost_; - ring->lost_count.fetch_add(1, std::memory_order_relaxed); - ++read_pos_; - continue; - } - - // Pin the slot via refcount increment to prevent it from being - // freed while we memcpy. Without the pin, a publisher could evict - // the ring entry and push the slot back to the free stack, letting - // another publisher overwrite the data mid-copy. - auto* slot = slot_at(base_, header_, slot_idx); - uint32_t rc = slot->refcount.load(std::memory_order_acquire); - bool pinned = false; - // rc == UINT32_MAX is unreachable for a healthy slot (refcount is - // bounded by max_subs + live views); treat it as corrupt residue - // so rc + 1 can't wrap to 0 and make a pinned slot look freeable. - while (rc > 0 and rc != UINT32_MAX) - { - if (slot->refcount.compare_exchange_weak(rc, rc + 1, - std::memory_order_acq_rel, std::memory_order_acquire)) - { - pinned = true; - break; - } - } - - if (not pinned) - { - // refcount == 0 (or corrupt): slot not pinnable, count as lost. - ++lost_; - ring->lost_count.fetch_add(1, std::memory_order_relaxed); - ++read_pos_; - continue; - } - - // Seqlock validation: re-read the sequence after pinning. If it - // changed, the entry was overwritten between our first read and - // the pin, so the slot_idx we pinned may be stale. - uint64_t seq2 = e.sequence.load(std::memory_order_acquire); - if (seq2 != seq1) - { - uint32_t prev = slot->refcount.fetch_sub(1, std::memory_order_acq_rel); - if (prev == 1) - { - treiber_push(header_->free_top, slot, slot_idx); - } - ++lost_; - ring->lost_count.fetch_add(1, std::memory_order_relaxed); - ++read_pos_; - continue; - } - - std::memcpy(recv_buf_.data(), slot_data(slot), payload_len); - - // Unpin: we have our copy, release the slot reference. - uint32_t prev = slot->refcount.fetch_sub(1, std::memory_order_acq_rel); - if (prev == 1) - { - treiber_push(header_->free_top, slot, slot_idx); - } - - ++read_pos_; - return SampleRef{recv_buf_.data(), payload_len, read_pos_ - 1}; - } - return std::nullopt; + std::memcpy(recv_buf_.data(), view->data(), view->len()); + return SampleRef{recv_buf_.data(), view->len(), view->ring_pos()}; } std::optional Subscriber::receive(nanoseconds timeout) @@ -419,7 +288,7 @@ namespace kickmsg { return std::nullopt; } - auto* ring = sub_ring_at(base_, header_, ring_idx_); + auto* ring = sub_ring_at(base_, geometry_, ring_idx_); nanoseconds start = kickmsg::monotonic_ns(); int idle_spins = 0; @@ -481,7 +350,7 @@ namespace kickmsg { return std::nullopt; } - auto* ring = sub_ring_at(base_, header_, ring_idx_); + auto* ring = sub_ring_at(base_, geometry_, ring_idx_); for (int retries = 0; retries < 64; ++retries) { @@ -491,7 +360,7 @@ namespace kickmsg return std::nullopt; } - uint64_t capacity = header_->sub_ring_capacity; + uint64_t capacity = geometry_.sub_ring_capacity; if (wp - read_pos_ > capacity) { uint64_t skipped = (wp - read_pos_) - capacity; @@ -500,7 +369,7 @@ namespace kickmsg read_pos_ = wp - capacity; } - uint64_t idx = read_pos_ & header_->sub_ring_mask; + uint64_t idx = read_pos_ & geometry_.sub_ring_mask; auto* entries = ring_entries(ring); auto& e = entries[idx]; @@ -524,19 +393,22 @@ namespace kickmsg continue; } - uint32_t slot_idx = e.slot_idx.load(std::memory_order_relaxed); - uint32_t payload_len = e.payload_len.load(std::memory_order_relaxed); - - if (slot_idx >= header_->pool_size or payload_len > header_->slot_data_size) + // Acquire orders a newer publisher's sequence lock before seq2 below; the tag + // rejects a claim that belongs to another position. + uint64_t meta = e.meta.load(std::memory_order_acquire); + uint32_t biased = meta_slot_biased(meta); + if (meta_tag(meta) != ((read_pos_ + 1) & META_TAG_MASK) + or biased == 0 or biased - 1 >= geometry_.pool_size) { ++lost_; ring->lost_count.fetch_add(1, std::memory_order_relaxed); ++read_pos_; continue; } + uint32_t slot_idx = biased - 1; // Pin the slot so it survives until ~SampleView(). - auto* slot = slot_at(base_, header_, slot_idx); + auto* slot = slot_at(base_, geometry_, slot_idx); uint32_t rc = slot->refcount.load(std::memory_order_acquire); bool pinned = false; // rc == UINT32_MAX is corrupt residue; skip so rc + 1 can't wrap. @@ -574,8 +446,23 @@ namespace kickmsg continue; } + // Read under the pin, after the seqlock: before that the slot may be recycled. + uint32_t payload_len = slot->payload_len.load(std::memory_order_relaxed); + if (payload_len > geometry_.slot_data_size) + { + uint32_t bad = slot->refcount.fetch_sub(1, std::memory_order_acq_rel); + if (bad == 1) + { + treiber_push(header_->free_top, slot, slot_idx); + } + ++lost_; + ring->lost_count.fetch_add(1, std::memory_order_relaxed); + ++read_pos_; + continue; + } + ++read_pos_; - return SampleView{base_, header_, slot_idx, payload_len, read_pos_ - 1}; + return SampleView{header_, slot, slot_idx, payload_len, read_pos_ - 1}; } return std::nullopt; } @@ -588,7 +475,7 @@ namespace kickmsg { return std::nullopt; } - auto* ring = sub_ring_at(base_, header_, ring_idx_); + auto* ring = sub_ring_at(base_, geometry_, ring_idx_); nanoseconds start = kickmsg::monotonic_ns(); int idle_spins = 0; @@ -645,7 +532,7 @@ namespace kickmsg void Subscriber::drain_unconsumed(SubRingHeader* ring) { auto* entries = ring_entries(ring); - uint64_t capacity = header_->sub_ring_capacity; + uint64_t capacity = geometry_.sub_ring_capacity; // write_pos is final: the in_flight spin in the destructor guarantees // no publisher is mid-commit on this ring. @@ -669,34 +556,33 @@ namespace kickmsg oldest = start_pos_; } - // Release this ring's reference for ALL committed entries in the live window: - // - [oldest, read_pos_): consumed by try_receive (pin/unpin is net-zero, - // so the ring's original rc=1 reference still needs releasing). - // For try_receive_view, rc=2 (ring ref + SampleView pin); we release - // the ring ref here, ~SampleView releases the pin later. - // - [read_pos_, wp): unconsumed entries, also need their ring ref released. - // Evicted entries have seq != pos+1, so the check safely skips them. + // Release each remaining claim, including consumed and skip-marked entries. + // SampleView pins are separate references and survive this drain. for (uint64_t pos = oldest; pos < wp; ++pos) { - auto& e = entries[pos & header_->sub_ring_mask]; - uint64_t seq = e.sequence.load(std::memory_order_acquire); + auto& e = entries[pos & geometry_.sub_ring_mask]; + uint64_t meta = e.meta.load(std::memory_order_acquire); - if (seq != pos + 1) + uint32_t biased = meta_slot_biased(meta); + if (biased == 0 or biased - 1 >= geometry_.pool_size) { continue; } + uint32_t slot_idx = biased - 1; - uint32_t slot_idx = e.slot_idx.load(std::memory_order_relaxed); - if (slot_idx < header_->pool_size) + // Clear the claim before releasing its reference to prevent a second release. + if (not e.meta.compare_exchange_strong(meta, meta & ~META_SLOT_MASK, + std::memory_order_acq_rel, std::memory_order_relaxed)) { - auto* slot = slot_at(base_, header_, slot_idx); - uint32_t prev = slot->refcount.fetch_sub(1, - std::memory_order_acq_rel); - if (prev == 1) - { - treiber_push(header_->free_top, slot, slot_idx); - } - e.slot_idx.store(INVALID_SLOT, std::memory_order_seq_cst); + continue; + } + + auto* slot = slot_at(base_, geometry_, slot_idx); + uint32_t prev = slot->refcount.fetch_sub(1, + std::memory_order_acq_rel); + if (prev == 1) + { + treiber_push(header_->free_top, slot, slot_idx); } } diff --git a/src/os/posix/WakeBackends.cc b/src/os/posix/WakeBackends.cc index 0dcdd84..70ab5fc 100644 --- a/src/os/posix/WakeBackends.cc +++ b/src/os/posix/WakeBackends.cc @@ -27,9 +27,8 @@ namespace kickmsg /// Descriptors gathered on the stack before WaitSet::wait allocates. constexpr std::size_t STACK_FDS = 64; - /// Every option matters. Non-blocking keeps signal() off the publish path. TTL 0 - /// and the loopback interface keep the wake on this host, and LOOP is what still - /// delivers it. A socket missing any of them is thrown away. + /// Non-blocking sockets keep signal() from waiting. TTL 0, loopback + /// interface, and multicast loopback restrict delivery to this host. bool configure_sender(int fd) { int flags = ::fcntl(fd, F_GETFL, 0); @@ -93,8 +92,7 @@ namespace kickmsg } (void) ::fcntl(fd, F_SETFD, FD_CLOEXEC); - // Every subscriber of this channel binds the same port, which SO_REUSEADDR is - // what permits; without it this socket blocks every later joiner. + // SO_REUSEADDR allows all channel subscribers to bind the same port. int on = 1; if (::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) != 0) { @@ -133,8 +131,7 @@ namespace kickmsg void UdpMulticastBackend::drain(int fd) { - // Bounded: the group is joinable by any local process, which could otherwise - // feed this loop indefinitely. What is left reads as a spurious wake. + // Bound draining so incoming traffic cannot keep the caller here forever. uint8_t buffer[64]; for (int i = 0; i < DRAIN_MAX; ++i) { @@ -186,10 +183,10 @@ namespace kickmsg } auto const count = fds_.size(); - // poll() takes any count; only the stack buffer is bounded. - pollfd stack[STACK_FDS] = {}; - std::vector heap; - pollfd* entries = stack; + // Reuse per-thread storage when the descriptor count exceeds the stack buffer. + pollfd stack[STACK_FDS] = {}; + pollfd* entries = stack; + static thread_local std::vector heap; if (count > STACK_FDS) { heap.resize(count); @@ -214,8 +211,7 @@ namespace kickmsg { return false; } - // A count alone is not readability: POLLERR, POLLHUP and POLLNVAL also raise it, - // and a closed descriptor would then report ready forever and spin the caller. + // Only readability counts: poll also returns errors and closed descriptors. for (std::size_t i = 0; i < count; ++i) { if ((entries[i].revents & POLLIN) != 0) diff --git a/src/os/windows/Futex.cc b/src/os/windows/Futex.cc index 238a690..87c73bb 100644 --- a/src/os/windows/Futex.cc +++ b/src/os/windows/Futex.cc @@ -7,11 +7,8 @@ namespace kickmsg { - // The watched word is the LOW 32 bits of the 64-bit counter; on a - // big-endian target &word addresses the HIGH half and the value check - // silently breaks (lost wakeups until timeout). MSVC has no - // __BYTE_ORDER__, but every Windows target (x86, x64, ARM64) is - // little-endian. + // Wait on the low 32 bits of the counter. Supported Windows targets + // (x86, x64, ARM64) are little-endian. #if defined(__BYTE_ORDER__) static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "WaitOnAddress word aliasing assumes the low half of write_pos at offset 0"); @@ -22,9 +19,7 @@ namespace kickmsg auto* addr = reinterpret_cast(&word); auto val = static_cast(expected); - // to_poll_ms rounds up, so a sub-millisecond budget still waits rather than - // returning at once and spinning, and clamps to INT_MAX -- below INFINITE, which - // this must never pass by accident. + // Round up fractional milliseconds and clamp below INFINITE. DWORD timeout_ms = static_cast(to_poll_ms(timeout)); if (WaitOnAddress(addr, &val, sizeof(val), timeout_ms)) @@ -38,6 +33,9 @@ namespace kickmsg return -EINVAL; } + // WakeByAddressAll wakes only this process. Cross-process receive() can + // wait until timeout, during which unread messages may overflow the ring. + // See the Windows limitation in ARCHITECTURE.md. void futex_wake_all(std::atomic& word) { WakeByAddressAll(reinterpret_cast(&word)); diff --git a/src/os/windows/WakeBackends.cc b/src/os/windows/WakeBackends.cc index 8d48918..971a247 100644 --- a/src/os/windows/WakeBackends.cc +++ b/src/os/windows/WakeBackends.cc @@ -21,8 +21,7 @@ namespace kickmsg /// Descriptors gathered on the stack before WaitSet::wait allocates. constexpr std::size_t STACK_FDS = 64; - /// Winsock needs a process-wide init before any socket call. Function-local static, - /// so it happens once and only if a wake backend is actually used. + /// Initialize Winsock once per process, when first used. bool winsock_ready() { static bool const ready = [] @@ -33,9 +32,7 @@ namespace kickmsg return ready; } - /// SOCKET is a UINT_PTR. Windows keeps handle values small for interop, but that - /// is convention, not contract: a value past INT_MAX would wrap negative and read - /// back as a failed bind, so it fails closed instead. + /// SOCKET is pointer-sized; reject values that cannot fit this API's int. int to_fd(SOCKET socket) { if (socket == INVALID_SOCKET) @@ -55,9 +52,8 @@ namespace kickmsg return static_cast(fd); } - /// Every option matters. Non-blocking keeps signal() off the publish path. TTL 0 - /// and the loopback interface keep the wake on this host, and LOOP is what still - /// delivers it. A socket missing any of them is thrown away. + /// Non-blocking sockets keep signal() from waiting. TTL 0, loopback + /// interface, and multicast loopback restrict delivery to this host. bool configure_sender(SOCKET socket) { u_long non_blocking = 1; @@ -132,8 +128,7 @@ namespace kickmsg return -1; } - // Every subscriber of this channel binds the same port, which SO_REUSEADDR is - // what permits. Without it this socket blocks every later joiner. + // SO_REUSEADDR allows all channel subscribers to bind the same port. BOOL on = TRUE; if (::setsockopt(socket, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&on), sizeof(on)) == SOCKET_ERROR) @@ -173,8 +168,7 @@ namespace kickmsg void UdpMulticastBackend::drain(int fd) { - // Bounded, as on POSIX: the group is joinable by any local process, which could - // otherwise feed this loop indefinitely. What is left reads as a spurious wake. + // Bound draining so incoming traffic cannot keep the caller here forever. char buffer[64]; for (int i = 0; i < DRAIN_MAX; ++i) { @@ -216,12 +210,11 @@ namespace kickmsg } auto const count = fds_.size(); - // WSAPoll, not select: fd_set is a fixed array of FD_SETSIZE sockets, which - // would cap this API at a compile-time constant. WSAPoll's documented defect is - // POLLOUT on a failed connect; this only ever asks to read. - WSAPOLLFD stack[STACK_FDS] = {}; - std::vector heap; - WSAPOLLFD* entries = stack; + // WSAPoll avoids select's FD_SETSIZE limit; only readability is requested. + // Reuse per-thread storage beyond the stack buffer. + WSAPOLLFD stack[STACK_FDS] = {}; + WSAPOLLFD* entries = stack; + static thread_local std::vector heap; if (count > STACK_FDS) { heap.resize(count); @@ -243,8 +236,7 @@ namespace kickmsg { return false; } - // A count alone is not readability: POLLERR, POLLHUP and POLLNVAL also raise it, - // and a closed socket would then report ready forever and spin the caller. + // Only readability counts: poll also returns errors and closed descriptors. for (std::size_t i = 0; i < count; ++i) { if ((entries[i].revents & POLLRDNORM) != 0) diff --git a/src/types.cc b/src/types.cc index b9609b8..24bf931 100644 --- a/src/types.cc +++ b/src/types.cc @@ -5,18 +5,17 @@ namespace kickmsg { void clear_owner(SubRingHeader* ring) { - // Relaxed: the release of Free that follows is what publishes these. - // has_waiter included, or a tenant killed mid-wait leaves its mode set and every - // later publish pays a wake for a waiter that no longer exists. + // Relaxed: the following Free release-store publishes these fields. + // Clear has_waiter so a replacement does not inherit a stale wake mode. ring->has_waiter.store(ring::WaiterNone, std::memory_order_relaxed); ring->owner_pid.store(0, std::memory_order_relaxed); ring->owner_starttime.store(0, std::memory_order_relaxed); } - SubRingHeader* sub_ring_at(void* base, Header const* h, uint32_t idx) + SubRingHeader* sub_ring_at(void* base, Geometry const& geometry, uint32_t idx) { - auto* p = static_cast(base) + h->sub_rings_offset; - return reinterpret_cast(p + idx * h->sub_ring_stride); + auto* p = static_cast(base) + geometry.sub_rings_offset; + return reinterpret_cast(p + idx * geometry.sub_ring_stride); } Entry* ring_entries(SubRingHeader* ring) @@ -25,10 +24,10 @@ namespace kickmsg reinterpret_cast(ring) + sizeof(SubRingHeader)); } - SlotHeader* slot_at(void* base, Header const* h, uint32_t idx) + SlotHeader* slot_at(void* base, Geometry const& geometry, uint32_t idx) { - auto* p = static_cast(base) + h->pool_offset; - return reinterpret_cast(p + idx * h->slot_stride); + auto* p = static_cast(base) + geometry.pool_offset; + return reinterpret_cast(p + idx * geometry.slot_stride); } SlotHeader* slot_at(void* pool_base, std::size_t slot_stride, uint32_t idx) @@ -47,9 +46,7 @@ namespace kickmsg return reinterpret_cast(h) + sizeof(Header); } - // FNV-1a over the config fields, detecting parameter mismatches at - // open time. Field order is part of the on-disk hash -- do NOT - // reorder without bumping VERSION. + // Config hash field order is part of the ABI; changes require a VERSION bump. uint64_t compute_config_hash(channel::Type type, channel::Config const& cfg) { uint64_t h = hash::fnv1a_64(type); @@ -69,22 +66,18 @@ namespace kickmsg if (a.identity != b.identity) d |= Identity; if (a.layout != b.layout) d |= Layout; if (a.version != b.version) d |= Version; - // Compare name up to the full 128-byte slot: strncmp stops at - // the first NUL so trailing zero padding doesn't register as a - // diff, but a non-terminated stray byte inside the slot still - // does (better to flag than to silently drop). + // Compare through NUL or the fixed field size. if (std::strncmp(a.name, b.name, sizeof(a.name)) != 0) d |= Name; if (a.identity_algo != b.identity_algo) d |= IdentityAlgo; if (a.layout_algo != b.layout_algo) d |= LayoutAlgo; - // Intentionally NOT compared: flags, reserved[] -- see Diff - // doc in types.h for rationale (forward compatibility). + // Ignore reserved fields for forward compatibility. return d; } } - bool entry_steal_and_clear(Entry& e, uint64_t pos, uint64_t observed) + bool entry_steal_and_skip(Entry& e, uint64_t pos, uint64_t observed) { - // CAS-own before touching metadata; a live writer that moves first wins. + // CAS-own the sequence; a live writer that moves first wins. uint64_t expected = observed; if (not e.sequence.compare_exchange_strong(expected, seq_repair(pos), std::memory_order_acquire, std::memory_order_relaxed)) @@ -92,14 +85,7 @@ namespace kickmsg return false; } - // No slot release here: the holder may have batch-released this - // ring's ref already (excess path) -- releasing again could - // double-free. The leak is bounded and GC-recoverable. - e.slot_idx.store(INVALID_SLOT, std::memory_order_relaxed); - e.payload_len.store(0, std::memory_order_relaxed); - // Skip tag, not a plain sequence: the holder's late metadata stores - // must never be trusted (see types.h). The INVALID stores are - // diagnostics only. + // Keep the slot claim; the next publisher or drainer releases its reference. e.sequence.store(seq_skip(pos), std::memory_order_release); return true; } @@ -117,9 +103,9 @@ namespace kickmsg while (not top.compare_exchange_weak(old_top, new_top, std::memory_order_release, std::memory_order_relaxed)); } - uint32_t treiber_pop(std::atomic& top, void* base, Header const* h) + uint32_t treiber_pop(std::atomic& top, void* base, Geometry const& geometry) { - return treiber_pop(top, static_cast(base) + h->pool_offset, h->slot_stride); + return treiber_pop(top, static_cast(base) + geometry.pool_offset, geometry.slot_stride, geometry.pool_size); } void treiber_push(std::atomic& top, void* pool_base, std::size_t slot_stride, uint32_t slot_idx) @@ -127,22 +113,32 @@ namespace kickmsg treiber_push(top, slot_at(pool_base, slot_stride, slot_idx), slot_idx); } - uint32_t treiber_pop(std::atomic& top, void* pool_base, std::size_t slot_stride) + uint32_t treiber_pop(std::atomic& top, void* pool_base, std::size_t slot_stride, + uint64_t pool_size) { // Acquire: pairs with the release in push to see the pushed slot's next_free. uint64_t old_top = top.load(std::memory_order_acquire); - while (tagged_idx(old_top) != INVALID_SLOT) + while (true) { - auto* slot = slot_at(pool_base, slot_stride, tagged_idx(old_top)); + uint32_t idx = tagged_idx(old_top); + if (idx == INVALID_SLOT) + { + return INVALID_SLOT; + } + // Bounds-check each shared free-list index before dereferencing it. + if (idx >= pool_size) + { + return INVALID_SLOT; + } + auto* slot = slot_at(pool_base, slot_stride, idx); uint32_t next = slot->next_free.load(std::memory_order_relaxed); uint64_t new_top = tagged_pack(tagged_gen(old_top) + 1, next); // Acq_rel: release publishes the new top, acquire synchronizes with the last push. // Failure is acquire: on retry we need to see the next_free written by whoever changed top. if (top.compare_exchange_weak(old_top, new_top, std::memory_order_acq_rel, std::memory_order_acquire)) { - return tagged_idx(old_top); + return idx; } } - return INVALID_SLOT; } } diff --git a/tests/blackboard_crash_test.cc b/tests/blackboard_crash_test.cc index 7b5f6bf..d2e77b0 100644 --- a/tests/blackboard_crash_test.cc +++ b/tests/blackboard_crash_test.cc @@ -25,6 +25,7 @@ /// /// Exits 0 on success, non-zero on any assertion failure. +#include #include #include #include @@ -362,21 +363,34 @@ static bool test_declare_race_across_processes() auto bb = Blackboard::open_or_create(NS, NAME, cfg()); + // A winner holds its claim until the parent closes this pipe, which it does + // only once every loser has exited. A timed hold let a child that started + // late (TSAN's fork is slow) win legitimately after the release, which + // reads exactly like two simultaneous owners. + int hold[2]; + if (::pipe(hold) != 0) + { + std::perror("pipe"); + std::exit(1); + } + pid_t pids[CHILDREN]; for (int i = 0; i < CHILDREN; ++i) { pids[i] = checked_fork(); if (pids[i] == 0) { + ::close(hold[1]); auto child_bb = Blackboard::open_or_create(NS, NAME, cfg()); int code = 1; try { auto w = child_bb.declare(KEY, "racer"); code = 0; - // Hold the claim until the parent reaps everyone, so a loser - // can never win by outliving the winner's release. - kickmsg::sleep(300ms); + char byte; + while (::read(hold[0], &byte, 1) < 0 and errno == EINTR) + { + } w.release(); } catch (std::exception const&) @@ -386,11 +400,40 @@ static bool test_declare_race_across_processes() ::_exit(code); } } + ::close(hold[0]); - int winners = 0; + // Reap losers until only the holders remain. More than one holder never + // exits on its own, so the deadline bounds that failure instead of hanging. + int winners = 0; + int reaped = 0; + bool done[CHILDREN] = {}; + auto const deadline = kickmsg::monotonic_ns() + 60s; + while (reaped < CHILDREN - 1 and kickmsg::monotonic_ns() < deadline) + { + for (int i = 0; i < CHILDREN; ++i) + { + int status = 0; + if (done[i] or ::waitpid(pids[i], &status, WNOHANG) != pids[i]) + { + continue; + } + done[i] = true; + ++reaped; + if (WIFEXITED(status) and WEXITSTATUS(status) == 0) + { + ++winners; // exited while holding: only possible after release + } + } + kickmsg::sleep(5ms); + } + ::close(hold[1]); for (int i = 0; i < CHILDREN; ++i) { int status = 0; + if (done[i]) + { + continue; + } ::waitpid(pids[i], &status, 0); if (WIFEXITED(status) and WEXITSTATUS(status) == 0) { diff --git a/tests/crash_test.cc b/tests/crash_test.cc index c4849ca..123110d 100644 --- a/tests/crash_test.cc +++ b/tests/crash_test.cc @@ -97,8 +97,8 @@ static void child_publisher_main(int /*round*/) for (uint32_t i = 0; ; ++i) { - auto a = pub.allocate(); - if (a.data == nullptr) + auto slot = pub.allocate(); + if (not slot.valid()) { kickmsg::yield(); continue; @@ -108,9 +108,9 @@ static void child_publisher_main(int /*round*/) msg.magic = CrashPayload::MAGIC; msg.seq = i; msg.checksum = compute_checksum(msg); - std::memcpy(a.data, &msg, sizeof(msg)); + slot.write(&msg, sizeof(msg)); - pub.publish(sizeof(msg)); + slot.publish(sizeof(msg)); } } @@ -411,8 +411,8 @@ static bool test_multi_publisher_crash() kickmsg::Publisher p(r); for (uint32_t seq = 0; ; ++seq) { - auto a = p.allocate(); - if (a.data == nullptr) + auto slot = p.allocate(); + if (not slot.valid()) { kickmsg::yield(); continue; @@ -421,8 +421,8 @@ static bool test_multi_publisher_crash() msg.magic = CrashPayload::MAGIC; msg.seq = seq; msg.checksum = compute_checksum(msg); - std::memcpy(a.data, &msg, sizeof(msg)); - p.publish(sizeof(msg)); + slot.write(&msg, sizeof(msg)); + slot.publish(sizeof(msg)); } } } diff --git a/tests/mp_stress_test.cc b/tests/mp_stress_test.cc index b3d5a88..30f5936 100644 --- a/tests/mp_stress_test.cc +++ b/tests/mp_stress_test.cc @@ -284,11 +284,11 @@ static int child_subscriber_main(int sub_id, int ready_wfd, int report_wfd) static bool verify_rings_free(kickmsg::SharedRegion& region) { - auto* hdr = region.header(); + auto* header = region.header(); bool ok = true; - for (uint32_t i = 0; i < hdr->max_subs; ++i) + for (uint32_t i = 0; i < header->max_subs; ++i) { - auto* ring = kickmsg::sub_ring_at(region.base(), hdr, i); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), i); uint32_t packed = ring->state_flight.load(std::memory_order_acquire); if (kickmsg::ring::get_state(packed) != kickmsg::ring::Free) { @@ -312,14 +312,14 @@ static bool verify_rings_free(kickmsg::SharedRegion& region) static bool verify_slots(kickmsg::SharedRegion& region) { auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); - std::vector in_free(hdr->pool_size, false); - uint32_t top = kickmsg::tagged_idx(hdr->free_top.load(std::memory_order_acquire)); + std::vector in_free(header->pool_size, false); + uint32_t top = kickmsg::tagged_idx(header->free_top.load(std::memory_order_acquire)); while (top != kickmsg::INVALID_SLOT) { - if (top >= hdr->pool_size) + if (top >= header->pool_size) { std::fprintf(stderr, " [FAIL] free stack contains out-of-range index %u\n", top); return false; @@ -332,18 +332,18 @@ static bool verify_slots(kickmsg::SharedRegion& region) } in_free[top] = true; - auto* slot = kickmsg::slot_at(base, hdr, top); + auto* slot = kickmsg::slot_at(base, region.geometry(), top); top = slot->next_free; } bool ok = true; - for (uint32_t i = 0; i < hdr->pool_size; ++i) + for (uint32_t i = 0; i < header->pool_size; ++i) { if (in_free[i]) { continue; } - auto* slot = kickmsg::slot_at(base, hdr, i); + auto* slot = kickmsg::slot_at(base, region.geometry(), i); uint32_t rc = slot->refcount; if (rc != 0) { @@ -565,7 +565,7 @@ int main() uint64_t dropped = 0; for (uint32_t i = 0; i < cfg.max_subscribers; ++i) { - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), i); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), i); dropped += ring->dropped_count.load(std::memory_order_acquire); } std::size_t repaired = region.repair_locked_entries(); diff --git a/tests/python/test_zerocopy.py b/tests/python/test_zerocopy.py index 32df71d..e76cc8f 100644 --- a/tests/python/test_zerocopy.py +++ b/tests/python/test_zerocopy.py @@ -5,7 +5,9 @@ - `memoryview(view)` → read-only view into the subscriber-pinned SHM slot The memoryview pins its source object alive (Py_buffer::obj + Py_INCREF), so retaining a memoryview past the Python reference to slot/view keeps the -underlying shared memory valid until the last memoryview is released. +mapping alive until the last memoryview is released. It does not keep a +reservation valid: a writable view outlives publish() or the next allocate() +only as a dangling pointer into a recycled slot. """ from __future__ import annotations diff --git a/tests/stall_repair_test.cc b/tests/stall_repair_test.cc index 14f39f4..fec069d 100644 --- a/tests/stall_repair_test.cc +++ b/tests/stall_repair_test.cc @@ -1,17 +1,6 @@ /// @file stall_repair_test.cc -/// @brief False-positive-death fuzz test for the theft-safe commit protocol. -/// -/// A child publisher is SIGSTOPped at random instants, so it sometimes -/// freezes while holding a position-tagged entry lock. With a tight -/// commit_timeout the stall makes the lock "provably stale": an external -/// repairer (the parent) runs repair_locked_entries() during the stall and -/// steals the entry. The publisher is then SIGCONTed and resumes. -/// -/// The theft guard + CAS commit in Publisher::publish() must turn every -/// such steal into a clean publisher drop: -/// - never a torn payload (magic/checksum validated on every sample), -/// - never a per-publisher sequence rewind (seq strictly increasing), -/// - never refcount corruption (structural pool checks at the end). +/// Pause child publishers with SIGSTOP, repair their locks, then resume them. +/// Check payloads, sequence order, and pool references after lock theft. #include #include @@ -50,10 +39,7 @@ static uint32_t compute_checksum(StallPayload const& p) return p.magic ^ p.pub_id ^ p.seq ^ 0xDEADBEEF; } -// --- Seeded stall-timing fuzzer --------------------------------------------- -// Each SIGSTOP fires at a random instant so a long soak explores new stall -// windows instead of re-hitting a fixed schedule. The seed is logged at -// startup; set KICKMSG_STALL_SEED to replay a specific run. +// Randomize stall timing. Set KICKMSG_STALL_SEED to replay a logged seed. namespace { uint64_t g_rng_state = 0; @@ -116,9 +102,7 @@ static pid_t checked_fork(char const* site) /// increasing seq in a tight loop until SIGTERM flips the stop flag. static void child_publisher_main() { - // Replace the inherited shm-cleanup SIGTERM handler: the parent still - // uses the segment, so the child must convert SIGTERM into a clean loop - // exit instead of unlinking the region out from under it. + // The child must not unlink shared memory still used by the parent. struct sigaction sa; std::memset(&sa, 0, sizeof(sa)); sa.sa_handler = child_stop_handler; @@ -132,8 +116,8 @@ static void child_publisher_main() uint32_t seq = 0; while (g_child_stop == 0) { - auto a = pub.allocate(); - if (a.data == nullptr) + auto slot = pub.allocate(); + if (not slot.valid()) { kickmsg::yield(); continue; @@ -144,9 +128,9 @@ static void child_publisher_main() msg.pub_id = 1; msg.seq = seq; msg.checksum = compute_checksum(msg); - std::memcpy(a.data, &msg, sizeof(msg)); + slot.write(&msg, sizeof(msg)); - pub.publish(sizeof(msg)); + slot.publish(sizeof(msg)); // A dropped publish (theft detected) leaves a gap -- gaps are // legitimate; the subscriber only rejects a seq going backward. ++seq; @@ -237,11 +221,11 @@ static void subscriber_main(kickmsg::SharedRegion& region, SubStats& stats) static bool verify_rings_free(kickmsg::SharedRegion& region) { - auto* hdr = region.header(); + auto* header = region.header(); bool ok = true; - for (uint32_t i = 0; i < hdr->max_subs; ++i) + for (uint32_t i = 0; i < header->max_subs; ++i) { - auto* ring = kickmsg::sub_ring_at(region.base(), hdr, i); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), i); uint32_t packed = ring->state_flight.load(std::memory_order_acquire); if (kickmsg::ring::get_state(packed) != kickmsg::ring::Free) { @@ -265,14 +249,14 @@ static bool verify_rings_free(kickmsg::SharedRegion& region) static bool verify_slots(kickmsg::SharedRegion& region) { auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); - std::vector in_free(hdr->pool_size, false); - uint32_t top = kickmsg::tagged_idx(hdr->free_top.load(std::memory_order_acquire)); + std::vector in_free(header->pool_size, false); + uint32_t top = kickmsg::tagged_idx(header->free_top.load(std::memory_order_acquire)); while (top != kickmsg::INVALID_SLOT) { - if (top >= hdr->pool_size) + if (top >= header->pool_size) { std::fprintf(stderr, " [FAIL] free stack contains out-of-range index %u\n", top); return false; @@ -285,18 +269,18 @@ static bool verify_slots(kickmsg::SharedRegion& region) } in_free[top] = true; - auto* slot = kickmsg::slot_at(base, hdr, top); + auto* slot = kickmsg::slot_at(base, region.geometry(), top); top = slot->next_free; } bool ok = true; - for (uint32_t i = 0; i < hdr->pool_size; ++i) + for (uint32_t i = 0; i < header->pool_size; ++i) { if (in_free[i]) { continue; } - auto* slot = kickmsg::slot_at(base, hdr, i); + auto* slot = kickmsg::slot_at(base, region.geometry(), i); uint32_t rc = slot->refcount; if (rc != 0) { @@ -439,9 +423,8 @@ int main(int argc, char** argv) steals += region.repair_locked_entries(); std::size_t const reclaimed = region.reclaim_orphaned_slots(); - // Each steal can orphan at most one slot ref (entry_steal_and_clear - // deliberately leaks the displaced reference); more reclaims than - // steals means the normal path leaked. + // Repair preserves slot claims. Only terminated publishers can orphan + // references; more reclaims than steals indicates a normal-path leak. std::printf(" Reclaimed slots: %zu (steal budget %" PRIu64 ")\n", reclaimed, steals); if (reclaimed > steals) { diff --git a/tests/stress/big_payload.cc b/tests/stress/big_payload.cc index 8a09ec7..a3cafd8 100644 --- a/tests/stress/big_payload.cc +++ b/tests/stress/big_payload.cc @@ -53,37 +53,37 @@ namespace return false; } - BigHeader hdr; - std::memcpy(&hdr, data, sizeof(hdr)); + BigHeader header; + std::memcpy(&header, data, sizeof(header)); - if (hdr.magic != BigHeader::MAGIC - or hdr.pub_id >= static_cast(NUM_PUBS) - or hdr.byte_count != BIG_BODY_SIZE) + if (header.magic != BigHeader::MAGIC + or header.pub_id >= static_cast(NUM_PUBS) + or header.byte_count != BIG_BODY_SIZE) { std::fprintf(stderr, " [FAIL] sub%d (%s): bad header (magic=%08x pub=%u bytes=%u)\n", - sub_id, pass_label, hdr.magic, hdr.pub_id, hdr.byte_count); + sub_id, pass_label, header.magic, header.pub_id, header.byte_count); return false; } uint8_t const* body = data + sizeof(BigHeader); for (std::size_t i = 0; i < BIG_BODY_SIZE; ++i) { - if (body[i] != pattern_byte(hdr.pub_id, hdr.seq, i)) + if (body[i] != pattern_byte(header.pub_id, header.seq, i)) { std::fprintf(stderr, " [FAIL] sub%d (%s): torn body at byte %zu " "(pub %u seq %u: got %02x, want %02x)\n", - sub_id, pass_label, i, hdr.pub_id, hdr.seq, - body[i], pattern_byte(hdr.pub_id, hdr.seq, i)); + sub_id, pass_label, i, header.pub_id, header.seq, + body[i], pattern_byte(header.pub_id, header.seq, i)); return false; } } uint64_t sum = kickmsg::hash::fnv1a_64(body, BIG_BODY_SIZE); - if (sum != hdr.checksum) + if (sum != header.checksum) { std::fprintf(stderr, " [FAIL] sub%d (%s): checksum mismatch " "(pub %u seq %u: got %016" PRIx64 ", want %016" PRIx64 ")\n", - sub_id, pass_label, hdr.pub_id, hdr.seq, sum, hdr.checksum); + sub_id, pass_label, header.pub_id, header.seq, sum, header.checksum); return false; } return true; @@ -92,17 +92,17 @@ namespace void check_reorder(uint8_t const* data, std::vector& last_seq, BigSubStats& stats, int sub_id) { - BigHeader hdr; - std::memcpy(&hdr, data, sizeof(hdr)); - auto& prev = last_seq[hdr.pub_id]; - if (prev != UINT32_MAX and hdr.seq <= prev) + BigHeader header; + std::memcpy(&header, data, sizeof(header)); + auto& prev = last_seq[header.pub_id]; + if (prev != UINT32_MAX and header.seq <= prev) { std::fprintf(stderr, " [FAIL] sub%d: pub %u seq %u after seq %u (reorder)\n", - sub_id, hdr.pub_id, hdr.seq, prev); + sub_id, header.pub_id, header.seq, prev); ++stats.reordered; return; } - prev = hdr.seq; + prev = header.seq; ++stats.received; } } @@ -148,13 +148,13 @@ bool run_big_payload() body[b] = pattern_byte(static_cast(pub_id), i, b); } - BigHeader hdr; - hdr.magic = BigHeader::MAGIC; - hdr.pub_id = static_cast(pub_id); - hdr.seq = i; - hdr.byte_count = static_cast(BIG_BODY_SIZE); - hdr.checksum = kickmsg::hash::fnv1a_64(body, BIG_BODY_SIZE); - std::memcpy(buf.data(), &hdr, sizeof(hdr)); + BigHeader header; + header.magic = BigHeader::MAGIC; + header.pub_id = static_cast(pub_id); + header.seq = i; + header.byte_count = static_cast(BIG_BODY_SIZE); + header.checksum = kickmsg::hash::fnv1a_64(body, BIG_BODY_SIZE); + std::memcpy(buf.data(), &header, sizeof(header)); int32_t rc; while ((rc = pub.send(buf.data(), buf.size())) < 0) diff --git a/tests/stress/common.cc b/tests/stress/common.cc index 84d7b06..ff38ad1 100644 --- a/tests/stress/common.cc +++ b/tests/stress/common.cc @@ -300,7 +300,7 @@ bool verify_gc_zero(kickmsg::SharedRegion& region, kickmsg::channel::Config cons uint64_t dropped = 0; for (uint32_t i = 0; i < cfg.max_subscribers; ++i) { - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), i); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), i); dropped += ring->dropped_count.load(std::memory_order_acquire); } @@ -333,11 +333,11 @@ bool verify_gc_zero(kickmsg::SharedRegion& region, kickmsg::channel::Config cons bool verify_pool_free(kickmsg::SharedRegion& region, kickmsg::channel::Config const& cfg) { auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); std::vector seen(cfg.pool_size, false); uint32_t count = 0; - uint32_t top = kickmsg::tagged_idx(hdr->free_top.load(std::memory_order_acquire)); + uint32_t top = kickmsg::tagged_idx(header->free_top.load(std::memory_order_acquire)); while (top != kickmsg::INVALID_SLOT) { @@ -354,7 +354,7 @@ bool verify_pool_free(kickmsg::SharedRegion& region, kickmsg::channel::Config co seen[top] = true; ++count; - auto* slot = kickmsg::slot_at(base, hdr, top); + auto* slot = kickmsg::slot_at(base, region.geometry(), top); top = slot->next_free; } @@ -370,11 +370,10 @@ bool verify_pool_free(kickmsg::SharedRegion& region, kickmsg::channel::Config co bool verify_rings_inactive(kickmsg::SharedRegion& region, kickmsg::channel::Config const& cfg) { auto* base = region.base(); - auto* hdr = region.header(); for (uint32_t i = 0; i < cfg.max_subscribers; ++i) { - auto* ring = kickmsg::sub_ring_at(base, hdr, i); + auto* ring = kickmsg::sub_ring_at(base, region.geometry(), i); uint32_t packed = ring->state_flight.load(std::memory_order_acquire); if (kickmsg::ring::get_state(packed) != kickmsg::ring::Free) { @@ -394,11 +393,10 @@ bool verify_rings_inactive(kickmsg::SharedRegion& region, kickmsg::channel::Conf bool verify_refcounts_zero(kickmsg::SharedRegion& region, kickmsg::channel::Config const& cfg) { auto* base = region.base(); - auto* hdr = region.header(); for (uint32_t i = 0; i < cfg.pool_size; ++i) { - auto* slot = kickmsg::slot_at(base, hdr, i); + auto* slot = kickmsg::slot_at(base, region.geometry(), i); uint32_t rc = slot->refcount; if (rc != 0) { diff --git a/tests/stress/gc_recovery.cc b/tests/stress/gc_recovery.cc index 1059562..e595ca7 100644 --- a/tests/stress/gc_recovery.cc +++ b/tests/stress/gc_recovery.cc @@ -17,7 +17,7 @@ bool run_gc_recovery() shm_name, kickmsg::channel::PubSub, cfg, "gc_test"); auto* base = region.base(); - auto* h = region.header(); + auto* header = region.header(); bool ok = true; @@ -33,12 +33,12 @@ bool run_gc_recovery() // Now poison the committed entry { - auto* ring = kickmsg::sub_ring_at(base, h, 0); + auto* ring = kickmsg::sub_ring_at(base, region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); uint64_t wp = ring->write_pos.load(std::memory_order_acquire); if (wp > 0) { - entries[(wp - 1) & h->sub_ring_mask].sequence = kickmsg::seq_lock(wp - 1); + entries[(wp - 1) & header->sub_ring_mask].sequence = kickmsg::seq_lock(wp - 1); } } @@ -52,8 +52,8 @@ bool run_gc_recovery() // Simulate an orphaned slot: pop one from the free stack (no ring references it) // and set its refcount > 0 as if a publisher crashed after refcount pre-set. { - uint32_t idx = kickmsg::treiber_pop(h->free_top, base, h); - auto* slot = kickmsg::slot_at(base, h, idx); + uint32_t idx = kickmsg::treiber_pop(header->free_top, base, region.geometry()); + auto* slot = kickmsg::slot_at(base, region.geometry(), idx); slot->refcount = 3; } diff --git a/tests/stress/live_repair.cc b/tests/stress/live_repair.cc index 73d3588..f3f782b 100644 --- a/tests/stress/live_repair.cc +++ b/tests/stress/live_repair.cc @@ -18,7 +18,7 @@ bool run_live_repair() shm_name, kickmsg::channel::PubSub, cfg, "live_repair"); auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); constexpr int NUM_PUBS = 4; constexpr int NUM_SUBS = 4; @@ -119,14 +119,14 @@ bool run_live_repair() { kickmsg::sleep(10ms); - auto* ring = kickmsg::sub_ring_at(base, hdr, 0); + auto* ring = kickmsg::sub_ring_at(base, region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // fetch_add, NOT load+store: real publishers fetch_add this // counter concurrently and a lost increment would hand two // publishers the same position (harness-induced corruption). uint64_t wp = ring->write_pos.fetch_add(1, std::memory_order_acq_rel); - auto& e = entries[(wp) & hdr->sub_ring_mask]; + auto& e = entries[(wp) & header->sub_ring_mask]; // CAS like a real publisher, never a blind store: a descheduled // injector's late store would land over an already-repaired @@ -134,9 +134,9 @@ bool run_live_repair() // crash can produce. If the entry moved on first, skip the // injection (the claimed position heals as Case-B residue). uint64_t prev = 0; - if (wp >= hdr->sub_ring_capacity) + if (wp >= header->sub_ring_capacity) { - prev = wp - hdr->sub_ring_capacity + 1; + prev = wp - header->sub_ring_capacity + 1; } uint64_t observed = e.sequence.load(std::memory_order_acquire); if (not kickmsg::seq_is_locked(observed) diff --git a/tests/stress/treiber.cc b/tests/stress/treiber.cc index 6314a6f..db04c7f 100644 --- a/tests/stress/treiber.cc +++ b/tests/stress/treiber.cc @@ -16,7 +16,7 @@ bool run_treiber_stress() shm_name, kickmsg::channel::PubSub, cfg, "treiber"); auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); constexpr int NUM_THREADS = 8; int const CYCLES = 100000 / TSAN_SCALE; @@ -26,7 +26,7 @@ bool run_treiber_stress() { for (int i = 0; i < CYCLES; ++i) { - uint32_t idx = kickmsg::treiber_pop(hdr->free_top, base, hdr); + uint32_t idx = kickmsg::treiber_pop(header->free_top, base, region.geometry()); if (idx == kickmsg::INVALID_SLOT) { contention_hits.fetch_add(1); @@ -35,11 +35,11 @@ bool run_treiber_stress() continue; } - auto* slot = kickmsg::slot_at(base, hdr, idx); + auto* slot = kickmsg::slot_at(base, region.geometry(), idx); auto* data = kickmsg::slot_data(slot); std::memset(data, static_cast(idx & 0xFF), cfg.max_payload_size); - kickmsg::treiber_push(hdr->free_top, slot, idx); + kickmsg::treiber_push(header->free_top, slot, idx); } }; diff --git a/tests/tsan.supp b/tests/tsan.supp index 6cd5f3c..e8d90e2 100644 --- a/tests/tsan.supp +++ b/tests/tsan.supp @@ -1,33 +1,5 @@ -# Blackboard value cells: a detected-and-discarded race on the payload bytes. +# ThreadSanitizer suppressions, passed via TSAN_OPTIONS by CI and tests/soak_all.sh. # -# A writer copies into the cell readers are NOT on, so the common case has no -# concurrent access. A reader overtaken by CELLS_PER_KEY writes does overlap -# the writer's memcpy; its publish-word re-check detects exactly that and -# discards the copy, so no torn value is ever returned (the blackboard stress -# scenario asserts torn == 0 over hundreds of thousands of reads). The -# byte-level overlap is still a data race in the C11 model, and TSAN's slowdown -# widens the overtake window enough that it fires at any write rate. -# -# Same class as the seqlock read in Subscriber::try_receive_view(), which is -# race-free only because of its refcount pin. A blackboard cannot pin: that -# would let a crashed reader block a writer. -race:bb_copy_payload - -# Blackboard key bytes: Reader::resolve() compares e->key unlocked, and a slot -# being re-claimed has its key rewritten by declare() under the board lock. A -# reader reaches that compare only through a stale Active state -- the claim -# publishes Active last, and that release store is what orders the key bytes -# for every reader that sees it. read() then re-checks the entry's tenancy and -# retries when it moved, so a torn compare costs a retry and can never return -# another key's value. -race:bb_key_equals - -# Blackboard key bytes, copy side: snapshot() reads them under the board lock, -# but keys()/read_all() walk the entry array unlocked and hit the same -# claim_free_slot() write that bb_key_equals races. A torn copy is discarded by -# the tenancy re-check that follows it, so it can never be returned. -race:bb_read_key - -# All three helpers are noinline (KICKMSG_BB_NOINLINE in src/Blackboard.cc) so these -# suppressions match one frame each: write(), read() and resolve() themselves -# stay checked, as does the whole publish/receive/repair core. +# Intentionally empty. A new entry needs a comment proving the race benign; a +# seqlock-style overlap does not qualify -- move those bytes as relaxed atomics +# instead, as src/Blackboard.cc does for value payloads and key text. diff --git a/tests/unit/blackboard-t.cc b/tests/unit/blackboard-t.cc index 5ecbb02..d607496 100644 --- a/tests/unit/blackboard-t.cc +++ b/tests/unit/blackboard-t.cc @@ -58,15 +58,14 @@ class BlackboardTest : public ::testing::Test /// having gone away without running its destructor. static void orphan(Blackboard& bb, char const* key) { - for (uint32_t i = 0; i < bb.header()->capacity; ++i) + for (uint32_t i = 0; i < bb.capacity(); ++i) { auto* e = entry(bb, i); if (e->state.load(std::memory_order_acquire) != blackboard::Active) { continue; } - if (::strnlen(e->key, blackboard::KEY_MAX) != std::strlen(key) - or std::memcmp(e->key, key, std::strlen(key)) != 0) + if (bb_load_key(e) != key) { continue; } @@ -536,7 +535,7 @@ TEST_F(BlackboardTest, CorruptValueLenIsClamped) ASSERT_FALSE(w.write(Sample{1, 1})); auto* h = bb.header(); - auto* cell = bb_cell_at(static_cast(h), 0, 1); + auto* cell = bb_cell_at(static_cast(h), bb.geometry(), 0, 1); cell->value_len.store(0xFFFFFFFFu, std::memory_order_relaxed); std::vector out; @@ -545,13 +544,76 @@ TEST_F(BlackboardTest, CorruptValueLenIsClamped) EXPECT_LE(result.len, bb.max_value_size()); } +// A peer rewrites the geometry after open: every walk and cell offset must keep +// using the values validated at open. +TEST_F(BlackboardTest, HandlesUseTheGeometryValidatedAtOpen) +{ + auto bb = open(); + auto writer = bb.declare("k"); + auto reader = bb.observe("k"); + ASSERT_FALSE(writer.write(Sample{7, 3})); + + auto* header = bb.header(); + header->capacity = blackboard::MAX_CAPACITY; + header->max_value_size = blackboard::MAX_VALUE_SIZE; + + EXPECT_EQ(bb.capacity(), small_cfg().capacity); + EXPECT_EQ(bb.max_value_size(), small_cfg().max_value_size); + + Sample got{}; + ASSERT_FALSE(reader.read(got).ec); + EXPECT_EQ(got.id, 7u); + EXPECT_EQ(got.state, 3u); + + std::vector big(small_cfg().max_value_size + 1, 0); + EXPECT_EQ(writer.write(big.data(), big.size()), std::make_error_code(std::errc::message_size)); + ASSERT_FALSE(writer.write(Sample{8, 4})); + ASSERT_FALSE(reader.read(got).ec); + EXPECT_EQ(got.id, 8u); + + EXPECT_EQ(bb.snapshot().size(), 1u); + EXPECT_EQ(bb.keys().size(), 1u); + EXPECT_EQ(bb.read_all().size(), 1u); + EXPECT_EQ(bb.sweep_stale(), 0u); + auto other = bb.declare("other"); + EXPECT_FALSE(other.release()); + EXPECT_FALSE(writer.release()); +} + +// Payloads move as whole words; a length ending mid-word must round-trip exactly, +// including after a longer value left bytes in the same cell. +TEST_F(BlackboardTest, PartialWordValuesRoundTrip) +{ + auto bb = open(); + auto writer = bb.declare("k"); + auto reader = bb.observe("k"); + + std::vector in(small_cfg().max_value_size); + for (std::size_t i = 0; i < in.size(); ++i) + { + in[i] = static_cast(i * 7 + 1); + } + ASSERT_FALSE(writer.write(in.data(), in.size())); + + for (std::size_t len = 0; len <= 2 * sizeof(uint64_t) + 1; ++len) + { + ASSERT_FALSE(writer.write(in.data(), len)) << len; + std::vector out; + auto result = reader.read(out); + ASSERT_FALSE(result.ec) << len; + ASSERT_EQ(out.size(), len); + EXPECT_TRUE(std::equal(out.begin(), out.end(), in.begin())) << len; + } +} + TEST_F(BlackboardTest, CorruptKeyBytesAreNotOverread) { auto bb = open(); auto w = bb.declare("k"); auto* e = entry(bb, 0); - std::memset(e->key, 'x', sizeof(e->key)); // no NUL anywhere + std::string const no_nul(blackboard::KEY_MAX, 'x'); + bb_store_key(e, no_nul.data(), no_nul.size()); auto snap = bb.snapshot(); ASSERT_EQ(snap.size(), 1u); @@ -562,7 +624,12 @@ TEST_F(BlackboardTest, RejectsVersionMismatch) { auto bb = open(); bb.header()->version = blackboard::VERSION + 1; - EXPECT_THROW(Blackboard::try_open(NS, NAME), std::runtime_error); + EXPECT_THROW(Blackboard::try_open(NS, NAME), kickmsg::VersionMismatch); + + // A version-1 peer copies payloads and keys with plain memcpy, which would race ours. + bb.header()->version = 1; + EXPECT_THROW(Blackboard::try_open(NS, NAME), kickmsg::VersionMismatch); + EXPECT_THROW(Blackboard::open_or_create(NS, NAME, small_cfg()), kickmsg::VersionMismatch); bb.header()->version = blackboard::VERSION; } @@ -951,8 +1018,8 @@ namespace void set_key(BlackboardEntry* e, uint64_t kh) { - std::memset(e->key, 0, sizeof(e->key)); - std::memcpy(e->key, "victim", 6); + bb_store_key(e, "", 0); + bb_store_key(e, "victim", 6); e->key_hash.store(kh, std::memory_order_relaxed); } @@ -970,15 +1037,15 @@ namespace constexpr CrashPoint CRASH_POINTS[] = { // --- claim: Free -> Claiming -> Active, all under the lock --- {"claim/lock-taken-nothing-done", true, - [](BlackboardEntry* e, uint64_t) { zero_meta(e); std::memset(e->key, 0, sizeof(e->key)); + [](BlackboardEntry* e, uint64_t) { zero_meta(e); bb_store_key(e, "", 0); put(e, blackboard::Free); }}, {"claim/state-claiming", true, - [](BlackboardEntry* e, uint64_t) { zero_meta(e); std::memset(e->key, 0, sizeof(e->key)); + [](BlackboardEntry* e, uint64_t) { zero_meta(e); bb_store_key(e, "", 0); put(e, blackboard::Claiming); }}, {"claim/key-written", true, [](BlackboardEntry* e, uint64_t) { zero_meta(e); - std::memset(e->key, 0, sizeof(e->key)); - std::memcpy(e->key, "victim", 6); + bb_store_key(e, "", 0); + bb_store_key(e, "victim", 6); put(e, blackboard::Claiming); }}, {"claim/fully-filled-not-committed", true, [](BlackboardEntry* e, uint64_t kh) { zero_meta(e); dead_owner(e); set_key(e, kh); @@ -1014,7 +1081,7 @@ namespace [](BlackboardEntry* e, uint64_t kh) { zero_meta(e); dead_owner(e); set_key(e, kh); put(e, blackboard::Free); }}, {"sweepfree/complete-lock-not-dropped", true, - [](BlackboardEntry* e, uint64_t) { zero_meta(e); std::memset(e->key, 0, sizeof(e->key)); + [](BlackboardEntry* e, uint64_t) { zero_meta(e); bb_store_key(e, "", 0); put(e, blackboard::Free); }}, // --- forged states, lock NOT held: a corrupt peer can write these and @@ -1024,7 +1091,7 @@ namespace put(e, blackboard::Claiming); }}, {"corrupt/active-without-key", false, [](BlackboardEntry* e, uint64_t) { zero_meta(e); dead_owner(e); - std::memset(e->key, 0, sizeof(e->key)); + bb_store_key(e, "", 0); put(e, blackboard::Active); }}, {"corrupt/free-carrying-identity", false, [](BlackboardEntry* e, uint64_t kh) { zero_meta(e); dead_owner(e); set_key(e, kh); @@ -1072,7 +1139,7 @@ TEST_F(BlackboardTest, CrashPointMatrix) if (st == blackboard::Active) { ok = ok and e->key_hash.load(std::memory_order_relaxed) != 0; - ok = ok and ::strnlen(e->key, blackboard::KEY_MAX) != 0; + ok = ok and not bb_load_key(e).empty(); } if (st == blackboard::Free) { diff --git a/tests/unit/node-t.cc b/tests/unit/node-t.cc index cbfa7e1..2311c62 100644 --- a/tests/unit/node-t.cc +++ b/tests/unit/node-t.cc @@ -1,6 +1,7 @@ #include +#include "kickmsg/Naming.h" #include "kickmsg/Node.h" class NodeTest : public ::testing::Test @@ -14,11 +15,19 @@ class NodeTest : public ::testing::Test } } + /// Unlinked up front too, so a region left by a crashed run cannot leak in. void track(std::string name) { + kickmsg::SharedMemory::unlink(name); shm_names_.push_back(std::move(name)); } + /// Same composition as Node: on macOS the shm name is a hash, not "/ns_suffix". + void track(char const* ns, char const* suffix) + { + track(kickmsg::compose_shm_name(ns, suffix)); + } + kickmsg::channel::Config small_cfg() { kickmsg::channel::Config cfg; @@ -36,7 +45,7 @@ class NodeTest : public ::testing::Test TEST_F(NodeTest, AdvertiseAndSubscribe) { // Topic-centric: SHM name is /{prefix}_{topic}, no node name in path - track("/test_data"); + track("test", "data"); kickmsg::Node pub_node("pubnode", "test"); auto pub = pub_node.advertise("data", small_cfg()); @@ -62,7 +71,7 @@ TEST_F(NodeTest, AdvertiseTwiceDoesNotWipeLiveRegion) // SharedRegion::create() (O_TRUNC + memset) on the live segment. A // subscriber that joined after the first advertise must keep working // across the second advertise. - track("/test_dup"); + track("test", "dup"); kickmsg::Node node("node", "test"); auto pub1 = node.advertise("dup", small_cfg()); @@ -100,7 +109,7 @@ TEST_F(NodeTest, NamingConventions) TEST_F(NodeTest, JoinBroadcastTwoNodes) { - track("/test_broadcast_events"); + track("test", "broadcast_events"); auto cfg = small_cfg(); @@ -131,7 +140,7 @@ TEST_F(NodeTest, JoinBroadcastTwoNodes) TEST_F(NodeTest, MailboxPattern) { - track("/test_nodeA_mbx_inbox"); + track("test", "nodeA_mbx_inbox"); auto cfg = small_cfg(); @@ -167,7 +176,7 @@ namespace TEST_F(NodeTest, TopicSchemaBakedViaAdvertise) { - track("/test_imu"); + track("test", "imu"); auto cfg = small_cfg(); cfg.schema = make_node_schema("app/Imu", 2, 0xAA); @@ -197,7 +206,7 @@ TEST_F(NodeTest, TryClaimTopicSchemaLateBinding) { // Late-arrival flow: subscriber creates the region, publisher arrives // and claims the schema via the Node API. - track("/test_telemetry"); + track("test", "telemetry"); auto cfg = small_cfg(); @@ -238,7 +247,7 @@ TEST_F(NodeTest, UnlinkTopicRemovesShm) // the unlink path (without it, the /dev/shm entry would persist); on // Windows the last-handle-close already removed the mapping and // unlink is a harmless no-op -- both produce the same post-condition. - track("/test_ephemeral"); + track("test", "ephemeral"); { kickmsg::Node node("node", "test"); @@ -294,7 +303,7 @@ TEST_F(NodeTest, SubscribeOrCreateTwiceReusesSameRegion) // on the same topic yields independent handles that wrap the SAME // underlying mmap (emplace_or_reuse dedupes). A publisher on one // handle must be visible to a subscriber on either. - track("/test_shared"); + track("test", "shared"); kickmsg::Node node("node", "test"); auto cfg = small_cfg(); @@ -333,7 +342,7 @@ TEST_F(NodeTest, RosStyleTopicNamesAreSanitizedIntoShmPath) // sanitize_shm_component: ROS-style absolute paths with interior '/' // must round-trip into a POSIX-valid "/_robot.arm.joint1" // region, reachable by a peer that passes the same raw topic string. - track("/test.ns_robot.arm.joint1"); + track("test.ns", "robot.arm.joint1"); kickmsg::Node pub_node("drv", "/test/ns"); auto pub = pub_node.advertise("/robot/arm/joint1", small_cfg()); @@ -359,7 +368,7 @@ TEST_F(NodeTest, ShmNameCollisionDetectedByIdentityStamp) // two distinct logical topics land on the same shm name "/test_a_b". // The identity hash stamped at create (over the RAW topic string) must // reject the open instead of silently sharing the region. - track("/test_a_b"); + track("test", "a_b"); kickmsg::Node pub_node("pubnode", "test"); auto pub = pub_node.advertise("a:b", small_cfg()); @@ -399,7 +408,7 @@ TEST_F(NodeTest, EmptyTopicNameThrows) TEST_F(NodeTest, MailboxMultipleWriters) { - track("/test_owner_mbx_inbox"); + track("test", "owner_mbx_inbox"); auto cfg = small_cfg(); @@ -430,7 +439,7 @@ TEST_F(NodeTest, MailboxMultipleWriters) TEST_F(NodeTest, RelaxedMailboxOwnerFirst) { - track("/test_owner_mbx_inbox"); + track("test", "owner_mbx_inbox"); auto cfg = small_cfg(); @@ -450,7 +459,7 @@ TEST_F(NodeTest, RelaxedMailboxOwnerFirst) TEST_F(NodeTest, RelaxedMailboxSenderFirst) { - track("/test_owner_mbx_inbox"); + track("test", "owner_mbx_inbox"); auto cfg = small_cfg(); @@ -473,7 +482,7 @@ TEST_F(NodeTest, RelaxedMailboxForcesMaxSubscribersOne) { // If a caller passes cfg.max_subscribers != 1, the mailbox APIs must // override it. Verify by reading back the region info. - track("/test_owner_mbx_inbox"); + track("test", "owner_mbx_inbox"); auto cfg = small_cfg(); cfg.max_subscribers = 4; // mailbox should clamp to 1 diff --git a/tests/unit/publisher-t.cc b/tests/unit/publisher-t.cc index cfe9230..8f3abfd 100644 --- a/tests/unit/publisher-t.cc +++ b/tests/unit/publisher-t.cc @@ -5,6 +5,13 @@ using namespace std::chrono; +// One reservation, one owner. +static_assert(std::is_move_constructible_v); +static_assert(std::is_move_assignable_v); +static_assert(not std::is_copy_constructible_v, + "copying would give two objects the same slot to publish and release"); +static_assert(not std::is_copy_assignable_v); + class PublisherTest : public ::testing::Test { public: @@ -31,6 +38,84 @@ class PublisherTest : public ::testing::Test } }; +// Moving a publisher invalidates its outstanding handle, and nothing else can +// publish that reservation, so the slot must go back to the pool at once. +TEST_F(PublisherTest, MovingAPublisherDetachesItsOutstandingSlot) +{ + auto cfg = default_cfg(); + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + kickmsg::Subscriber sub(region); + + auto held = region.stats().pool_free; + { + kickmsg::Publisher first(region); + auto slot = first.allocate(); + ASSERT_TRUE(slot.valid()); + uint32_t val = 42; + slot.write(&val, sizeof(val)); + held = region.stats().pool_free; + + kickmsg::Publisher second(std::move(first)); + EXPECT_FALSE(slot.valid()) << "handle still claims a reservation it lost"; + EXPECT_EQ(slot.publish(sizeof(val)), 0u); + EXPECT_FALSE(sub.try_receive().has_value()); + + EXPECT_EQ(region.stats().pool_free, held + 1) + << "an unreachable reservation stayed pinned after the move"; + } + EXPECT_EQ(region.stats().pool_free, cfg.pool_size); +} + +// Move-assignment used to hand the destination the source's reservation +// counter. A handle issued earlier by the destination then matched again, so +// two handles believed they owned one slot -- and the stale one's destructor +// returned the live one's slot to the pool while it was still being written. +TEST_F(PublisherTest, MoveAssignmentDoesNotRevalidateOlderHandles) +{ + auto cfg = default_cfg(); + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + + kickmsg::Publisher first(region); + kickmsg::Publisher second(region); + + // Both counters are at 1, which is what used to make these collide. + auto stale = first.allocate(); + auto live = second.allocate(); + ASSERT_TRUE(stale.valid()); + ASSERT_TRUE(live.valid()); + + first = std::move(second); + EXPECT_FALSE(stale.valid()) << "an older handle matched the inherited counter"; + EXPECT_FALSE(live.valid()); + EXPECT_EQ(region.stats().pool_free, cfg.pool_size) + << "a reservation no handle can publish stayed pinned after the move"; + + auto before = region.stats().pool_free; + { + auto sink = std::move(stale); // destructor runs here + } + EXPECT_EQ(region.stats().pool_free, before) + << "a stale handle released a slot it did not own"; +} + +TEST_F(PublisherTest, ReusedMovedFromPublisherDoesNotRevalidateOlderHandles) +{ + auto cfg = default_cfg(); + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + kickmsg::Subscriber sub(region); + + kickmsg::Publisher first(region); + auto stale = first.allocate(); + ASSERT_TRUE(stale.valid()); + + kickmsg::Publisher second(std::move(first)); + auto fresh = first.allocate(); + ASSERT_TRUE(fresh.valid()); + EXPECT_FALSE(stale.valid()) << "the moved-from publisher reissued an old id"; + EXPECT_EQ(stale.publish(0), 0u); + EXPECT_FALSE(sub.try_receive().has_value()); +} + TEST_F(PublisherTest, SendReceiveSingleMessage) { auto cfg = default_cfg(); @@ -59,14 +144,13 @@ TEST_F(PublisherTest, AllocatePublishSeparately) kickmsg::Subscriber sub(region); kickmsg::Publisher pub(region); - auto a = pub.allocate(); - ASSERT_NE(a.data, nullptr); - EXPECT_GE(a.max_size, sizeof(uint32_t)); + auto slot = pub.allocate(); + ASSERT_TRUE(slot.valid()); + EXPECT_GE(slot.max_size(), sizeof(uint32_t)); uint32_t val = 42; - std::memcpy(a.data, &val, sizeof(val)); - std::size_t delivered = pub.publish(sizeof(val)); - EXPECT_EQ(delivered, 1u); + EXPECT_EQ(slot.write(&val, sizeof(val)), sizeof(val)); + EXPECT_EQ(slot.publish(sizeof(val)), 1u); auto sample = sub.try_receive(); ASSERT_TRUE(sample.has_value()); @@ -82,9 +166,9 @@ TEST_F(PublisherTest, AllocateExposesMaxSize) auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); kickmsg::Publisher pub(region); - auto a = pub.allocate(); - ASSERT_NE(a.data, nullptr); - EXPECT_EQ(a.max_size, cfg.max_payload_size); + auto slot = pub.allocate(); + ASSERT_TRUE(slot.valid()); + EXPECT_EQ(slot.max_size(), cfg.max_payload_size); } TEST_F(PublisherTest, SendReturnsEmsgsize) @@ -109,15 +193,17 @@ TEST_F(PublisherTest, PublishOversizedLenReturnsZeroAndRecyclesSlot) kickmsg::Subscriber sub(region); kickmsg::Publisher pub(region); - auto a = pub.allocate(); - ASSERT_NE(a.data, nullptr); - EXPECT_EQ(pub.publish(cfg.max_payload_size + 1), 0u); + auto slot = pub.allocate(); + ASSERT_TRUE(slot.valid()); + EXPECT_EQ(slot.publish(cfg.max_payload_size + 1), 0u); // The oversized publish must have recycled the pending slot: with // pool_size == 1, a leak would make this allocate() fail. - auto b = pub.allocate(); - ASSERT_NE(b.data, nullptr); - EXPECT_EQ(pub.publish(sizeof(uint32_t)), 1u); + auto next = pub.allocate(); + ASSERT_TRUE(next.valid()); + uint32_t val = 42; + EXPECT_EQ(next.write(&val, sizeof(val)), sizeof(val)); + EXPECT_EQ(next.publish(sizeof(val)), 1u); } // Pins the slot-recycling contract that caused a multi-hour soak hang: a slot @@ -293,7 +379,7 @@ TEST_F(PublisherTest, SelfRepairCaseA_LockedSequence) // Poison entry at idx=0: simulate a publisher that locked pos=4 then // crashed. write_pos is already 4 (from the 4 publishes above). - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // Advance write_pos past pos=4 so the ring has wrapped. @@ -381,7 +467,7 @@ TEST_F(PublisherTest, SelfRepairCaseB_StaleEntry) } // Entry at idx=0 has seq=1 (committed for pos=0). - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // Simulate: publisher claimed pos=4 (fetch_add) but crashed before diff --git a/tests/unit/region-t.cc b/tests/unit/region-t.cc index 84b7959..b810644 100644 --- a/tests/unit/region-t.cc +++ b/tests/unit/region-t.cc @@ -21,9 +21,7 @@ using namespace std::chrono; namespace { - // CACHE_LINE-aligned heap buffer for the injected-region tests. - // posix_memalign is POSIX-only; Windows uses _aligned_malloc, whose - // memory MUST be released with _aligned_free (not free()). + // Windows aligned allocations must be released with _aligned_free. void* aligned_buffer_alloc(std::size_t align, std::size_t size) { #if defined(_WIN32) @@ -78,19 +76,19 @@ TEST_F(RegionTest, CreateAndValidateHeader) { auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg, "test"); - auto* hdr = region.header(); - - EXPECT_EQ(hdr->magic, kickmsg::MAGIC); - EXPECT_EQ(hdr->version, kickmsg::VERSION); - EXPECT_EQ(hdr->channel_type, kickmsg::channel::PubSub); - EXPECT_EQ(hdr->max_subs, cfg.max_subscribers); - EXPECT_EQ(hdr->sub_ring_capacity, cfg.sub_ring_capacity); - EXPECT_EQ(hdr->sub_ring_mask, cfg.sub_ring_capacity - 1); - EXPECT_EQ(hdr->pool_size, cfg.pool_size); - EXPECT_EQ(hdr->slot_data_size, cfg.max_payload_size); - EXPECT_EQ(hdr->creator_name_len, 4u); - - std::string creator(kickmsg::header_creator_name(hdr), hdr->creator_name_len); + auto* header = region.header(); + + EXPECT_EQ(header->magic, kickmsg::MAGIC); + EXPECT_EQ(header->version, kickmsg::VERSION); + EXPECT_EQ(header->channel_type, kickmsg::channel::PubSub); + EXPECT_EQ(header->max_subs, cfg.max_subscribers); + EXPECT_EQ(header->sub_ring_capacity, cfg.sub_ring_capacity); + EXPECT_EQ(header->sub_ring_mask, cfg.sub_ring_capacity - 1); + EXPECT_EQ(header->pool_size, cfg.pool_size); + EXPECT_EQ(header->slot_data_size, cfg.max_payload_size); + EXPECT_EQ(header->creator_name_len, 4u); + + std::string creator(kickmsg::header_creator_name(header), header->creator_name_len); EXPECT_EQ(creator, "test"); } @@ -112,9 +110,7 @@ TEST_F(RegionTest, OpenNonexistentThrows) #ifndef _WIN32 TEST(SharedMemoryTest, TryOpenOnSizeZeroSegmentReturnsFalse) { - // A creator that did shm_open(O_CREAT) but not yet ftruncate() leaves a - // size-0 object. try_open must report not-ready (so create_or_open / - // spin_open retry) rather than mmap(., 0, .) -> EINVAL -> throw. + // An object created before ftruncate has size zero; opening must retry. char const* name = "/kickmsg_test_size0"; ::shm_unlink(name); int fd = ::shm_open(name, O_RDWR | O_CREAT, 0666); @@ -171,10 +167,7 @@ TEST_F(RegionTest, CreateOrOpenValidatesGeometryOnOpenBranch) auto creator = kickmsg::SharedRegion::create_or_open( SHM_NAME, kickmsg::channel::PubSub, cfg, "creator"); - // Corrupt a geometry field that config_hash does NOT cover (pool_offset - // is computed layout, not a cfg field). A second create_or_open hits - // the open branch with a matching config_hash, so only the geometry - // validation can catch it. + // Change an offset not covered by config_hash, then exercise the open branch. creator.header()->pool_offset = UINT64_MAX; EXPECT_THROW( @@ -219,11 +212,11 @@ TEST_F(RegionTest, HeaderStoresCreatorMetadata) auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg, "my_node"); - auto* hdr = region.header(); + auto* header = region.header(); - EXPECT_EQ(hdr->creator_pid, kickmsg::current_pid()); - EXPECT_GT(hdr->created_at_ns, 0u); - EXPECT_NE(hdr->config_hash, 0u); + EXPECT_EQ(header->creator_pid, kickmsg::current_pid()); + EXPECT_GT(header->created_at_ns, 0u); + EXPECT_NE(header->config_hash, 0u); } TEST_F(RegionTest, NonPowerOfTwoRingThrows) @@ -240,29 +233,29 @@ TEST_F(RegionTest, TreiberPopAllThenPushBack) auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); auto* base = region.base(); - auto* hdr = region.header(); + auto* header = region.header(); std::vector popped; for (uint32_t i = 0; i < cfg.pool_size; ++i) { - uint32_t idx = kickmsg::treiber_pop(hdr->free_top, base, hdr); + uint32_t idx = kickmsg::treiber_pop(header->free_top, base, region.geometry()); ASSERT_NE(idx, kickmsg::INVALID_SLOT) << "Pop failed at iteration " << i; popped.push_back(idx); } - EXPECT_EQ(kickmsg::treiber_pop(hdr->free_top, base, hdr), kickmsg::INVALID_SLOT); + EXPECT_EQ(kickmsg::treiber_pop(header->free_top, base, region.geometry()), kickmsg::INVALID_SLOT); for (auto idx : popped) { - auto* slot = kickmsg::slot_at(base, hdr, idx); - kickmsg::treiber_push(hdr->free_top, slot, idx); + auto* slot = kickmsg::slot_at(base, region.geometry(), idx); + kickmsg::treiber_push(header->free_top, slot, idx); } uint32_t count = 0; - uint32_t top = kickmsg::tagged_idx(hdr->free_top.load(std::memory_order_acquire)); + uint32_t top = kickmsg::tagged_idx(header->free_top.load(std::memory_order_acquire)); while (top != kickmsg::INVALID_SLOT) { - auto* slot = kickmsg::slot_at(base, hdr, top); + auto* slot = kickmsg::slot_at(base, region.geometry(), top); top = slot->next_free; ++count; } @@ -279,16 +272,16 @@ TEST_F(RegionTest, CollectGarbageReclaimsOrphanedSlots) auto region = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg); - auto* hdr = region.header(); + auto* header = region.header(); auto count_free = [&]() { uint32_t count = 0; - uint64_t top = hdr->free_top.load(std::memory_order_acquire); + uint64_t top = header->free_top.load(std::memory_order_acquire); uint32_t idx = kickmsg::tagged_idx(top); while (idx != kickmsg::INVALID_SLOT) { - auto* slot = kickmsg::slot_at(region.base(), hdr, idx); + auto* slot = kickmsg::slot_at(region.base(), region.geometry(), idx); idx = slot->next_free; ++count; } @@ -299,9 +292,9 @@ TEST_F(RegionTest, CollectGarbageReclaimsOrphanedSlots) for (int i = 0; i < 3; ++i) { - uint32_t idx = kickmsg::treiber_pop(hdr->free_top, region.base(), hdr); + uint32_t idx = kickmsg::treiber_pop(header->free_top, region.base(), region.geometry()); ASSERT_NE(idx, kickmsg::INVALID_SLOT); - auto* slot = kickmsg::slot_at(region.base(), hdr, idx); + auto* slot = kickmsg::slot_at(region.base(), region.geometry(), idx); slot->refcount.store(static_cast(cfg.max_subscribers), std::memory_order_release); } @@ -340,7 +333,7 @@ TEST_F(RegionTest, RepairLockedEntryUnblocksPublishing) ASSERT_TRUE(sample.has_value()); // Simulate a crash at pos=1: manually lock the entry - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // Advance write_pos to simulate that a publisher claimed pos=1 @@ -356,14 +349,10 @@ TEST_F(RegionTest, RepairLockedEntryUnblocksPublishing) uint64_t seq = e1.sequence.load(std::memory_order_acquire); EXPECT_EQ(seq, kickmsg::seq_skip(1)); - // The repaired entry should have INVALID_SLOT - uint32_t slot_idx = e1.slot_idx.load(std::memory_order_acquire); - EXPECT_EQ(slot_idx, kickmsg::INVALID_SLOT); + // The entry has no slot claim; repair must leave it unchanged. + EXPECT_EQ(kickmsg::meta_slot_biased(e1.meta.load(std::memory_order_acquire)), 0u); - // Now publish enough to wrap around: pos 2, 3, 4, 5 - // pos=4 wraps to idx=0 and expects prev_seq=1 (pos 0's committed seq) -- OK - // pos=5 wraps to idx=1 and expects prev_seq=2 (the repaired seq) -- this - // would fail with the old code that stored prev_seq instead of pos+1 + // Publish positions 2 through 5 to reuse both repaired entries. for (int i = 0; i < 4; ++i) { val = static_cast(200 + i); @@ -383,11 +372,8 @@ TEST_F(RegionTest, RepairLockedEntryUnblocksPublishing) TEST_F(RegionTest, RepairStaleEntryFromCrashedPublisherBeforeCasLock) { - // Case B: publisher claimed write_pos (fetch_add) but crashed before - // CAS-locking the entry. The entry still has the committed sequence - // from the previous wrap. After more than one full wrap, the entry - // is detectably stale (> 1 ring revolution behind) and - // repair_locked_entries() should advance it. + // Stage a publisher crash after claiming write_pos but before locking. + // The entry retains a committed sequence from more than one wrap ago. kickmsg::channel::Config cfg; cfg.max_subscribers = 1; @@ -410,18 +396,12 @@ TEST_F(RegionTest, RepairStaleEntryFromCrashedPublisherBeforeCasLock) } // Entry at idx=0 now has seq=1 (committed for pos=0). - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); - // Simulate: a publisher claimed pos=4 (fetch_add) targeting idx=0, - // then crashed before the CAS lock. The entry stays at seq=1. - // Advance write_pos past pos=4 by TWO more full wraps so the entry - // becomes > 1 wrap stale. - // write_pos after the 4 real publishes is 4. Set it to 4 + 2*cap = 12. + // Leave idx 0 at seq 1 and advance write_pos to 12, two wraps ahead. ring->write_pos.store(12, std::memory_order_release); - // Don't touch entries -- they keep their old sequences. Entry idx=0 - // has seq=1, but expected seq at pos=8 (the slot in the scan window) - // is 9. (pos=8 maps to idx=0 because 8 & 3 = 0.) 1 + 4 < 9 -> stale. + // The scan sees pos 8 at idx 0; seq 1 is more than one wrap behind seq 9. auto report = region.diagnose(); EXPECT_GT(report.locked_entries, 0u) @@ -450,7 +430,6 @@ TEST_F(RegionTest, RepairStaleEntryFromCrashedPublisherBeforeCasLock) TEST_F(RegionTest, RepairLockedEntryAtPositionZero) { // Edge case: crash at pos=0 where prev_seq was 0. - // Old code stored prev_seq=0, new code stores pos+1=1. kickmsg::channel::Config cfg; cfg.max_subscribers = 1; @@ -463,7 +442,7 @@ TEST_F(RegionTest, RepairLockedEntryAtPositionZero) kickmsg::Subscriber sub(region); // Simulate crash at pos=0: lock the entry, advance write_pos - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); ring->write_pos.store(1, std::memory_order_release); entries[0].sequence.store(kickmsg::seq_lock(0), std::memory_order_release); @@ -472,7 +451,8 @@ TEST_F(RegionTest, RepairLockedEntryAtPositionZero) EXPECT_EQ(repaired, 1u); EXPECT_EQ(entries[0].sequence.load(std::memory_order_acquire), kickmsg::seq_skip(0)); - EXPECT_EQ(entries[0].slot_idx.load(std::memory_order_acquire), kickmsg::INVALID_SLOT); + EXPECT_EQ(kickmsg::meta_slot_biased(entries[0].meta.load(std::memory_order_acquire)), + 0u); // Publishing should work: pos=1,2,3 use fresh indices, pos=4 wraps to idx=0 // and expects prev_seq=1 -- matches the repaired value @@ -522,7 +502,7 @@ TEST_F(RegionTest, DiagnoseDetectsLockedEntries) ASSERT_GE(pub.send(&val, sizeof(val)), 0); // Simulate a crashed publisher at pos=1: lock the entry - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); ring->write_pos.store(2, std::memory_order_release); entries[1].sequence.store(kickmsg::seq_lock(1), std::memory_order_release); @@ -548,7 +528,7 @@ TEST_F(RegionTest, DiagnoseDetectsStuckRings) auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); // Simulate a stuck ring: Free with stale in_flight - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); ring->state_flight.store( kickmsg::ring::make_packed(kickmsg::ring::Free, 1), std::memory_order_release); @@ -588,13 +568,13 @@ TEST_F(RegionTest, ResetRetiredRingsLeavesDrainingUntouched) auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); // Ring 0: retired (Free | in_flight=1) -- should be reset - auto* ring0 = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring0 = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); ring0->state_flight.store( kickmsg::ring::make_packed(kickmsg::ring::Free, 1), std::memory_order_release); // Ring 1: draining (Draining | in_flight=1) -- must NOT be touched - auto* ring1 = kickmsg::sub_ring_at(region.base(), region.header(), 1); + auto* ring1 = kickmsg::sub_ring_at(region.base(), region.geometry(), 1); ring1->state_flight.store( kickmsg::ring::make_packed(kickmsg::ring::Draining, 1), std::memory_order_release); @@ -620,18 +600,17 @@ TEST_F(RegionTest, ReclaimDeadRingsRecoversCrashedOwnerRing) { auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); - auto* h = region.header(); // Ring 0: a subscriber crashed holding it Live (guaranteed-dead pid, // same sentinel the registry sweep test uses). - auto* ring0 = kickmsg::sub_ring_at(region.base(), h, 0); + auto* ring0 = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); ring0->owner_starttime.store(0, std::memory_order_relaxed); ring0->owner_pid.store(0x7fffffff, std::memory_order_release); ring0->state_flight.store(kickmsg::ring::make_packed(kickmsg::ring::Live), std::memory_order_release); // Ring 1: a LIVE owner (this process) must never be reclaimed. - auto* ring1 = kickmsg::sub_ring_at(region.base(), h, 1); + auto* ring1 = kickmsg::sub_ring_at(region.base(), region.geometry(), 1); ring1->owner_starttime.store( kickmsg::process_starttime(kickmsg::current_pid()), std::memory_order_relaxed); ring1->owner_pid.store(kickmsg::current_pid(), std::memory_order_release); @@ -659,12 +638,9 @@ TEST_F(RegionTest, ReclaimDeadRingsPreservesInFlight) { auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); - auto* h = region.header(); - // Dead owner holding a Live ring with a publisher still admitted: the - // reclaim must flip state to Free but keep in_flight (so the mid-commit - // publisher's fetch_sub can't underflow into the state bits). - auto* ring = kickmsg::sub_ring_at(region.base(), h, 0); + // Keep in_flight so a late publisher decrement cannot underflow state. + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); ring->owner_pid.store(0x7fffffff, std::memory_order_release); ring->state_flight.store(kickmsg::ring::make_packed(kickmsg::ring::Live, 1), std::memory_order_release); @@ -873,10 +849,7 @@ TEST_F(RegionTest, SchemaConcurrentClaimsOneWins) TEST_F(RegionTest, SchemaReaderDuringClaimingReturnsNullopt) { - // Invariant: schema() must return nullopt unless state == Set. - // We force the Claiming state directly (the real transition is too - // brief to observe deterministically from another thread) and confirm - // readers don't see torn payload bytes. + // Stage Claiming directly so the reader cannot race past this state. auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg); @@ -901,10 +874,7 @@ TEST_F(RegionTest, SchemaReaderDuringClaimingReturnsNullopt) TEST_F(RegionTest, SchemaResetRecoversWedgedClaimingState) { - // Crash scenario: a claimant CAS'd Unset -> Claiming and died before - // the release-store of Set. Every try_claim_schema() caller will - // observe Claiming and return false after bounded yields. - // reset_schema_claim() is the operator-driven recovery. + // Stage a claimant that stopped before publishing Set. auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg); @@ -956,9 +926,6 @@ TEST_F(RegionTest, SchemaResetIsNoOpWhenNotClaiming) TEST_F(RegionTest, SchemaCreateOrOpenIgnoresOpenerSchemaWhenCreatorHadNone) { - // Separation of concerns, open-branch path: if the creator leaves - // schema unset and a later opener passes cfg.schema, that schema is - // silently ignored (use try_claim_schema to publish it instead). auto cfg = default_cfg(); // Note: cfg.schema intentionally left empty. auto existing = kickmsg::SharedRegion::create( @@ -977,10 +944,7 @@ TEST_F(RegionTest, SchemaCreateOrOpenIgnoresOpenerSchemaWhenCreatorHadNone) TEST_F(RegionTest, SchemaCrossHandleObservesClaim) { - // Mirror the real cross-process flow: one SharedRegion handle claims, - // a second SharedRegion handle opened against the same SHM observes - // the claim. This exercises the acquire-load in schema() across - // independent mapping handles, not just within a single object. + // Use independent mappings to check schema publication. auto cfg = default_cfg(); auto r1 = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg); @@ -1006,9 +970,7 @@ TEST_F(RegionTest, SchemaCrossHandleObservesClaim) TEST_F(RegionTest, SchemaResetViaSecondHandleAfterCrash) { - // Mirror the cross-process crash-recovery flow: one handle wedges - // (simulated claimant crashed mid-claim), a second handle opened - // against the same SHM calls reset_schema_claim(). + // Recover an abandoned claim through a second mapping. auto cfg = default_cfg(); auto r1 = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg); @@ -1033,9 +995,6 @@ TEST_F(RegionTest, SchemaResetViaSecondHandleAfterCrash) TEST_F(RegionTest, DiagnoseReportsSchemaStuck) { - // Wedged Claiming must surface via HealthReport alongside the other - // crash-residue indicators so supervisors can detect it on a - // routine health-check loop. auto cfg = default_cfg(); auto region = kickmsg::SharedRegion::create( SHM_NAME, kickmsg::channel::PubSub, cfg); @@ -1058,9 +1017,6 @@ TEST_F(RegionTest, DiagnoseReportsSchemaStuck) TEST_F(RegionTest, SchemaDoesNotAffectConfigHash) { - // Separation of concerns: schema presence is orthogonal to channel - // geometry, so create_or_open() from a different Config::schema must - // NOT trip the config mismatch check. auto cfg = default_cfg(); cfg.schema = make_schema("creator/Type", 1, 0xAA, 0xBB); @@ -1081,9 +1037,7 @@ TEST_F(RegionTest, SchemaDoesNotAffectConfigHash) EXPECT_STREQ(got->name, "creator/Type"); } -// ----------------------------------------------------------------------------- // stats() -- cross-process counter snapshot -// ----------------------------------------------------------------------------- TEST_F(RegionTest, StatsOnFreshRegionReportsZeros) { @@ -1190,17 +1144,15 @@ TEST_F(RegionTest, StatsPoolFreeTracksAllocations) kickmsg::Publisher pub(region); // Hold a slot mid-publish (allocate without publish). - auto a = pub.allocate(); - ASSERT_NE(a.data, nullptr); + auto slot = pub.allocate(); + ASSERT_TRUE(slot.valid()); auto s = region.stats(); // One slot is popped from the free stack and not yet returned. EXPECT_EQ(s.pool_free, cfg.pool_size - 1); } -// ----------------------------------------------------------------------------- // attach_create / attach_open -- caller-provided memory -// ----------------------------------------------------------------------------- class InjectedRegionTest : public ::testing::Test { @@ -1348,9 +1300,6 @@ TEST_F(InjectedRegionTest, UnlinkOnInjectedRegionIsNoOp) TEST_F(InjectedRegionTest, AttachOpenRejectsBufferSmallerThanHeader) { - // A buffer smaller than sizeof(Header) must be rejected BEFORE any - // dereference of magic/version/total_size -- otherwise the load is - // an out-of-bounds read on hostile or accidentally-small input. alignas(kickmsg::CACHE_LINE) std::byte tiny[kickmsg::CACHE_LINE]{}; static_assert(sizeof(tiny) < sizeof(kickmsg::Header)); @@ -1371,9 +1320,6 @@ TEST_F(InjectedRegionTest, MoveLeavesSourceWithNullBase) auto dst = std::move(src); EXPECT_EQ(dst.base(), live_base); - // After move, the source must NOT still alias the destination's - // live memory -- otherwise base()/header() on the moved-from object - // returns a dangling-looking-live pointer instead of nullptr. EXPECT_EQ(src.base(), nullptr); } @@ -1393,10 +1339,7 @@ TEST_F(InjectedRegionTest, MoveAssignLeavesSourceWithNullBase) EXPECT_EQ(src.base(), nullptr); } -// Threat-model tests for validate_header_geometry: a kickmsg-stamped -// buffer always passes; deliberately corrupting any geometry field must -// fail attach_open with a runtime_error, never let downstream code -// compute wild pointers. +// Reject invalid shared-memory geometry before using it for pointer arithmetic. class CorruptedHeaderTest : public InjectedRegionTest { public: @@ -1410,13 +1353,13 @@ class CorruptedHeaderTest : public InjectedRegionTest return buf; } - kickmsg::Header* hdr(Buffer& b) { return static_cast(b.get()); } + kickmsg::Header* header(Buffer& b) { return static_cast(b.get()); } }; TEST_F(CorruptedHeaderTest, RejectsZeroMaxSubs) { auto buf = make_stamped(default_cfg()); - hdr(buf)->max_subs = 0; + header(buf)->max_subs = 0; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1424,7 +1367,7 @@ TEST_F(CorruptedHeaderTest, RejectsZeroMaxSubs) TEST_F(CorruptedHeaderTest, RejectsNonPowerOfTwoRingCapacity) { auto buf = make_stamped(default_cfg()); - hdr(buf)->sub_ring_capacity = 7; + header(buf)->sub_ring_capacity = 7; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1432,7 +1375,7 @@ TEST_F(CorruptedHeaderTest, RejectsNonPowerOfTwoRingCapacity) TEST_F(CorruptedHeaderTest, RejectsInconsistentRingMask) { auto buf = make_stamped(default_cfg()); - hdr(buf)->sub_ring_mask = 3; // capacity is 8, mask should be 7 + header(buf)->sub_ring_mask = 3; // capacity is 8, mask should be 7 EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1440,7 +1383,7 @@ TEST_F(CorruptedHeaderTest, RejectsInconsistentRingMask) TEST_F(CorruptedHeaderTest, RejectsCreatorNameLenOverflow) { auto buf = make_stamped(default_cfg()); - hdr(buf)->creator_name_len = UINT16_MAX; + header(buf)->creator_name_len = UINT16_MAX; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1448,7 +1391,7 @@ TEST_F(CorruptedHeaderTest, RejectsCreatorNameLenOverflow) TEST_F(CorruptedHeaderTest, RejectsPoolOffsetPastTotalSize) { auto buf = make_stamped(default_cfg()); - hdr(buf)->pool_offset = UINT64_MAX; + header(buf)->pool_offset = UINT64_MAX; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1456,7 +1399,7 @@ TEST_F(CorruptedHeaderTest, RejectsPoolOffsetPastTotalSize) TEST_F(CorruptedHeaderTest, RejectsTotalSizeSmallerThanHeader) { auto buf = make_stamped(default_cfg()); - hdr(buf)->total_size = sizeof(kickmsg::Header) - 1; + header(buf)->total_size = sizeof(kickmsg::Header) - 1; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1465,7 +1408,7 @@ TEST_F(CorruptedHeaderTest, RejectsRingsOverflowingPoolOffset) { auto buf = make_stamped(default_cfg()); // max_subs * sub_ring_stride must fit in [sub_rings_offset, pool_offset). - hdr(buf)->max_subs = UINT64_MAX; + header(buf)->max_subs = UINT64_MAX; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1473,7 +1416,7 @@ TEST_F(CorruptedHeaderTest, RejectsRingsOverflowingPoolOffset) TEST_F(CorruptedHeaderTest, RejectsPoolOverflowingTotalSize) { auto buf = make_stamped(default_cfg()); - hdr(buf)->pool_size = UINT64_MAX; + header(buf)->pool_size = UINT64_MAX; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1481,7 +1424,7 @@ TEST_F(CorruptedHeaderTest, RejectsPoolOverflowingTotalSize) TEST_F(CorruptedHeaderTest, RejectsTinySlotStride) { auto buf = make_stamped(default_cfg()); - hdr(buf)->slot_stride = 1; // smaller than sizeof(SlotHeader) + header(buf)->slot_stride = 1; // smaller than sizeof(SlotHeader) EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1489,7 +1432,7 @@ TEST_F(CorruptedHeaderTest, RejectsTinySlotStride) TEST_F(CorruptedHeaderTest, RejectsTinyRingStride) { auto buf = make_stamped(default_cfg()); - hdr(buf)->sub_ring_stride = 1; // smaller than a SubRingHeader + entries + header(buf)->sub_ring_stride = 1; // smaller than a SubRingHeader + entries EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1499,8 +1442,8 @@ TEST_F(CorruptedHeaderTest, RejectsRingCapacityOverflowingRegion) auto buf = make_stamped(default_cfg()); // Huge power-of-two capacity with a consistent mask: passes the // power-of-two and mask checks, must trip the pre-multiply overflow guard. - hdr(buf)->sub_ring_capacity = uint64_t{1} << 60; - hdr(buf)->sub_ring_mask = (uint64_t{1} << 60) - 1; + header(buf)->sub_ring_capacity = uint64_t{1} << 60; + header(buf)->sub_ring_mask = (uint64_t{1} << 60) - 1; EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } @@ -1517,18 +1460,12 @@ TEST_F(CorruptedHeaderTest, RejectsCreatorNameLenPastTail) auto buf = make_stamped(default_cfg()); // Within total_size but past the creator-name tail (would let info() // read into the subscriber rings / pool). - hdr(buf)->creator_name_len = - static_cast(hdr(buf)->sub_rings_offset); + header(buf)->creator_name_len = + static_cast(header(buf)->sub_rings_offset); EXPECT_THROW(kickmsg::SharedRegion::attach_open(buf.get(), buf.size), std::runtime_error); } -// --------------------------------------------------------------------------- -// Repair theft-safety: a slow-but-alive publisher whose lock is stolen must -// be detected at its commit CAS (never blind-stored over), a healthy commit -// must survive the grace pass, and a steal must back off if the entry -// changes first. -// --------------------------------------------------------------------------- #include "kickmsg/os/Time.h" @@ -1552,12 +1489,13 @@ TEST_F(RegionTest, RepairStealsProvenStaleLockAndResumedCommitFails) ASSERT_TRUE(sub.try_receive().has_value()); } - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // Stalled holder at pos=4 (idx 0): claimed the position, locked the // entry, then was descheduled past commit_timeout. ring->write_pos.store(5, std::memory_order_release); + uint64_t const claim_before = entries[0].meta.load(std::memory_order_acquire); uint64_t expected = 1; ASSERT_TRUE(entries[0].sequence.compare_exchange_strong( expected, kickmsg::seq_lock(4), @@ -1567,21 +1505,22 @@ TEST_F(RegionTest, RepairStealsProvenStaleLockAndResumedCommitFails) EXPECT_EQ(region.repair_locked_entries(), 1u); EXPECT_EQ(entries[0].sequence.load(std::memory_order_acquire), kickmsg::seq_skip(4)); - EXPECT_EQ(entries[0].slot_idx.load(std::memory_order_acquire), - kickmsg::INVALID_SLOT); - EXPECT_EQ(entries[0].payload_len.load(std::memory_order_acquire), 0u); - - // The holder resumes and commits: the CAS from its own lock value must - // fail and leave the repaired entry untouched. The old blind-store - // protocol re-stamped the same sequence here -- the torn-entry / - // sequence-rewind corruption this protocol exists to prevent. + // Keep the predecessor's claim for the next publisher to release. + EXPECT_EQ(entries[0].meta.load(std::memory_order_acquire), claim_before); + + // The resumed holder's commit CAS must fail. uint64_t lock_val = kickmsg::seq_lock(4); EXPECT_FALSE(entries[0].sequence.compare_exchange_strong( lock_val, 5u, std::memory_order_release, std::memory_order_relaxed)); EXPECT_EQ(entries[0].sequence.load(std::memory_order_acquire), kickmsg::seq_skip(4)); - EXPECT_EQ(entries[0].slot_idx.load(std::memory_order_acquire), - kickmsg::INVALID_SLOT); + EXPECT_EQ(entries[0].meta.load(std::memory_order_acquire), claim_before); + + // Once the next wrap has taken the entry over, the stale holder's + // take-over is refused too: its position is no longer the newest here. + entries[0].meta.store(kickmsg::meta_pack(8, 3), std::memory_order_release); + EXPECT_FALSE(kickmsg::meta_precedes( + entries[0].meta.load(std::memory_order_acquire), 4)); } TEST_F(RegionTest, RepairGraceSparesInFlightCommit) @@ -1606,7 +1545,7 @@ TEST_F(RegionTest, RepairGraceSparesInFlightCommit) ASSERT_TRUE(sub.try_receive().has_value()); } - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // Healthy holder mid-commit at pos=4. @@ -1624,17 +1563,14 @@ TEST_F(RegionTest, RepairGraceSparesInFlightCommit) repaired = region.repair_locked_entries(); }); - // Commit while the repairer sits in its grace sleep: the re-check sees - // the value changed and must NOT steal. (If the repairer is so delayed - // that its scan runs after the commit, it finds no candidate and the - // assertions below still hold.) + // Commit during the grace period; repair must leave the changed entry alone. while (not started.load(std::memory_order_acquire)) { kickmsg::yield(); } kickmsg::sleep(10ms); - entries[0].slot_idx.store(kickmsg::INVALID_SLOT, std::memory_order_relaxed); - entries[0].payload_len.store(0, std::memory_order_relaxed); + entries[0].meta.store(kickmsg::meta_pack(4, kickmsg::INVALID_SLOT), + std::memory_order_relaxed); uint64_t lock_val = kickmsg::seq_lock(4); EXPECT_TRUE(entries[0].sequence.compare_exchange_strong( lock_val, 5u, std::memory_order_release, std::memory_order_relaxed)); @@ -1654,17 +1590,327 @@ TEST_F(RegionTest, StealBacksOffWhenEntryChangesFirst) auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); auto* entries = kickmsg::ring_entries(ring); // A repairer observed the lock, but the holder committed first. - entries[0].slot_idx.store(3, std::memory_order_relaxed); - entries[0].payload_len.store(7, std::memory_order_relaxed); + uint64_t const committed = kickmsg::meta_pack(4, 3); + entries[0].meta.store(committed, std::memory_order_relaxed); entries[0].sequence.store(5, std::memory_order_release); - EXPECT_FALSE(kickmsg::entry_steal_and_clear(entries[0], 4, + EXPECT_FALSE(kickmsg::entry_steal_and_skip(entries[0], 4, kickmsg::seq_lock(4))); EXPECT_EQ(entries[0].sequence.load(std::memory_order_acquire), 5u); - EXPECT_EQ(entries[0].slot_idx.load(std::memory_order_acquire), 3u); - EXPECT_EQ(entries[0].payload_len.load(std::memory_order_acquire), 7u); + EXPECT_EQ(entries[0].meta.load(std::memory_order_acquire), committed); +} + +// Handles must use their validated geometry after shared header changes. +TEST_F(RegionTest, HandlesUseAValidatedGeometrySnapshot) +{ + kickmsg::channel::Config cfg; + cfg.max_subscribers = 1; + cfg.sub_ring_capacity = 4; + cfg.pool_size = 8; + cfg.max_payload_size = 8; + + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + kickmsg::Subscriber sub(region); + kickmsg::Publisher pub(region); + + uint32_t first = 7; + ASSERT_GE(pub.send(&first, sizeof(first)), 0); + + // Change every shared field used for pointer arithmetic. + auto* header = region.header(); + header->pool_offset = 1; + header->slot_stride = 1ULL << 40; + header->sub_rings_offset = 1; + header->sub_ring_stride = 1ULL << 40; + header->pool_size = UINT32_MAX; + header->slot_data_size = UINT64_MAX; + header->sub_ring_capacity = 1ULL << 40; + header->sub_ring_mask = UINT64_MAX; + header->max_subs = UINT32_MAX; + + // Both handles keep working off their own copies. + auto sample = sub.try_receive(); + ASSERT_TRUE(sample.has_value()); + uint32_t got = 0; + std::memcpy(&got, sample->data(), sizeof(got)); + EXPECT_EQ(got, 7u); + + uint32_t second = 9; + ASSERT_GE(pub.send(&second, sizeof(second)), 0); + auto view = sub.try_receive_view(); + ASSERT_TRUE(view.has_value()); + ASSERT_EQ(view->len(), sizeof(second)); + std::memcpy(&got, view->data(), sizeof(got)); + EXPECT_EQ(got, 9u); +} + +TEST_F(RegionTest, VersionMismatchIsFatalAndImmediate) +{ + auto cfg = default_cfg(); + + // Held open: a Windows mapping is destroyed with its last handle, and an old + // region only exists because an old process still maps it. + auto old_build = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + old_build.header()->version = kickmsg::VERSION - 1; + + EXPECT_THROW(kickmsg::SharedRegion::open(SHM_NAME), kickmsg::VersionMismatch); + + auto start = steady_clock::now(); + EXPECT_THROW(kickmsg::SharedRegion::create_or_open(SHM_NAME, kickmsg::channel::PubSub, cfg), + kickmsg::VersionMismatch); + EXPECT_LT(steady_clock::now() - start, 500ms) << "create_or_open waited on a region that will never match"; +} + +TEST_F(RegionTest, RegionWalksUseTheValidatedGeometrySnapshot) +{ + kickmsg::channel::Config cfg; + cfg.max_subscribers = 2; + cfg.sub_ring_capacity = 4; + cfg.pool_size = 8; + cfg.max_payload_size = 8; + + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg, "my_node"); + kickmsg::Subscriber sub(region); + kickmsg::Publisher pub(region); + + uint32_t value = 7; + ASSERT_GE(pub.send(&value, sizeof(value)), 0); + + // A peer rewrites every geometry field after validation. + auto* header = region.header(); + header->pool_offset = 1; + header->slot_stride = 1ULL << 40; + header->sub_rings_offset = 1; + header->sub_ring_stride = 1ULL << 40; + header->pool_size = UINT32_MAX; + header->slot_data_size = UINT64_MAX; + header->sub_ring_capacity = 1ULL << 40; + header->sub_ring_mask = UINT64_MAX; + header->max_subs = UINT32_MAX; + header->creator_name_len = UINT16_MAX; + + auto report = region.diagnose(); + EXPECT_EQ(report.live_rings, 1u); + EXPECT_EQ(report.locked_entries, 0u); + + auto stats = region.stats(); + EXPECT_EQ(stats.rings.size(), cfg.max_subscribers); + EXPECT_EQ(stats.pool_size, cfg.pool_size); + EXPECT_EQ(stats.total_writes, 1u); + + auto info = region.info(); + EXPECT_EQ(info.max_subs, cfg.max_subscribers); + EXPECT_EQ(info.pool_size, cfg.pool_size); + EXPECT_LE(info.creator_name.size(), region.geometry().sub_rings_offset - sizeof(kickmsg::Header)); + EXPECT_EQ(info.creator_name.rfind("my_node", 0), 0u); + + EXPECT_EQ(region.repair_locked_entries(), 0u); + EXPECT_EQ(region.reset_retired_rings(), 0u); + EXPECT_EQ(region.reclaim_dead_rings(), 0u); + EXPECT_EQ(region.reclaim_orphaned_slots(), 0u); + + auto sample = sub.try_receive(); + ASSERT_TRUE(sample.has_value()); + std::memcpy(&value, sample->data(), sizeof(value)); + EXPECT_EQ(value, 7u); +} + +// Each claim owns one reference; a leaked extra reference must not pin the slot. +TEST_F(RegionTest, ReclaimOrphanedSlotsResetsOvercountedRefcounts) +{ + kickmsg::channel::Config cfg; + cfg.max_subscribers = 1; + cfg.sub_ring_capacity = 4; + cfg.pool_size = 8; + cfg.max_payload_size = 8; + + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + { + kickmsg::Subscriber sub(region); + kickmsg::Publisher pub(region); + + uint32_t value = 1; + ASSERT_GE(pub.send(&value, sizeof(value)), 0); + + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), sub.ring_index()); + auto* entries = kickmsg::ring_entries(ring); + uint32_t biased = kickmsg::meta_slot_biased(entries[0].meta.load(std::memory_order_acquire)); + ASSERT_NE(biased, 0u); + auto* slot = kickmsg::slot_at(region.base(), region.geometry(), biased - 1); + ASSERT_EQ(slot->refcount.load(std::memory_order_acquire), 1u); + + slot->refcount.store(5, std::memory_order_release); + EXPECT_EQ(region.reclaim_orphaned_slots(), 0u) << "a claimed slot was freed"; + EXPECT_EQ(slot->refcount.load(std::memory_order_acquire), 1u); + } + EXPECT_EQ(region.stats().pool_free, cfg.pool_size) << "the leaked reference still pins the slot"; +} + +// A committed entry whose claim names another position must not be delivered. +TEST_F(RegionTest, ReceiveRejectsAClaimTaggedForAnotherPosition) +{ + kickmsg::channel::Config cfg; + cfg.max_subscribers = 1; + cfg.sub_ring_capacity = 4; + cfg.pool_size = 8; + cfg.max_payload_size = 8; + + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + kickmsg::Subscriber sub(region); + kickmsg::Publisher pub(region); + + uint32_t value = 1; + ASSERT_GE(pub.send(&value, sizeof(value)), 0); + + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), sub.ring_index()); + auto& e = kickmsg::ring_entries(ring)[0]; + uint64_t claimed = e.meta.load(std::memory_order_acquire); + uint32_t slot = kickmsg::meta_slot_biased(claimed) - 1; + e.meta.store(kickmsg::meta_pack(4, slot), std::memory_order_release); + + EXPECT_FALSE(sub.try_receive_view().has_value()); + EXPECT_EQ(sub.lost(), 1u); + + e.meta.store(claimed, std::memory_order_release); +} + +TEST_F(RegionTest, CorruptFreeStackHeadFailsAllocationInsteadOfFaulting) +{ + kickmsg::channel::Config cfg; + cfg.max_subscribers = 1; + cfg.sub_ring_capacity = 4; + cfg.pool_size = 8; + cfg.max_payload_size = 8; + + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + kickmsg::Publisher pub(region); + + region.header()->free_top.store(kickmsg::tagged_pack(0, 0xfffffffe), + std::memory_order_release); + + auto slot = pub.allocate(); + EXPECT_FALSE(slot.valid()); + + uint32_t value = 1; + EXPECT_EQ(pub.send(&value, sizeof(value)), -EAGAIN); +} + +// Resume a stalled writer after a newer writer commits to the same entry. +TEST_F(RegionTest, StalledPublisherCannotOverwriteANewerEntry) +{ + kickmsg::channel::Config cfg; + cfg.max_subscribers = 1; + cfg.sub_ring_capacity = 1; // every position lands on the same entry + cfg.pool_size = 8; + cfg.max_payload_size = 8; + cfg.commit_timeout = 1us; + + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + kickmsg::Subscriber sub(region); + + auto* header = region.header(); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); + auto& entry = kickmsg::ring_entries(ring)[0]; + + // Publisher A claims pos 0 and locks the entry, then stalls before + // taking the metadata over. + uint32_t a_slot = kickmsg::treiber_pop(header->free_top, region.base(), region.geometry()); + ASSERT_NE(a_slot, kickmsg::INVALID_SLOT); + auto* a = kickmsg::slot_at(region.base(), region.geometry(), a_slot); + uint32_t stale = 111; + std::memcpy(kickmsg::slot_data(a), &stale, sizeof(stale)); + a->payload_len.store(sizeof(stale), std::memory_order_relaxed); + a->refcount.store(1, std::memory_order_release); + ring->state_flight.fetch_add(kickmsg::ring::IN_FLIGHT_ONE, std::memory_order_acq_rel); + uint64_t pos = ring->write_pos.fetch_add(1, std::memory_order_acq_rel); + uint64_t expected = 0; + ASSERT_TRUE(entry.sequence.compare_exchange_strong( + expected, kickmsg::seq_lock(pos), + std::memory_order_acquire, std::memory_order_relaxed)); + + // A repairer steals the stalled lock, then publisher B commits the next + // generation over the same entry. + EXPECT_EQ(region.repair_locked_entries(), 1u); + kickmsg::Publisher b(region); + uint32_t fresh = 222; + ASSERT_GE(b.send(&fresh, sizeof(fresh)), 0); + + // A resumes and runs the take-over from Publisher::publish(). + uint64_t const my_meta = kickmsg::meta_pack(pos, a_slot); + uint64_t old_meta = entry.meta.load(std::memory_order_acquire); + bool taken = false; + while (kickmsg::meta_precedes(old_meta, pos)) + { + if (entry.meta.compare_exchange_weak(old_meta, my_meta, + std::memory_order_acq_rel, std::memory_order_acquire)) + { + taken = true; + break; + } + } + EXPECT_FALSE(taken); + + // B's message is delivered intact. + auto sample = sub.try_receive(); + ASSERT_TRUE(sample.has_value()); + uint32_t got = 0; + std::memcpy(&got, sample->data(), sizeof(got)); + EXPECT_EQ(got, 222u); + + ring->state_flight.fetch_sub(kickmsg::ring::IN_FLIGHT_ONE, std::memory_order_release); +} + +// Repair must retain the predecessor's claim until its reference is released. +TEST_F(RegionTest, StealKeepsThePredecessorReleasable) +{ + kickmsg::channel::Config cfg; + cfg.max_subscribers = 1; + cfg.sub_ring_capacity = 1; + cfg.pool_size = 8; + cfg.max_payload_size = 8; + cfg.commit_timeout = 1us; + + auto region = kickmsg::SharedRegion::create(SHM_NAME, kickmsg::channel::PubSub, cfg); + + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); + auto& entry = kickmsg::ring_entries(ring)[0]; + + { + kickmsg::Subscriber sub(region); + kickmsg::Publisher pub(region); + uint32_t value = 42; + ASSERT_GE(pub.send(&value, sizeof(value)), 0); + + // The published slot is the entry's claim. + uint32_t claimed = kickmsg::meta_slot_biased( + entry.meta.load(std::memory_order_acquire)); + ASSERT_NE(claimed, 0u); + + // A publisher claims pos 1 and stalls before its take-over. + ring->state_flight.fetch_add(kickmsg::ring::IN_FLIGHT_ONE, + std::memory_order_acq_rel); + uint64_t pos = ring->write_pos.fetch_add(1, std::memory_order_acq_rel); + uint64_t expected = 1; + ASSERT_TRUE(entry.sequence.compare_exchange_strong( + expected, kickmsg::seq_lock(pos), + std::memory_order_acquire, std::memory_order_relaxed)); + + EXPECT_EQ(region.repair_locked_entries(), 1u); + + // The steal left the claim intact, so the predecessor is still + // reachable for whoever takes the entry over next. + EXPECT_EQ(kickmsg::meta_slot_biased(entry.meta.load(std::memory_order_acquire)), + claimed); + + ring->state_flight.fetch_sub(kickmsg::ring::IN_FLIGHT_ONE, + std::memory_order_release); + } + + // Without a crash, all slots must return without orphan recovery. + EXPECT_EQ(region.stats().pool_free, cfg.pool_size); + EXPECT_EQ(region.reclaim_orphaned_slots(), 0u); } diff --git a/tests/unit/registry-t.cc b/tests/unit/registry-t.cc index f3258ed..6772b95 100644 --- a/tests/unit/registry-t.cc +++ b/tests/unit/registry-t.cc @@ -1,4 +1,7 @@ +#include +#include #include +#include #include #include @@ -6,15 +9,14 @@ #include "kickmsg/Naming.h" #include "kickmsg/Node.h" #include "kickmsg/Registry.h" +#include "kickmsg/os/Process.h" class RegistryTest : public ::testing::Test { protected: static constexpr char const* KMSG_NAMESPACE = "kickmsg_regtest"; - // Mirror Node::make_topic_name / make_broadcast_name so test - // expectations match what Node actually composes on every platform - // (readable on Linux, hashed on macOS to fit PSHMNAMLEN). + // Use the same platform-specific naming rules as Node. static std::string topic_shm(char const* topic) { return kickmsg::compose_shm_name( @@ -29,8 +31,6 @@ class RegistryTest : public ::testing::Test "broadcast_" + kickmsg::sanitize_shm_component(channel, "channel")); } - // Mirror Registry::make_shm_name (private) so tests stay aligned with - // whatever the Registry actually composes per platform. static std::string registry_shm() { return kickmsg::compose_shm_name( @@ -151,9 +151,6 @@ TEST_F(RegistryTest, CapacityExhaustionReturnsInvalidSlot) TEST_F(RegistryTest, VersionMismatchOnSmallerExistingRegionThrows) { - // Validate the open path still works when the region already exists: - // open_or_create should happily attach to an existing compatible - // region of a different capacity (capacity is only used on create). auto created = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 8); EXPECT_EQ(created.capacity(), 8u); @@ -164,9 +161,7 @@ TEST_F(RegistryTest, VersionMismatchOnSmallerExistingRegionThrows) TEST_F(RegistryTest, OpenRejectsCorruptCapacity) { - // Establish an 8-slot registry, then corrupt capacity in the raw segment - // to far exceed the mapping. A fresh open must reject it instead of - // letting snapshot()/sweep_stale() walk entries past the mapped pages. + // Change the stored capacity so the entry array exceeds the mapping. auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 8); kickmsg::SharedMemory raw; @@ -202,10 +197,7 @@ TEST_F(RegistryTest, SweepStaleRemovesDeadPidEntries) TEST_F(RegistryTest, SweepStaleReclaimsWedgedClaimingSlot) { - // A registrant that dies between the Free→Claiming CAS and the - // release-store of Active leaves the slot stuck. sweep_stale must - // reclaim it — otherwise the registry leaks capacity on every such - // crash. Simulate by reaching into the raw SHM and patching a slot. + // Stage an abandoned claim before identity publication. auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE); // Fill slot 0 with a legitimate entry. @@ -235,8 +227,7 @@ TEST_F(RegistryTest, SweepStaleReclaimsWedgedClaimingSlot) TEST_F(RegistryTest, SweepStaleSkipsClaimingSlotsWithoutPid) { - // A Claiming slot with pid==0 may be a registrant between CAS and its - // first field write. Reclaiming would race with its stores. Must skip. + // A claim with pid == 0 may still have a live writer. auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE); auto shm_name = registry_shm(); @@ -258,9 +249,633 @@ TEST_F(RegistryTest, SweepStaleSkipsClaimingSlotsWithoutPid) std::memory_order_release); } -// ----------------------------------------------------------------------------- -// Node integration — Node advertise/subscribe/etc should populate the registry -// ----------------------------------------------------------------------------- +TEST_F(RegistryTest, SnapshotRejectsARowCaughtMidRetirement) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE); + + uint32_t slot = reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT); + ASSERT_EQ(reg.snapshot().size(), 1u); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[slot]; + + // A settled row carries an even generation. + ASSERT_EQ(e.generation.load(std::memory_order_acquire) & 1u, 0u); + + // Pause retirement after clearing pid but before settling generation. + e.generation.fetch_add(1, std::memory_order_relaxed); + e.pid.store(0, std::memory_order_relaxed); + e.pid_starttime.store(0, std::memory_order_relaxed); + + EXPECT_TRUE(reg.snapshot().empty()); + + e.pid.store(kickmsg::current_pid(), std::memory_order_relaxed); + e.generation.fetch_add(1, std::memory_order_relaxed); + auto settled = reg.snapshot(); + ASSERT_EQ(settled.size(), 1u); + EXPECT_NE(settled[0].pid, 0u); + + reg.deregister(slot); +} + +// Keep one row stable while another is repeatedly registered and retired. +TEST_F(RegistryTest, ConcurrentChurnNeverYieldsAnIncoherentSnapshot) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE); + + uint32_t stable = reg.register_participant( + "/stable", "/stable-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Publisher, "stable-node"); + ASSERT_NE(stable, kickmsg::INVALID_SLOT); + + std::atomic stop{false}; + std::atomic stable_seen{0}; + std::atomic zero_pid{0}; + std::atomic mixed{0}; + std::atomic free_unsettled{0}; + + std::thread churn([&] + { + while (not stop.load(std::memory_order_relaxed)) + { + uint32_t slot = reg.register_participant( + "/churn", "/churn-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Subscriber, + "churn-node"); + if (slot != kickmsg::INVALID_SLOT) + { + reg.deregister(slot); + } + } + }); + + // Check that a stable even Active row has a PID. + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + uint32_t const cap = reinterpret_cast( + raw.address())->capacity; + + auto const deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (std::chrono::steady_clock::now() < deadline) + { + for (int spin = 0; spin < 2000; ++spin) + { + for (uint32_t i = 0; i < cap; ++i) + { + uint32_t s1 = entries[i].state.load(std::memory_order_acquire); + if (s1 != kickmsg::registry::Active) + { + continue; + } + uint32_t g1 = entries[i].generation.load(std::memory_order_acquire); + if ((g1 & 1u) != 0) + { + continue; + } + uint64_t pid = entries[i].pid.load(std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + uint32_t g2 = entries[i].generation.load(std::memory_order_acquire); + uint32_t s2 = entries[i].state.load(std::memory_order_acquire); + if (s2 != kickmsg::registry::Active or g1 != g2) + { + continue; + } + if (pid == 0) + { + zero_pid.fetch_add(1, std::memory_order_relaxed); + } + } + + // Free permits a new writer, so its generation must already be even. + for (uint32_t i = 0; i < cap; ++i) + { + uint32_t g1 = entries[i].generation.load(std::memory_order_acquire); + uint32_t st = entries[i].state.load(std::memory_order_acquire); + if (st != kickmsg::registry::Free) + { + continue; + } + std::atomic_thread_fence(std::memory_order_acquire); + uint32_t g2 = entries[i].generation.load(std::memory_order_acquire); + if (g1 != g2) + { + continue; // the row moved under us; nothing proven + } + if ((g1 & 1u) != 0) + { + free_unsettled.fetch_add(1, std::memory_order_relaxed); + } + } + } + for (auto const& p : reg.snapshot()) + { + if (p.pid == 0) + { + zero_pid.fetch_add(1, std::memory_order_relaxed); + continue; + } + bool const is_stable = p.shm_name == "/stable" + and p.topic_name == "/stable-topic" + and p.node_name == "stable-node" + and p.role == kickmsg::registry::Publisher; + bool const is_churn = p.shm_name == "/churn" + and p.topic_name == "/churn-topic" + and p.node_name == "churn-node" + and p.role == kickmsg::registry::Subscriber; + if (is_stable) + { + stable_seen.fetch_add(1, std::memory_order_relaxed); + } + else if (not is_churn) + { + // Fields came from different registrations. + mixed.fetch_add(1, std::memory_order_relaxed); + } + } + } + stop.store(true, std::memory_order_relaxed); + churn.join(); + + EXPECT_EQ(zero_pid.load(), 0u) << "snapshot returned a retired identity"; + EXPECT_EQ(free_unsettled.load(), 0u) + << "a claimable row was published with an unsettled generation"; + EXPECT_EQ(mixed.load(), 0u) << "snapshot spliced two tenancies"; + EXPECT_GT(stable_seen.load(), 0u) << "oracle never saw the stable row"; + + reg.deregister(stable); +} + +TEST_F(RegistryTest, RowHandoffBetweenOwnersKeepsEveryTenancyVisible) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + + for (int round = 0; round < 64; ++round) + { + uint32_t slot = reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT) << "round " << round; + + uint32_t gen = entries[0].generation.load(std::memory_order_acquire); + EXPECT_EQ(gen & 1u, 0u) << "settled row has an odd generation, round " << round; + + auto rows = reg.snapshot(); + ASSERT_EQ(rows.size(), 1u) << "registered owner invisible, round " << round; + EXPECT_NE(rows[0].pid, 0u); + + reg.deregister(slot); + EXPECT_TRUE(reg.snapshot().empty()) << "round " << round; + EXPECT_EQ(entries[0].generation.load(std::memory_order_acquire) & 1u, 0u) + << "row left odd after retirement, round " << round; + } +} + +// Use two rows so registration can proceed without an automatic sweep. +TEST_F(RegistryTest, ARetiringRowIsNeitherVisibleNorClaimable) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 2); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + + uint32_t slot = reg.register_participant( + "/retiring", "/retiring-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Publisher, "retiring-node"); + ASSERT_EQ(slot, 0u); + + // Pause retirement after clearing identity, before publishing Free. + entries[0].state.store(kickmsg::registry::Reclaiming, std::memory_order_release); + entries[0].pid.store(0, std::memory_order_relaxed); + entries[0].pid_starttime.store(0, std::memory_order_relaxed); + + // Invisible: no Active row with a cleared identity. + EXPECT_TRUE(reg.snapshot().empty()); + + // Unclaimable: the next registrant must take the OTHER row, not this one. + uint32_t next = reg.register_participant( + "/next", "/next-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Subscriber, "next-node"); + ASSERT_NE(next, kickmsg::INVALID_SLOT); + EXPECT_NE(next, slot) << "a retiring row was handed to a second owner"; + + auto rows = reg.snapshot(); + ASSERT_EQ(rows.size(), 1u); + EXPECT_EQ(rows[0].node_name, "next-node"); + EXPECT_NE(rows[0].pid, 0u); + EXPECT_EQ(entries[next].generation.load(std::memory_order_acquire) & 1u, 0u); + + reg.deregister(next); +} + +// Odd abandoned claims remain unavailable because a sweep cannot +// distinguish them from an active writer or reclaimer. +TEST_F(RegistryTest, SweepRefusesAnAbandonedOddClaimRatherThanRaceItsHolder) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[0]; + + // Died after opening its seqlock and publishing an identity that is gone. + e.state.store(kickmsg::registry::Claiming, std::memory_order_release); + e.generation.store(1, std::memory_order_relaxed); + e.pid_starttime.store(1, std::memory_order_relaxed); + e.pid.store(0x3fffffff, std::memory_order_release); + + EXPECT_EQ(reg.sweep_stale(), 0u) + << "recovery raced a row whose writer it cannot identify"; + EXPECT_EQ(e.state.load(std::memory_order_acquire), kickmsg::registry::Claiming); + EXPECT_EQ(e.generation.load(std::memory_order_acquire), 1u); + + EXPECT_EQ(reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Publisher, "node"), + kickmsg::INVALID_SLOT); + + e.state.store(kickmsg::registry::Free, std::memory_order_release); + e.generation.store(0, std::memory_order_relaxed); +} + +TEST_F(RegistryTest, SweepLeavesARowHeldByAnotherOwnerAlone) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[0]; + + uint32_t slot = reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT); + + // Mid-retirement: Reclaiming held, identity cleared, Free not yet published. + e.state.store(kickmsg::registry::Reclaiming, std::memory_order_release); + e.pid.store(0, std::memory_order_relaxed); + uint32_t const gen_before = e.generation.load(std::memory_order_acquire); + + EXPECT_TRUE(reg.snapshot().empty()) << "a retiring row must not be visible"; + + EXPECT_EQ(reg.sweep_stale(), 0u); + EXPECT_EQ(e.state.load(std::memory_order_acquire), kickmsg::registry::Reclaiming); + EXPECT_EQ(e.generation.load(std::memory_order_acquire), gen_before); + + EXPECT_EQ(reg.register_participant( + "/other", "/other-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Subscriber, + "other-node"), + kickmsg::INVALID_SLOT) + << "registration recycled a row another owner is still writing"; + EXPECT_EQ(e.state.load(std::memory_order_acquire), kickmsg::registry::Reclaiming); + + // Let the owner finish; the row comes back on its own. + e.pid_starttime.store(0, std::memory_order_relaxed); + reg.deregister(slot); +} + +TEST_F(RegistryTest, FullRegistryRegistrationCannotStealAPausedRetirement) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[0]; + + uint32_t slot = reg.register_participant( + "/old", "/old-topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "old-node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT); + + // Pause A after clearing pid, before clearing starttime and settling generation. + e.state.store(kickmsg::registry::Reclaiming, std::memory_order_release); + e.generation.fetch_add(1, std::memory_order_relaxed); // bracket opened + e.pid.store(0, std::memory_order_relaxed); + + // B registers. The registry is full, so this sweeps. + uint32_t b = reg.register_participant( + "/new", "/new-topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Subscriber, "new-node"); + EXPECT_EQ(b, kickmsg::INVALID_SLOT) + << "a paused retirement was recycled out from under its owner"; + + // Resume A's remaining retirement writes. + e.pid_starttime.store(0, std::memory_order_relaxed); + e.generation.store((e.generation.load(std::memory_order_relaxed) + 2) & ~1u, + std::memory_order_relaxed); + uint32_t retiring = kickmsg::registry::Reclaiming; + EXPECT_TRUE(e.state.compare_exchange_strong(retiring, kickmsg::registry::Free, + std::memory_order_release, + std::memory_order_relaxed)) + << "the owner lost its own row"; + + // Only now is the row available, with a coherent identity. + uint32_t next = reg.register_participant( + "/new", "/new-topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Subscriber, "new-node"); + ASSERT_NE(next, kickmsg::INVALID_SLOT); + auto rows = reg.snapshot(); + ASSERT_EQ(rows.size(), 1u); + EXPECT_EQ(rows[0].node_name, "new-node"); + EXPECT_NE(rows[0].pid, 0u); + EXPECT_NE(rows[0].pid_starttime, 0u) + << "a resuming owner zeroed the replacement's start time"; + reg.deregister(next); +} + +TEST_F(RegistryTest, ASecondSweepCannotTakeARowAlreadyHeldBySweeping) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[0]; + + // Pause sweeper A after claiming Reclaiming, before clearing the dead identity. + e.pid.store(0x3fffffff, std::memory_order_relaxed); + e.pid_starttime.store(1, std::memory_order_relaxed); + e.state.store(kickmsg::registry::Reclaiming, std::memory_order_release); + uint32_t const gen_before = e.generation.load(std::memory_order_acquire); + + // Sweeper B runs, explicitly and via a full-registry registration. + EXPECT_EQ(reg.sweep_stale(), 0u) + << "a second sweeper took a row the first one holds"; + EXPECT_EQ(reg.register_participant( + "/replacement", "/replacement-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Publisher, + "replacement-node"), + kickmsg::INVALID_SLOT); + EXPECT_EQ(e.state.load(std::memory_order_acquire), kickmsg::registry::Reclaiming); + EXPECT_EQ(e.generation.load(std::memory_order_acquire), gen_before); + EXPECT_EQ(e.pid.load(std::memory_order_acquire), 0x3fffffffu) + << "a second sweeper cleared the identity under the first one"; + + // A finishes; the row returns to service. + e.pid.store(0, std::memory_order_relaxed); + e.pid_starttime.store(0, std::memory_order_relaxed); + e.state.store(kickmsg::registry::Free, std::memory_order_release); +} + +TEST_F(RegistryTest, AcquireTenancyRefusesAVersionItDidNotValidate) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[0]; + + uint32_t first = reg.register_participant( + "/old", "/old-topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "old-node"); + ASSERT_NE(first, kickmsg::INVALID_SLOT); + + // A sweeper validated this tenancy and captured its version, then paused. + uint32_t const validated = e.generation.load(std::memory_order_acquire); + ASSERT_EQ(validated & 1u, 0u); + + // A real handoff: the owner retires, a second live owner takes the row. + reg.deregister(first); + uint32_t second = reg.register_participant( + "/new", "/new-topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Subscriber, "new-node"); + ASSERT_EQ(second, first) << "the test needs the row to be reused"; + ASSERT_EQ(e.state.load(std::memory_order_acquire), kickmsg::registry::Active); + + // Try acquisition with the generation saved before the row was reused. + EXPECT_FALSE(kickmsg::acquire_tenancy(e, validated)) + << "a sweeper acquired a tenancy it never looked at"; + EXPECT_EQ(e.state.load(std::memory_order_acquire), kickmsg::registry::Active); + EXPECT_EQ(e.generation.load(std::memory_order_acquire) & 1u, 0u) + << "a refused acquisition left the row marked in flux"; + + reg.deregister(second); + EXPECT_TRUE(reg.snapshot().empty()) << "deregistration was silently dropped"; + uint32_t reuse = reg.register_participant( + "/reuse", "/reuse-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Publisher, "reuse-node"); + EXPECT_NE(reuse, kickmsg::INVALID_SLOT) << "registry capacity leaked"; + reg.deregister(reuse); +} + +TEST_F(RegistryTest, AcquireTenancySucceedsOnceForTheValidatedVersion) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[0]; + + uint32_t slot = reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT); + + uint32_t const validated = e.generation.load(std::memory_order_acquire); + EXPECT_TRUE(kickmsg::acquire_tenancy(e, validated)); + EXPECT_EQ(e.generation.load(std::memory_order_acquire) & 1u, 1u) + << "an acquired row must read as in flux"; + + // A second caller holding the same validated version loses. + EXPECT_FALSE(kickmsg::acquire_tenancy(e, validated)); + + // Put the row back the way sweep_stale's phase 2 would. + e.pid.store(0, std::memory_order_relaxed); + e.pid_starttime.store(0, std::memory_order_relaxed); + uint32_t g = e.generation.load(std::memory_order_relaxed); + e.generation.store((g + 2) & ~1u, std::memory_order_relaxed); + e.state.store(kickmsg::registry::Free, std::memory_order_release); +} + +TEST_F(RegistryTest, AcquireTenancyRefusesAVersionAnotherAcquirerIsHolding) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + auto& e = entries[0]; + + uint32_t slot = reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT); + + // Pause A before publishing Reclaiming; only its odd generation marks the hold. + uint32_t const validated = e.generation.load(std::memory_order_acquire); + ASSERT_TRUE(kickmsg::acquire_tenancy(e, validated)); + uint32_t const held = e.generation.load(std::memory_order_acquire); + ASSERT_EQ(held & 1u, 1u); + ASSERT_EQ(e.state.load(std::memory_order_acquire), kickmsg::registry::Active); + + // B reads that new version and must be refused. + EXPECT_FALSE(kickmsg::acquire_tenancy(e, held)) + << "a second acquirer took the version the first one is holding"; + EXPECT_EQ(e.generation.load(std::memory_order_acquire), held) + << "a refused acquisition moved the version"; + + // Nor may a real sweep take it, however dead the row's identity looks. + e.pid.store(0x3fffffff, std::memory_order_relaxed); + e.pid_starttime.store(1, std::memory_order_relaxed); + EXPECT_EQ(reg.sweep_stale(), 0u) + << "sweep_stale acquired a row another recoverer holds"; + EXPECT_EQ(e.generation.load(std::memory_order_acquire), held); + + // A settles its own reclamation. + e.state.store(kickmsg::registry::Reclaiming, std::memory_order_release); + e.pid.store(0, std::memory_order_relaxed); + e.pid_starttime.store(0, std::memory_order_relaxed); + e.generation.store((held + 2) & ~1u, std::memory_order_relaxed); + e.state.store(kickmsg::registry::Free, std::memory_order_release); +} + +TEST_F(RegistryTest, SweepsConcurrentWithChurnNeverLeakCapacity) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 2); + + std::atomic stop{false}; + std::atomic lost{0}; + std::atomic cycles{0}; + + std::thread sweeper([&] + { + while (not stop.load(std::memory_order_relaxed)) + { + // All owners are alive; no row should be reclaimed. + if (reg.sweep_stale() != 0) + { + lost.fetch_add(1, std::memory_order_relaxed); + } + } + }); + + auto const deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (std::chrono::steady_clock::now() < deadline) + { + uint32_t slot = reg.register_participant( + "/churn", "/churn-topic", kickmsg::channel::PubSub, + kickmsg::registry::Pubsub, kickmsg::registry::Publisher, "churn-node"); + if (slot == kickmsg::INVALID_SLOT) + { + lost.fetch_add(1, std::memory_order_relaxed); + break; // capacity leaked: a deregistration was dropped earlier + } + reg.deregister(slot); + cycles.fetch_add(1, std::memory_order_relaxed); + } + stop.store(true, std::memory_order_relaxed); + sweeper.join(); + + EXPECT_EQ(lost.load(), 0u) + << "a sweep reclaimed a live row or a deregistration was dropped"; + EXPECT_GT(cycles.load(), 0u); + EXPECT_TRUE(reg.snapshot().empty()); +} + +TEST_F(RegistryTest, DeregisterIsANoOpOnARowItNoLongerHolds) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 1); + + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* entries = reinterpret_cast( + static_cast(raw.address()) + sizeof(kickmsg::RegistryHeader)); + + uint32_t slot = reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT); + + reg.deregister(slot); + ASSERT_EQ(entries[0].state.load(std::memory_order_acquire), + kickmsg::registry::Free); + uint32_t const settled = entries[0].generation.load(std::memory_order_acquire); + + // Second call: the row is Free, so nothing may move. + reg.deregister(slot); + EXPECT_EQ(entries[0].state.load(std::memory_order_acquire), + kickmsg::registry::Free); + EXPECT_EQ(entries[0].generation.load(std::memory_order_acquire), settled); +} + +// Node integration -- Node advertise/subscribe/etc should populate the registry + +// A namespace cannot mix kickmsg builds: an old registry must stop the Node, not silently +// switch discovery off. +TEST_F(RegistryTest, VersionMismatchIsFatalForRegistryAndNode) +{ + // Held open: a Windows mapping is destroyed with its last handle. + auto old_build = kickmsg::Registry::open_or_create(KMSG_NAMESPACE); + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* header = static_cast(raw.address()); + header->version = kickmsg::registry::VERSION - 1; + + EXPECT_THROW(kickmsg::Registry::open_or_create(KMSG_NAMESPACE), kickmsg::VersionMismatch); + EXPECT_THROW(kickmsg::Registry::try_open(KMSG_NAMESPACE), kickmsg::VersionMismatch); + + kickmsg::channel::Config cfg; + cfg.max_subscribers = 2; + cfg.sub_ring_capacity = 4; + cfg.pool_size = 16; + cfg.max_payload_size = 32; + + kickmsg::Node n("mixed_node", KMSG_NAMESPACE); + track(topic_shm("mixed")); + EXPECT_THROW(n.advertise("mixed", cfg), kickmsg::VersionMismatch); + EXPECT_THROW(n.advertise("mixed", cfg), kickmsg::VersionMismatch) + << "the mismatch was latched away instead of staying fatal"; +} + +TEST_F(RegistryTest, WalksUseTheCapacityValidatedAtOpen) +{ + auto reg = kickmsg::Registry::open_or_create(KMSG_NAMESPACE, 8); + uint32_t slot = reg.register_participant( + "/shm", "/topic", kickmsg::channel::PubSub, kickmsg::registry::Pubsub, + kickmsg::registry::Publisher, "node"); + ASSERT_NE(slot, kickmsg::INVALID_SLOT); + + // A peer rewrites the capacity after open. + kickmsg::SharedMemory raw; + raw.open(registry_shm()); + auto* header = static_cast(raw.address()); + header->capacity = UINT32_MAX; + + EXPECT_EQ(reg.capacity(), 8u); + EXPECT_EQ(reg.snapshot().size(), 1u); + EXPECT_EQ(reg.list_topics().size(), 1u); + EXPECT_EQ(reg.sweep_stale(), 0u); + reg.deregister(UINT32_MAX - 1); + reg.deregister(slot); + EXPECT_TRUE(reg.snapshot().empty()); +} TEST_F(RegistryTest, NodeAdvertiseRegistersPublisher) { @@ -310,8 +925,6 @@ TEST_F(RegistryTest, NodeBroadcastRegistersBoth) TEST_F(RegistryTest, NodeAdvertiseThenSubscribeUpgradesToBoth) { - // A Node that both advertises and subscribes to the same topic should - // appear once in the registry with role=Both (not two entries). kickmsg::channel::Config cfg; cfg.max_subscribers = 2; cfg.sub_ring_capacity = 4; @@ -361,9 +974,7 @@ TEST_F(RegistryTest, MultipleNodesEachAppearOnce) EXPECT_TRUE(nodes.count("sub_b")); } -// ----------------------------------------------------------------------------- -// list_topics — topic-centric aggregation -// ----------------------------------------------------------------------------- +// list_topics -- topic-centric aggregation TEST_F(RegistryTest, ListTopicsGroupsByShmName) { diff --git a/tests/unit/subscriber-t.cc b/tests/unit/subscriber-t.cc index c528a48..f6de64f 100644 --- a/tests/unit/subscriber-t.cc +++ b/tests/unit/subscriber-t.cc @@ -148,12 +148,12 @@ TEST_F(SubscriberTest, DrainReleasesSlots) auto count_free = [&]() { uint32_t count = 0; - auto* hdr = region.header(); - uint64_t top = hdr->free_top.load(std::memory_order_acquire); + auto* header = region.header(); + uint64_t top = header->free_top.load(std::memory_order_acquire); uint32_t idx = kickmsg::tagged_idx(top); while (idx != kickmsg::INVALID_SLOT) { - auto* slot = kickmsg::slot_at(region.base(), hdr, idx); + auto* slot = kickmsg::slot_at(region.base(), region.geometry(), idx); idx = slot->next_free; ++count; } @@ -259,10 +259,9 @@ TEST_F(SubscriberTest, DrainDoesNotDoubleDecrementOnChurn) // All slots should have refcount 0 auto* base = region.base(); - auto* h = region.header(); for (uint32_t i = 0; i < cfg.pool_size; ++i) { - auto* slot = kickmsg::slot_at(base, h, i); + auto* slot = kickmsg::slot_at(base, region.geometry(), i); uint32_t rc = slot->refcount; EXPECT_EQ(rc, 0u) << "slot " << i << " has refcount " << rc; } @@ -314,7 +313,7 @@ TEST_F(SubscriberTest, StuckPublisherCausesDrainTimeout) EXPECT_EQ(sub.drain_timeouts(), 0u); // Simulate a stuck publisher: inflate in_flight on ring 0 - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); ring->state_flight.fetch_add(kickmsg::ring::IN_FLIGHT_ONE, std::memory_order_acq_rel); @@ -322,7 +321,7 @@ TEST_F(SubscriberTest, StuckPublisherCausesDrainTimeout) } // The ring should be Free with stale in_flight preserved - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); uint32_t packed = ring->state_flight.load(std::memory_order_acquire); EXPECT_EQ(kickmsg::ring::get_state(packed), kickmsg::ring::Free); EXPECT_GT(kickmsg::ring::get_in_flight(packed), 0u); @@ -358,7 +357,7 @@ TEST_F(SubscriberTest, DrainTimeoutsCounterIncrementsOnTimeout) EXPECT_EQ(sub.drain_timeouts(), 0u); // Inflate in_flight on ring 0 to simulate a crashed publisher. - auto* ring0 = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring0 = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); ring0->state_flight.fetch_add(kickmsg::ring::IN_FLIGHT_ONE, std::memory_order_acq_rel); @@ -404,7 +403,7 @@ TEST_F(SubscriberTest, RejoinAfterDrainTimeout) uint32_t val = 1; ASSERT_GE(pub.send(&val, sizeof(val)), 0); - auto* ring = kickmsg::sub_ring_at(region.base(), region.header(), 0); + auto* ring = kickmsg::sub_ring_at(region.base(), region.geometry(), 0); ring->state_flight.fetch_add(kickmsg::ring::IN_FLIGHT_ONE, std::memory_order_acq_rel); // sub destructs — timeout, drain skipped, stale in_flight preserved @@ -550,10 +549,9 @@ TEST_F(SubscriberTest, ConcurrentChurnRefcountIntegrity) // Verify all refcounts are zero auto* base = region.base(); - auto* h = region.header(); for (uint32_t i = 0; i < cfg.pool_size; ++i) { - auto* slot = kickmsg::slot_at(base, h, i); + auto* slot = kickmsg::slot_at(base, region.geometry(), i); uint32_t rc = slot->refcount; EXPECT_EQ(rc, 0u) << "slot " << i << " has refcount " << rc << " (round completed, all should be 0)"; diff --git a/tests/unit/wait_fd-t.cc b/tests/unit/wait_fd-t.cc index ba9309a..2a403eb 100644 --- a/tests/unit/wait_fd-t.cc +++ b/tests/unit/wait_fd-t.cc @@ -59,7 +59,7 @@ TEST_F(WaitFdTest, UnusedSubscriberLeavesTheRingOnTheFutexPath) auto region = SharedRegion::create(SHM_NAME, channel::PubSub, bare_cfg()); Subscriber sub(region); - auto* ring = sub_ring_at(region.base(), region.header(), sub.ring_index()); + auto* ring = sub_ring_at(region.base(), region.geometry(), sub.ring_index()); std::thread publisher([&]() { @@ -196,7 +196,7 @@ TEST_F(WaitFdTest, ArmMarksTheRingAsCarrierArmed) Subscriber sub(region); ASSERT_TRUE(sub.attach(waker)); - auto* ring = sub_ring_at(region.base(), region.header(), sub.ring_index()); + auto* ring = sub_ring_at(region.base(), region.geometry(), sub.ring_index()); EXPECT_EQ(ring::WaiterNone, ring->has_waiter.load()); ASSERT_EQ(Subscriber::Wait::Armed, sub.arm_wait()); EXPECT_EQ(ring::WaiterCarrier, ring->has_waiter.load()); @@ -210,7 +210,7 @@ TEST_F(WaitFdTest, DisarmIsIdempotent) Subscriber sub(region); ASSERT_TRUE(sub.attach(waker)); - auto* ring = sub_ring_at(region.base(), region.header(), sub.ring_index()); + auto* ring = sub_ring_at(region.base(), region.geometry(), sub.ring_index()); ASSERT_EQ(Subscriber::Wait::Armed, sub.arm_wait()); sub.disarm_wait(); sub.disarm_wait(); @@ -241,7 +241,7 @@ TEST_F(WaitFdTest, ArmReportsReadyWhenASampleIsAlreadyQueued) EXPECT_EQ(Subscriber::Wait::Ready, sub.arm_wait()); // Ready must not have armed the ring: no publisher should be sending. - auto* ring = sub_ring_at(region.base(), region.header(), sub.ring_index()); + auto* ring = sub_ring_at(region.base(), region.geometry(), sub.ring_index()); EXPECT_EQ(ring::WaiterNone, ring->has_waiter.load()); sub.disarm_wait(); @@ -380,7 +380,7 @@ TEST_F(WaitFdTest, ReclaimingARingClearsAStaleWaiterMode) { Subscriber sub(region); ASSERT_TRUE(sub.attach(waker)); - ring = sub_ring_at(region.base(), region.header(), sub.ring_index()); + ring = sub_ring_at(region.base(), region.geometry(), sub.ring_index()); ASSERT_EQ(Subscriber::Wait::Armed, sub.arm_wait()); ASSERT_EQ(ring::WaiterCarrier, ring->has_waiter.load()); // Out of scope still armed, as a killed process would be. @@ -519,7 +519,7 @@ TEST_F(WaitFdTest, ACallerSuppliedBackendIsSignalled) Subscriber sub(region); // Stands in for a subscriber armed on the carrier, which is all the publisher reads. - auto* ring = sub_ring_at(region.base(), region.header(), sub.ring_index()); + auto* ring = sub_ring_at(region.base(), region.geometry(), sub.ring_index()); ring->has_waiter.store(ring::WaiterCarrier, std::memory_order_relaxed); CountingBackend injected; @@ -537,7 +537,7 @@ TEST_F(WaitFdTest, APublisherWithNoBackendSignalsNothing) auto region = SharedRegion::create(SHM_NAME, channel::PubSub, bare_cfg()); Subscriber sub(region); - auto* ring = sub_ring_at(region.base(), region.header(), sub.ring_index()); + auto* ring = sub_ring_at(region.base(), region.geometry(), sub.ring_index()); ring->has_waiter.store(ring::WaiterCarrier, std::memory_order_relaxed); CountingBackend injected;