From 34e87b374a48ef3b77ddd3fca018849b489c6f1c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 13 Jul 2026 15:14:33 +0800 Subject: [PATCH 1/2] fix(cluster): bind the GCS X-transfer copy->drop span to the page-LSN generation A destructive X-transfer serve captures the ship image (stable copy) and then drops the local copy as two separate steps. A local writer holding a cached X grant (cluster.gcs_block_local_cache) passes the bufmgr PCM gate on the raw pcm_state read BEFORE the content lock and is never re-verified, so it can commit to the page INSIDE the copy->drop window: the write lands after the copy, and the drop then discards the page and ships the pre-write image. The write is durably committed yet absent from the live block everywhere -- a silent lost write. The round-5 PINNED check cannot catch it because the writer's pin is already gone by the time the drop runs. Bind the copy and the drop to the page LSN as a generation token: cluster_bufmgr_drop_block_for_gcs_no_wire takes the copy-time expected_lsn and, under the same header spinlock that the writer's unpin released (a proper happens-before), refuses the drop (new CLUSTER_BUFMGR_GCS_DROP_STALE) when the page LSN advanced past it. Both destructive serve branches (master==holder self-ship and holder-forward) pass the copy-time LSN and fail-close STALE (and PINNED) with a retryable deny, so the re-serve copies the current page. A new xfer_stale_deny_count surfaces how often the window was exercised and closed. Also close the pcm_state restore clobber (gap c): the raced-pin restore path now writes back the saved mode only when pcm_state is still the staged N, so a concurrent authoritative grant during the (spinlock-free) InvalidateBufferTry window is not overwritten with a stale value. Liveness: a transient revoke deny (DENIED_MASTER_NOT_HOLDER / DENIED_PENDING_X) now surfaces the retryable 53R9X instead of FEATURE_NOT_SUPPORTED (0A000), which an application driver never retries; a genuine timeout keeps 0A000. New t/393 manufactures the interleave deterministically with a copy->drop sleep inject: the window write commits (v=101), the first requester attempt fails closed retryable, the generation gate refuses the stale drop (xfer_stale_deny advances), the committed write survives on the writer node (v>=101, never the lost-write v<101), and the lost-write detector never fires. Spec: spec-2.36-gcs-block-transfer.md Spec: spec-5.2-cross-node-tx-wait.md --- src/backend/cluster/cluster_debug.c | 9 +- src/backend/cluster/cluster_gcs_block.c | 144 ++++++++--- src/backend/cluster/cluster_inject.c | 14 ++ src/backend/storage/buffer/bufmgr.c | 53 +++- src/include/cluster/cluster_gcs_block.h | 14 +- src/test/cluster_tap/t/110_gcs_loopback.pl | 6 +- .../393_gcs_xfer_copy_drop_lostwrite_2node.pl | 231 ++++++++++++++++++ src/test/cluster_unit/test_cluster_debug.c | 5 + 8 files changed, 436 insertions(+), 40 deletions(-) create mode 100644 src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 5151e97455..ddf7bbb867 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2167,7 +2167,12 @@ dump_gcs(ReturnSetInfo *rsinfo) * dispatch pump never waits on a foreign pin); park_expired / * park_overflow = directives the master's ack budget fail-closes; * drop_pinned_deny = grant/transfer drops refused with a retryable - * deny because a foreign pin held the copy. */ + * deny because a foreign pin held the copy. xfer_stale_deny (round-6) + * = a grant/transfer drop refused because a local writer committed to + * the page between the ship-image copy and the drop (page LSN advanced + * past copy-time = generation gate); the re-serve ships the current + * page — a non-zero delta proves the copy->drop window was exercised + * and closed instead of silently losing the write. */ emit_row(rsinfo, "gcs", "invalidate_parked_count", fmt_int64((int64)cluster_gcs_get_invalidate_parked_count())); emit_row(rsinfo, "gcs", "invalidate_park_expired_count", @@ -2176,6 +2181,8 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_invalidate_park_overflow_count())); emit_row(rsinfo, "gcs", "drop_pinned_deny_count", fmt_int64((int64)cluster_gcs_get_drop_pinned_deny_count())); + emit_row(rsinfo, "gcs", "xfer_stale_deny_count", + fmt_int64((int64)cluster_gcs_get_xfer_stale_deny_count())); /* PGRAC: GCS-race round-4c FUNC-1 — storage-fallback SCN verify rows: * local pre-read proven current (no I/O) / stale pre-read re-read from diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 523504acc6..5ef9262e1d 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -205,6 +205,13 @@ typedef struct ClusterGcsBlockShared { pg_atomic_uint64 invalidate_park_expired_count; pg_atomic_uint64 invalidate_park_overflow_count; pg_atomic_uint64 drop_pinned_deny_count; + /* PGRAC: GCS serve-stall round-6 — the generation gate refused a drop + * because a local writer committed to the page between the ship-image + * copy and the drop (page LSN advanced past copy-time); the serve + * fail-closes with a retryable deny so the re-serve ships the current + * page. A non-zero delta over a workload proves the copy->drop window + * was actually exercised and closed (the silent-lost-write guard). */ + pg_atomic_uint64 xfer_stale_deny_count; /* PGRAC: GCS-race round-4c FUNC-1 — storage-fallback SCN verify/refresh. * A state=N GRANTED_STORAGE_FALLBACK now carries the master's * pi_watermark_scn (reply page_lsn field reused as an SCN carrier); the @@ -497,6 +504,7 @@ cluster_gcs_block_shmem_init(void) pg_atomic_init_u64(&ClusterGcsBlock->invalidate_park_expired_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->invalidate_park_overflow_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->drop_pinned_deny_count, 0); + pg_atomic_init_u64(&ClusterGcsBlock->xfer_stale_deny_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->fallback_scn_verify_pass_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->fallback_scn_refresh_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->fallback_scn_failclosed_count, 0); @@ -3637,6 +3645,30 @@ cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, * never a silent stale grant (Rule 8.A). */ if (clean_eligible) pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_fail_closed_count, 1); + + /* + * PGRAC: GCS serve-stall round-6 — a transient revoke deny (the holder + * fail-closed the drop with DENIED_MASTER_NOT_HOLDER / DENIED_PENDING_X + * because the copy was pinned (round-5 A2) or a local writer committed + * inside the copy->drop window (round-6 generation gate)) is a RETRYABLE + * condition: the re-serve ships the current page once the pin clears / the + * window is done. Surface the retryable class-53 code (53R9X) so an + * application driver retries the statement, instead of FEATURE_NOT_SUPPORTED + * (0A000), which reads as "permanently unsupported" and is never retried. + * A genuine timeout (got_reply == false) keeps 0A000: we could not prove + * the holder's state, so it is not a bounded transient. + */ + if (got_reply + && (reply_status == (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER + || reply_status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X)) + ereport(ERROR, (errcode(ERRCODE_CLUSTER_CLEAN_PAGE_XFER_UNAVAILABLE), + errmsg("cluster_gcs_block: X holder %d transiently refused the transfer " + "for tag spc=%u db=%u relNumber=%u block=%u", + holder_node, tag.spcOid, tag.dbOid, + (unsigned int)BufTagGetRelNumber(&tag), (unsigned int)tag.blockNum), + errhint("The holder's copy was pinned, or a local writer committed during " + "the transfer; retry the transaction."))); + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cluster_gcs_block: could not obtain X transfer from X holder %d " "for tag spc=%u db=%u relNumber=%u block=%u", @@ -4472,14 +4504,33 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * deny contract is "back off and retry, the retry * re-evaluates". */ - if (cluster_bufmgr_drop_block_for_gcs_no_wire(req->tag, &drop_lsn) - == CLUSTER_BUFMGR_GCS_DROP_PINNED) { - pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, 1); - cluster_gcs_block_dedup_remove(dedup_worker_id, &key); - gcs_block_send_reply(req->sender_node, req, - GCS_BLOCK_REPLY_DENIED_PENDING_X, InvalidXLogRecPtr, - NULL); - return; + /* GCS serve-stall round-6 RED harness: hold the copy->drop + * window open (see cluster_inject.c registry note). */ + CLUSTER_INJECTION_POINT("cluster-gcs-xfer-copy-drop-window"); + { + /* GCS serve-stall round-6: pass the copy-time page_lsn + * (captured by get_ship_image above) as the generation + * token. PINNED (a live foreign pin) and STALE (a local + * writer committed since the copy) both mean the shipped + * image must NOT be granted — fail-closed with the same + * retryable deny so the requester re-asks and the re-serve + * copies the current page (Rule 8.A). */ + ClusterBufmgrGcsDropResult dres = cluster_bufmgr_drop_block_for_gcs_no_wire( + req->tag, page_lsn, &drop_lsn); + + if (dres == CLUSTER_BUFMGR_GCS_DROP_PINNED + || dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { + if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->xfer_stale_deny_count, 1); + else + pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, + 1); + cluster_gcs_block_dedup_remove(dedup_worker_id, &key); + gcs_block_send_reply(req->sender_node, req, + GCS_BLOCK_REPLY_DENIED_PENDING_X, + InvalidXLogRecPtr, NULL); + return; + } } /* PGRAC: spec-6.12h D-h3a — ordering pin: the PI * conversion (inside the drop above) samples its @@ -6404,29 +6455,48 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo * keep our X untouched. NOT_RESIDENT still grants: no * copy left to stale-flush. */ - if (cluster_bufmgr_drop_block_for_gcs_no_wire(fwd->tag, &drop_lsn) - == CLUSTER_BUFMGR_GCS_DROP_PINNED) { - pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, 1); - /* undo the from-holder ship count taken with the - * grant status above — this reply is a deny now */ - pg_atomic_fetch_sub_u64(&ClusterGcsBlock->block_from_holder_ship_count, 1); - hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; - } else { - /* PGRAC: spec-6.12h D-h3a — ordering pin: the PI - * conversion (inside the drop above) precedes the reply - * send at the bottom of this handler - * (cluster_ic_rdma_send_envelope_sge), so the requester - * observes an envelope stamped at-or-above the ship-SCN - * boundary (cluster_pi_shadow.h proof item 2). */ - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_x_transfer_ship_count, 1); - if (GcsBlockForwardPayloadIsCleanEligible(fwd)) - pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_count, 1); - /* PGRAC: spec-6.12h D-h2 — if the D-h1 conversion kept our - * outgoing copy as a Past Image, report it to the master - * (unsolicited PI_KEPT ride; fire-and-forget — a lost note - * only leaves the PI untracked, fail-safe lingering). */ - if (cluster_bufmgr_block_is_pi(fwd->tag)) - gcs_block_pi_kept_note_send(fwd->tag, fwd->master_node); + /* GCS serve-stall round-6 RED harness: hold the copy->drop + * window open (see cluster_inject.c registry note). */ + CLUSTER_INJECTION_POINT("cluster-gcs-xfer-copy-drop-window"); + /* GCS serve-stall round-6: page_lsn (copy-time, from + * get_ship_image above) is the generation token. STALE (a + * local writer committed since the copy) joins PINNED as a + * retryable deny — never ship a stale image over a committed + * write (Rule 8.A). */ + { + ClusterBufmgrGcsDropResult dres = cluster_bufmgr_drop_block_for_gcs_no_wire( + fwd->tag, page_lsn, &drop_lsn); + + if (dres == CLUSTER_BUFMGR_GCS_DROP_PINNED + || dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { + if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->xfer_stale_deny_count, 1); + else + pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, + 1); + /* undo the from-holder ship count taken with the + * grant status above — this reply is a deny now */ + pg_atomic_fetch_sub_u64(&ClusterGcsBlock->block_from_holder_ship_count, + 1); + hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; + } else { + /* PGRAC: spec-6.12h D-h3a — ordering pin: the PI + * conversion (inside the drop above) precedes the reply + * send at the bottom of this handler + * (cluster_ic_rdma_send_envelope_sge), so the requester + * observes an envelope stamped at-or-above the ship-SCN + * boundary (cluster_pi_shadow.h proof item 2). */ + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_x_transfer_ship_count, + 1); + if (GcsBlockForwardPayloadIsCleanEligible(fwd)) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_count, 1); + /* PGRAC: spec-6.12h D-h2 — if the D-h1 conversion kept our + * outgoing copy as a Past Image, report it to the master + * (unsolicited PI_KEPT ride; fire-and-forget — a lost note + * only leaves the PI untracked, fail-safe lingering). */ + if (cluster_bufmgr_block_is_pi(fwd->tag)) + gcs_block_pi_kept_note_send(fwd->tag, fwd->master_node); + } } } } @@ -7313,8 +7383,12 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) ack_status = 2; /* race: not resident */ break; case CLUSTER_BUFMGR_GCS_DROP_PINNED: + case CLUSTER_BUFMGR_GCS_DROP_STALE: /* GCS serve-stall round-5 (A2): nothing changed, no ACK — the - * caller parks the directive and the LMS loop retries it. */ + * caller parks the directive and the LMS loop retries it. STALE is + * unreachable here (the invalidate wrapper passes no expected_lsn, so + * its generation gate never fires); treated like PINNED defensively + * (round-6). */ return false; } @@ -8048,6 +8122,12 @@ cluster_gcs_get_drop_pinned_deny_count(void) return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->drop_pinned_deny_count) : 0; } +uint64 +cluster_gcs_get_xfer_stale_deny_count(void) +{ + return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->xfer_stale_deny_count) : 0; +} + /* PGRAC: GCS-race round-4c FUNC-1 — 3 storage-fallback verify accessors. */ uint64 cluster_gcs_get_fallback_scn_verify_pass_count(void) diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 9f97e8b548..aac4358d5e 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -571,6 +571,20 @@ static ClusterInjectPoint cluster_injection_points[] = { { .name = "cluster-recovery-anchor-force-failclosed" }, { .name = "cluster-relmap-crash-after-stage" }, { .name = "cluster-relmap-crash-before-publish" }, + /* + * GCS serve-stall round-6 — X-transfer copy->drop generation RED. + * + * cluster-gcs-xfer-copy-drop-window: + * Fires in BOTH destructive X-transfer branches (master==holder + * self-ship and holder-forward), AFTER the ship image was captured + * and BEFORE the local drop. SLEEP holds the copy->drop window + * open so a cached-X local writer can be admitted inside it (the + * write lands after the copy, the drop then discards it): the + * silent-lost-write interleave the generation gate closes (the + * drop's page-LSN check refuses the stale-image drop, retryable). + * No lock/pin is held across the point. Driven by t/393. + */ + { .name = "cluster-gcs-xfer-copy-drop-window" }, /* * spec-2.41 D5 / P1-C — SCN lost-write detector behavioral trigger. * diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 087f65c6e5..138a64b890 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -7717,7 +7717,14 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ { buf_state = LockBufHdr(buf); - if (BufferTagsEqual(&buf->tag, &tag)) + /* PGRAC: GCS serve-stall round-6 (gap (c)) — restore the staged N only + * when it is still staged. The header spinlock was dropped for the + * InvalidateBufferTry attempt, so a concurrent grant / transition could + * have moved pcm_state off the staged N; an unconditional write-back + * would clobber that newer authoritative state with our stale saved + * value (Rule 8.A stale-grant). Restore only if nobody else changed it. */ + if (BufferTagsEqual(&buf->tag, &tag) && + buf->pcm_state == (uint8) PCM_STATE_N) buf->pcm_state = saved_pcm_state; UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; @@ -7945,7 +7952,8 @@ cluster_bufmgr_block_pcm_state(BufferTag tag) * no-stale-flush. * ======================================================================== */ ClusterBufmgrGcsDropResult -cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr *out_page_lsn) +cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn, + XLogRecPtr *out_page_lsn) { uint32 hashcode; LWLock *partition_lock; @@ -7998,6 +8006,26 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr *out_page_ls page_lsn = PageGetLSN(page); } + /* + * PGRAC: GCS serve-stall round-6 — generation gate. A local writer that + * held a cached X grant could commit to this page in the window between + * the transfer's ship-image copy and this drop (gaps (a) cached-X + * no-reverify + (b) copy->drop admission). Its pin is already gone (the + * refcount check above passed), so PINNED cannot catch it; the page LSN, + * read here under the same header spinlock that the writer's unpin + * released, is the generation token. A LSN past the caller's copy-time + * expected_lsn means the captured ship image is STALE: refuse the drop so + * the caller fail-closes with a retryable deny and the re-serve copies the + * current page (Rule 8.A — never grant a stale image over a committed + * write). expected_lsn == Invalid skips the gate. + */ + if (!XLogRecPtrIsInvalid(expected_lsn) && page_lsn != expected_lsn) + { + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + return CLUSTER_BUFMGR_GCS_DROP_STALE; + } + UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -8038,6 +8066,18 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr *out_page_ls return CLUSTER_BUFMGR_GCS_DROP_PINNED; } + /* PGRAC: GCS serve-stall round-6 — re-check the generation gate under + * this second header-lock hold: a local writer could have committed + * during the (blocking) XLogFlush window above. A LSN past expected_lsn + * means the ship image is stale; refuse without touching pcm_state + * (Rule 8.A). */ + if (!XLogRecPtrIsInvalid(expected_lsn) && + PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) + { + UnlockBufHdr(buf, buf_state); + return CLUSTER_BUFMGR_GCS_DROP_STALE; + } + saved_pcm_state = buf->pcm_state; buf->pcm_state = (uint8) PCM_STATE_N; @@ -8051,7 +8091,14 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr *out_page_ls if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ { buf_state = LockBufHdr(buf); - if (BufferTagsEqual(&buf->tag, &tag)) + /* PGRAC: GCS serve-stall round-6 (gap (c)) — restore the staged N only + * when it is still staged. The header spinlock was dropped for the + * InvalidateBufferTry attempt, so a concurrent grant / transition could + * have moved pcm_state off the staged N; an unconditional write-back + * would clobber that newer authoritative state with our stale saved + * value (Rule 8.A stale-grant). Restore only if nobody else changed it. */ + if (BufferTagsEqual(&buf->tag, &tag) && + buf->pcm_state == (uint8) PCM_STATE_N) buf->pcm_state = saved_pcm_state; UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 9b58d1f1bf..91b8a533b4 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1858,11 +1858,21 @@ extern void cluster_bufmgr_unlock_resident_stamp(Buffer buffer); * (no state cleared, no flush relied upon). The caller * parks the job and retries, or fail-closes with a * retryable deny — never spins. + * STALE GCS serve-stall round-6: a local writer committed to + * the page between the ship-image copy and this drop (the + * page LSN advanced past the caller's expected copy-time + * LSN), so the already-captured ship image is stale. + * NOTHING was changed; the caller MUST NOT grant the + * stale image — it fail-closes with a retryable deny so + * the re-serve copies the current page (the generation + * binding that closes the copy->drop silent-lost-write + * window, gaps (a)+(b)). */ typedef enum ClusterBufmgrGcsDropResult { CLUSTER_BUFMGR_GCS_DROP_DROPPED = 0, CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT, CLUSTER_BUFMGR_GCS_DROP_PINNED, + CLUSTER_BUFMGR_GCS_DROP_STALE, } ClusterBufmgrGcsDropResult; extern ClusterBufmgrGcsDropResult cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, @@ -1890,7 +1900,8 @@ extern void cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag * the §3.5 IC-dispatch (LMON) context. XLogFlush+InvalidateBuffer, with the * cache-eviction release wire suppressed (clears pcm_state=N first). */ extern ClusterBufmgrGcsDropResult -cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr *out_page_lsn); +cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn, + XLogRecPtr *out_page_lsn); /* PGRAC: spec-6.12h D-h2 — Past Image discard helpers. * block_is_pi: does this tag's resident buffer hold a D-h1 Past Image @@ -2178,6 +2189,7 @@ extern uint64 cluster_gcs_get_invalidate_parked_count(void); extern uint64 cluster_gcs_get_invalidate_park_expired_count(void); extern uint64 cluster_gcs_get_invalidate_park_overflow_count(void); extern uint64 cluster_gcs_get_drop_pinned_deny_count(void); +extern uint64 cluster_gcs_get_xfer_stale_deny_count(void); /* ============================================================ * Observability accessors (dump_gcs +8 NEW rows for block plane). diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index 122683300e..e713530946 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -65,12 +65,12 @@ sub gcs_value { $node->start; -# L1 — pg_cluster_state.gcs surface has 108 keys (round-4c fallback-scn + spec-7.2 D6 hist). +# L1 — pg_cluster_state.gcs surface has 109 keys (round-4c fallback-scn + spec-7.2 D6 hist). is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L1 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6)'); + '109', + 'L1 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows; round-6 +1 xfer_stale_deny) (spec-7.2 D6)'); # L2 — api_state = "active" after postmaster phase 1 init. diff --git a/src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl b/src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl new file mode 100644 index 0000000000..d7b94e9851 --- /dev/null +++ b/src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl @@ -0,0 +1,231 @@ +# X-transfer copy->drop window silent lost write (2-node, deterministic). +# +# The destructive X-transfer serve captures the ship image (stable copy) +# and then drops the local copy as two separate steps with no admission +# barrier between them. A LOCAL writer holding a cached X grant +# (cluster.gcs_block_local_cache) passes the bufmgr PCM gate on the raw +# pcm_state read BEFORE taking the content lock and is never re-verified, +# so it can be admitted INSIDE the copy->drop window: its committed write +# lands on the local page after the copy was taken, the drop then discards +# the page, and the pre-write image ships to the requester. The write is +# durably committed (WAL flushed by the drop) yet absent from the live +# block everywhere -- the silent lost write the REVOKING+generation +# linearization closes (adjudication gaps (a) cached-X no-reverify and +# (b) copy->drop admission, one interleave). +# +# Deterministic rig: the cluster-gcs-xfer-copy-drop-window sleep inject +# holds the window open for 1.5s on the serving node; the local writer runs +# inside it, then the inject is disarmed so the requester's retry re-serves +# cleanly. Interleave validity is asserted, not assumed: the requester's +# UPDATE must have waited through at least one stall (elapsed >= 1.3s) and +# the window write must have run unblocked (elapsed < 1s, i.e. it was +# admitted, not queued behind the transfer). +# +# L1 pair boots, cached-X local cache on. +# L2 window write is admitted and commits inside the stall. +# L3 the committed window write survives the transfer -- final value on +# BOTH nodes reflects seed(1) + window(+100) + requester(+10) = 111. +# A broken window ships the pre-write image: both nodes read 11 and +# the +100 is silently gone. +# L4 the generation gate fired: xfer_stale_deny_count advanced (the +# window was exercised and CLOSED, not merely avoided) and the +# requester UPDATE ultimately succeeded on the re-serve. +# +# Author: SqlRush +# Portions Copyright (c) 2026, pgrac contributors +# +# Spec: spec-2.36-gcs-block-transfer.md +# Spec: spec-5.2-cross-node-tx-wait.md +use strict; +use warnings FATAL => 'all'; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use IPC::Run (); +use Test::More; +use Time::HiRes qw(usleep time); + +sub state_int { + my ($node, $cat, $key) = @_; + + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state + WHERE category='$cat' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + +sub arm_inject { + my ($node, $val) = @_; + + $node->safe_psql('postgres', + "ALTER SYSTEM SET cluster.injection_points = '$val'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + return; +} + +sub disarm_inject { + my ($node) = @_; + + $node->safe_psql('postgres', 'ALTER SYSTEM RESET cluster.injection_points'); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + return; +} + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'xfer_lostwrite', + quorum_voting_disks => 3, + data_port_span => 2, + shared_data => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.crossnode_runtime_visibility = on', + # The requester must outlast one stall + STALE deny + a clean + # re-serve, so give it a generous retransmit budget and reply window. + 'cluster.gcs_reply_timeout_ms = 2000', + 'cluster.gcs_block_retransmit_max_retries = 12', + 'cluster.gcs_block_starvation_max_retries = 60' ]); +$pair->start_pair; +usleep(3_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'L1 peers 0->1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'L1 peers 1->0 connected'); +is($pair->node0->safe_psql('postgres', 'SHOW cluster.gcs_block_local_cache'), + 'on', 'L1 cached-X local cache on (the gate-skip precondition)'); + +# Coinciding-filepath fixture (shared storage, per-node catalogs). +my $table; +for my $i (1 .. 8) { + my $t = "lw_t$i"; + + $_->safe_psql('postgres', "CREATE TABLE $t (k int, v int)") + for ($pair->node0, $pair->node1); + my $p0 = $pair->node0->safe_psql('postgres', qq{SELECT pg_relation_filepath('$t')}); + my $p1 = $pair->node1->safe_psql('postgres', qq{SELECT pg_relation_filepath('$t')}); + + if ($p0 eq $p1) { + $table = $t; + last; + } +} +die 'no coinciding filepath' unless defined $table; +diag("table=$table"); + +# Seed on node0 (node0 = first toucher = block master AND X holder), then +# read back + checkpoint so the seed ITL slot is cleaned out (an ACTIVE +# slot would route the request to the READ_IMAGE path, not the destructive +# transfer). +$pair->node0->safe_psql('postgres', "INSERT INTO $table VALUES (1, 1)"); +$pair->node0->safe_psql('postgres', 'CHECKPOINT'); +$pair->node0->safe_psql('postgres', "SELECT v FROM $table WHERE k = 1"); + +my $lw0_b = state_int($pair->node0, 'gcs', 'lost_write_detected_count'); +my $lw1_b = state_int($pair->node1, 'gcs', 'lost_write_detected_count'); +my $stale_b = state_int($pair->node0, 'gcs', 'xfer_stale_deny_count'); + +# Hold the copy->drop window open on the serving node (node0) for 1.5s. +arm_inject($pair->node0, 'cluster-gcs-xfer-copy-drop-window:sleep:1500000'); +usleep(500_000); + +# Requester: node1 UPDATE (N->X request into node0's self-ship path). +my ($in, $out, $err) = ('', '', ''); +my $t_req = time(); +my $reqh = IPC::Run::start( + [ 'psql', '-X', '-q', '-d', $pair->node1->connstr('postgres'), + '-c', "UPDATE $table SET v = v + 10 WHERE k = 1" ], + \$in, \$out, \$err); + +# Let the request reach the serve and take the stable copy, then run the +# window write on node0 while the serve is held before its drop. +usleep(600_000); + +my $t_win = time(); +my ($wrc, $wout, $werr) = $pair->node0->psql('postgres', + "UPDATE $table SET v = v + 100 WHERE k = 1 RETURNING v"); +my $win_elapsed = time() - $t_win; +diag(sprintf("window write: rc=%d v=[%s] elapsed=%.2fs err=[%s]", + $wrc, $wout // '', $win_elapsed, $werr // '')); + +is($wrc, 0, 'L2 window write committed (cached-X admission inside the window)'); +is($wout // '', '101', + 'L2 window write applied on the pre-transfer local copy (1 + 100)'); +cmp_ok($win_elapsed, '<', 1.0, + 'L2 window write was admitted, not queued behind the transfer'); + +# The window write is committed; disarm so the requester's retry after the +# STALE deny re-serves the CURRENT (post-write) image cleanly. The first +# requester attempt fails-closed RETRYABLE (53R9X, the round-6 transient +# revoke deny) — it must NEVER silently succeed on a stale image. +$reqh->finish; +my $req1_err = $err // ''; +my $req_elapsed = time() - $t_req; +diag(sprintf("requester UPDATE attempt1: exit=%d elapsed=%.2fs err=[%s]", + $reqh->result, $req_elapsed, $req1_err)); +isnt($reqh->result, 0, + 'L2 requester attempt1 fails-closed (never a silent stale grant)'); +like($req1_err, qr/transiently refused|clean-page|53R9X|retry/i, + 'L2 attempt1 failure is the retryable transient-revoke deny'); +cmp_ok($req_elapsed, '>', 1.3, + 'L2 requester attempt1 waited through the stall (interleave really happened)'); + +disarm_inject($pair->node0); +usleep(500_000); + +# L3: the committed +100 must SURVIVE the transfer. Read it on the writer +# node (node0) -- an own-xid read, no cross-node resolve, so the orthogonal +# fresh-cluster low-xid 53R97 (step-1 territory) cannot noise this check. +# A broken copy->drop window discards node0's committed v=101 and ships the +# pre-write image, so node0 loses the +100 (converges to 1 or, after the +# requester's +10, 11). The fixed generation gate refuses the stale drop, +# so node0 keeps its committed write: the value still contains +100. +my $v0; +for my $try (1 .. 15) { + my ($rc, $out0, $e0) = $pair->node0->psql('postgres', + "SELECT v FROM $table WHERE k = 1"); + if ($rc == 0 && defined $out0 && $out0 ne '') { + $v0 = int($out0); + last; + } + usleep(300_000); +} +diag("L3 node0 (writer) reads v=" . (defined $v0 ? $v0 : '(unreadable)')); +ok(defined $v0, 'L3 node0 can read its own committed row'); +cmp_ok($v0 // 0, '>=', 101, + 'L3 the committed window write (+100) survived the transfer ' + . '(v=' . ($v0 // 'undef') . '; a lost write would leave v<101)'); + +# L3b: best-effort application retry of the requester's +10 (the retryable +# contract). Cross-node, so it may transiently hit the orthogonal low-xid +# 53R97 on a fresh cluster; tolerated. When it lands, node0 converges to +# 111 -- still +100-present, never a lost write. +my $req2_ok = 0; +my $req2_err = ''; +for my $try (1 .. 15) { + my ($rc, $out2, $e2) = $pair->node1->psql('postgres', + "UPDATE $table SET v = v + 10 WHERE k = 1"); + if ($rc == 0) { + $req2_ok = 1; + last; + } + $req2_err = (split /\n/, ($e2 // 'unknown'))[0]; + usleep(400_000); +} +diag("L3b requester +10 retry: ok=$req2_ok last_err=[$req2_err]"); + +# L4: the generation gate fired -- the copy->drop window was exercised and +# CLOSED (a stale-image drop refused), not merely avoided. No lost-write +# detector fire either (the write was never dropped, so no torn history). +my $stale_delta = state_int($pair->node0, 'gcs', 'xfer_stale_deny_count') - $stale_b; +my $lw0_delta = state_int($pair->node0, 'gcs', 'lost_write_detected_count') - $lw0_b; +my $lw1_delta = state_int($pair->node1, 'gcs', 'lost_write_detected_count') - $lw1_b; +diag("L4 xfer_stale_deny delta node0=$stale_delta " + . "lost_write_detected delta node0=$lw0_delta node1=$lw1_delta"); +cmp_ok($stale_delta, '>=', 1, + 'L4 generation gate refused >=1 stale-image drop (copy->drop window closed)'); +is($lw0_delta + $lw1_delta, 0, + 'L4 no lost-write detector fire (the write was never dropped)'); + +$pair->stop_pair; +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index a1308d6ec0..9a6dc9f68b 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1271,6 +1271,11 @@ cluster_gcs_get_drop_pinned_deny_count(void) { return 0; } +uint64 +cluster_gcs_get_xfer_stale_deny_count(void) +{ + return 0; +} /* GCS-race round-4c FUNC-1 stubs: 3 storage-fallback SCN verify accessors. */ uint64 From 843367d421888b23a8a87c57bb0309324f4bab71 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 13 Jul 2026 15:43:30 +0800 Subject: [PATCH 2/2] test(cluster): bump injection + gcs-key baselines for the round-6 xfer inject Adding cluster-gcs-xfer-copy-drop-window (+1 injection point) and xfer_stale_deny_count (+1 gcs dump key) ripples the count baselines: t/030 acceptance: 173->174 injects, 346->348 inject sub-keys t/015 inject: 173->174 count, +name in the ORDER BY name registry list t/110/111-115: gcs category 108->109 t/015's name-list and t/116's gcs count were ALSO pre-existing-stale on main (t/015 was missing cluster-gcs-block-done-drop / -fallback-refresh-stale / -yield-notify-drop from round-4c/fix-2; t/116 still asserted the pre-round-5 value 95 vs the real 108) -- both are brought to the empirically-verified current value (registry list from the live view; gcs count 95->109) rather than left red. Also correct the t/393 header to match what it actually asserts: it covers ONLY the copy->drop page-LSN stale-image window (writer-node survival + xfer_stale_deny + no detector fire), NOT a full REVOKING linearization, NOT two-node re-serve convergence to 111, and NOT gap (c). Spec: spec-2.36-gcs-block-transfer.md --- src/test/cluster_tap/t/015_inject.pl | 7 ++- src/test/cluster_tap/t/030_acceptance.pl | 6 +-- .../cluster_tap/t/111_gcs_block_ship_2node.pl | 12 ++--- .../t/112_gcs_block_retransmit_2node.pl | 12 ++--- .../cluster_tap/t/113_gcs_block_2way_2node.pl | 12 ++--- .../cluster_tap/t/114_gcs_block_3way_2node.pl | 12 ++--- .../cluster_tap/t/115_gcs_block_3way_3node.pl | 4 +- .../t/116_gcs_block_lost_write_2node.pl | 10 ++--- .../393_gcs_xfer_copy_drop_lostwrite_2node.pl | 45 ++++++++++++------- 9 files changed, 66 insertions(+), 54 deletions(-) diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 572ad43277..932e6e5248 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,7 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '173', - 'pg_stat_cluster_injections returns 173 rows (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '174', 'pg_stat_cluster_injections returns 174 rows (serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +85,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '170 injection point names match the full registry (gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-done-drop,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-fallback-refresh-stale,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-gcs-block-yield-notify-drop,cluster-gcs-xfer-copy-drop-window,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '171 injection point names match the full registry (serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6fe1033fb3..c76a8eee9a 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '173', 'M1 173 injection points (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '174', 'M1 174 injection points (serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '346', - 'M5 inject category has 173×2 = 346 sub-keys (.fault_type + .hits; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + ) eq '348', + 'M5 inject category has 174×2 = 348 sub-keys (.fault_type + .hits; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index 0ca5271dc2..ad4cd3abab 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -8,7 +8,7 @@ # # L1 ClusterPair startup — both postmasters healthy + tier1 connected # L2 fresh baseline gcs counters on both nodes (block_* counters = 0) -# L3 pg_cluster_state.gcs category has 108 keys (spec-7.2 D6+flip) (cumulative through +# L3 pg_cluster_state.gcs category has 109 keys (spec-7.2 D6+flip) (cumulative through # spec-6.13 direct-land observability) # L4 4 NEW wait events registered in pg_stat_cluster_wait_events: # ClusterGCSBlockShipWait, ClusterGCSBlockRequestDispatch, @@ -114,19 +114,19 @@ sub gcs_int_value { # ============================================================ -# L3: pg_cluster_state.gcs category has 108 keys (spec-7.2 D6+flip) +# L3: pg_cluster_state.gcs category has 109 keys (spec-7.2 D6+flip) # (cumulative GCS surface through spec-6.13 direct-land observability). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node0 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '109', + 'L3 node0 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node1 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '109', + 'L3 node1 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index ab243cd9ce..9b67b3f726 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 9 NEW reliability counters all 0 on both nodes -# L3 pg_cluster_state.gcs has 108 keys (cumulative through spec-7.2 D6 hist) +# L3 pg_cluster_state.gcs has 109 keys (cumulative through spec-7.2 D6 hist) # L4 2 NEW wait events registered (ClusterGCSBlockRetransmitWait + # ClusterGCSBlockEpochStaleRetry) # L5 CLUSTER_WAIT_EVENTS_COUNT = 120 (spec-7.2 +2) @@ -111,18 +111,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs category has 108 keys (cumulative through spec-7.2 D6 hist). +# L3: pg_cluster_state.gcs category has 109 keys (cumulative through spec-7.2 D6 hist). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node0 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); + '109', + 'L3 node0 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node1 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); + '109', + 'L3 node1 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); # ============================================================ diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index 94356f27f0..d76f5422b3 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 7 NEW counters all 0 + catversion >= 202605420 -# L3 pg_cluster_state.gcs category has 108 keys (spec-7.2 D6+flip) (cumulative through spec-6.13) +# L3 pg_cluster_state.gcs category has 109 keys (spec-7.2 D6+flip) (cumulative through spec-6.13) # L4 cross-node forward path: node A read first → master_holder = A; # force same tag on node B via test-only injection → master # chooses forward path → A direct-ships to B → block_forward_sent @@ -109,18 +109,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 108 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 109 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node0 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '109', + 'L3 node0 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node1 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '109', + 'L3 node1 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 2614072f63..c4d368b15f 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -12,7 +12,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 6 NEW spec-2.36 counters all 0 -# L3 pg_cluster_state.gcs has 108 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) +# L3 pg_cluster_state.gcs has 109 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) # L4 catversion lower-bound >= 202605430; wait event count == 123 # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader @@ -107,18 +107,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 108 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 109 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node0 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '109', + 'L3 node0 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - 'L3 node1 pg_cluster_state.gcs category has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '109', + 'L3 node1 pg_cluster_state.gcs category has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index 2b0362a55a..1aa7bd1056 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -130,8 +130,8 @@ sub gcs_int is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '108', - "L4 node$i pg_cluster_state.gcs has 108 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)"); + '109', + "L4 node$i pg_cluster_state.gcs has 109 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)"); } diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index 1c3b1cdb90..10faa8fbf1 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -20,7 +20,7 @@ # L7 SQLSTATE 53R93 ERRCODE_CLUSTER_LOST_WRITE_DETECTED literal- # encodable in PG SQL (catalog 形式 verification) # L8 GUC switch back to 'error' SHOW returns 'error' -# L9 pg_cluster_state.gcs category has 95 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a) +# L9 pg_cluster_state.gcs category has 109 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a) # L10 Reply status enum value 12 (DENIED_LOST_WRITE) is新增的 # 最大 value (baseline workload must not trigger lost-write) # L11 spec-2.41 D / P1-C — behavioral lost-write inject: a @@ -108,8 +108,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '95', - 'L2 pg_cluster_state.gcs category has 95 keys (gcs-race-fix-2 +6 rows) (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); + '109', + 'L2 pg_cluster_state.gcs category has 109 keys (gcs-race-fix-2 +6 rows) (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); # ============================================================ @@ -199,8 +199,8 @@ sub gcs_int is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '95', - 'L9 node1 pg_cluster_state.gcs has 95 keys (gcs-race-fix-2 +6 rows) (cross-node parity)'); + '109', + 'L9 node1 pg_cluster_state.gcs has 109 keys (gcs-race-fix-2 +6 rows) (cross-node parity)'); # ============================================================ diff --git a/src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl b/src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl index d7b94e9851..c074467139 100644 --- a/src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl +++ b/src/test/cluster_tap/t/393_gcs_xfer_copy_drop_lostwrite_2node.pl @@ -1,5 +1,13 @@ # X-transfer copy->drop window silent lost write (2-node, deterministic). # +# SCOPE: this test covers exactly ONE defect -- the copy->drop page-LSN +# stale-image window (adjudication gaps (a) cached-X no-reverify and (b) +# copy->drop admission, which manifest as a single physical interleave). +# It does NOT prove a full REVOKING/generation linearization, does NOT +# assert two-node re-serve convergence to a final value, and does NOT +# cover the pcm_state restore ABA (gap (c)). See the closure doc for the +# remaining follow-ups. +# # The destructive X-transfer serve captures the ship image (stable copy) # and then drops the local copy as two separate steps with no admission # barrier between them. A LOCAL writer holding a cached X grant @@ -9,27 +17,32 @@ # lands on the local page after the copy was taken, the drop then discards # the page, and the pre-write image ships to the requester. The write is # durably committed (WAL flushed by the drop) yet absent from the live -# block everywhere -- the silent lost write the REVOKING+generation -# linearization closes (adjudication gaps (a) cached-X no-reverify and -# (b) copy->drop admission, one interleave). +# block everywhere -- a silent lost write. The page-LSN generation gate +# refuses the stale-image drop, so the committed write is not lost. # # Deterministic rig: the cluster-gcs-xfer-copy-drop-window sleep inject # holds the window open for 1.5s on the serving node; the local writer runs -# inside it, then the inject is disarmed so the requester's retry re-serves -# cleanly. Interleave validity is asserted, not assumed: the requester's -# UPDATE must have waited through at least one stall (elapsed >= 1.3s) and -# the window write must have run unblocked (elapsed < 1s, i.e. it was -# admitted, not queued behind the transfer). +# inside it. Interleave validity is asserted, not assumed: the requester's +# UPDATE must have waited through the stall (elapsed >= 1.3s) and the window +# write must have run unblocked (elapsed < 1s, i.e. it was admitted, not +# queued behind the transfer). # # L1 pair boots, cached-X local cache on. -# L2 window write is admitted and commits inside the stall. -# L3 the committed window write survives the transfer -- final value on -# BOTH nodes reflects seed(1) + window(+100) + requester(+10) = 111. -# A broken window ships the pre-write image: both nodes read 11 and -# the +100 is silently gone. -# L4 the generation gate fired: xfer_stale_deny_count advanced (the -# window was exercised and CLOSED, not merely avoided) and the -# requester UPDATE ultimately succeeded on the re-serve. +# L2 window write is admitted and commits inside the stall; the first +# requester attempt fails closed with the RETRYABLE transient revoke +# deny (53R9X) -- never a silent stale grant. +# L3 the committed window write SURVIVES the transfer -- read on the +# writer node (node0, own-xid, no cross-node resolve): v >= 101 (the +# +100 is present). A broken window drops node0's committed page and +# ships the pre-write image, leaving v < 101 (the +100 silently gone). +# L3b BEST-EFFORT ONLY: the requester's +10 retry. It is cross-node and +# on a fresh cluster may persistently hit the ORTHOGONAL low-xid +# 53R97 (a separate, already-fail-closed path), so its success is +# NOT asserted -- two-node re-serve convergence is a follow-up, not +# proven here. +# L4 the generation gate fired (xfer_stale_deny advanced: the window was +# exercised and closed, not merely avoided) and the lost-write +# detector did NOT fire (the write was never dropped). # # Author: SqlRush # Portions Copyright (c) 2026, pgrac contributors