diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index b9f85d0471..b081215921 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2282,6 +2282,9 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_lost_write_invalidscn_failclosed_count())); emit_row(rsinfo, "gcs", "lost_write_not_scn_tracked_skip_count", fmt_int64((int64)cluster_gcs_get_lost_write_not_scn_tracked_skip_count())); + /* PGRAC: branch-1 — master-direct stale ship rescued via shared storage. */ + emit_row(rsinfo, "gcs", "lost_write_master_direct_storage_fallback_count", + fmt_int64((int64)cluster_gcs_get_lost_write_master_direct_storage_fallback_count())); /* PGRAC: spec-5.2 D2 — X-holder read-image ship counter. */ emit_row(rsinfo, "gcs", "cf_xheld_read_ship_count", fmt_int64((int64)cluster_gcs_get_cf_xheld_read_ship_count())); diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 2bd5e647b4..631736d0aa 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -257,6 +257,12 @@ typedef struct ClusterGcsBlockShared { lost_write_invalidscn_failclosed_count; /* §2.6 b2: tracked block, shipped InvalidScn */ pg_atomic_uint64 lost_write_not_scn_tracked_skip_count; /* §2.6 b1: expected InvalidScn → skip */ + /* PGRAC: branch-1 (S3 step-2 forensics) — a STALE master-direct ship whose + * shared-storage version covers the watermark is rescued to + * GRANTED_STORAGE_FALLBACK instead of DENIED_LOST_WRITE (availability: + * the requester reads storage instead of aborting 53R93). The refused + * twin (storage unprovable) keeps bumping lost_write_detected_count. */ + pg_atomic_uint64 lost_write_master_direct_storage_fallback_count; pg_atomic_uint64 redo_coverage_required_lsn_zero_count; /* serve-gate required_lsn==0 (cold/degrade) */ pg_atomic_uint64 redo_coverage_gate_block_count; /* serve-gate not-covered (block) */ @@ -544,6 +550,7 @@ cluster_gcs_block_shmem_init(void) /* PGRAC: spec-2.41 D7 — 4 NEW SCN detector + redo-coverage counters init. */ pg_atomic_init_u64(&ClusterGcsBlock->lost_write_invalidscn_failclosed_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->lost_write_not_scn_tracked_skip_count, 0); + pg_atomic_init_u64(&ClusterGcsBlock->lost_write_master_direct_storage_fallback_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->redo_coverage_required_lsn_zero_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->redo_coverage_gate_block_count, 0); /* PGRAC: spec-5.2 D2 — X-holder read-image ship counter init. */ @@ -5435,6 +5442,22 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo if (cluster_injection_should_skip("cluster-gcs-block-stale-ship")) shipped_scn = InvalidScn; + /* branch-1 (S3 step-2 forensics) inject — simulate a RETAINED STALE + * RESIDENT (a kept Past Image serving as the grant payload): force the + * SHIPPED pd_block_scn one time-step below the valid watermark so the + * verdict is STALE (§2.6 branch 1) while shared storage keeps its real + * version. Master-direct site ONLY — the holder-forward twin must not + * fire this, so a test that greens can only have exercised THIS path. + * The predecessor comes from the SCN layer (fails closed to InvalidScn + * = leave the shipped value alone; the test round simply retries). */ + CLUSTER_INJECTION_POINT("cluster-gcs-block-stale-ship-resident"); + if (cluster_injection_should_skip("cluster-gcs-block-stale-ship-resident")) { + SCN forced_stale_scn = cluster_scn_time_predecessor(expected_scn); + + if (SCN_VALID(forced_stale_scn)) + shipped_scn = forced_stale_scn; + } + verdict = gcs_block_lost_write_verdict(expected_scn, shipped_scn); if (verdict == GCS_LOST_WRITE_SKIP) { /* spec-2.41 D7 observability — block not SCN-tracked (no fire). */ @@ -5473,6 +5496,103 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo wm_have ? wm_prov.epoch : 0, wm_have ? (uint64)wm_prov.old_scn : 0, wm_have ? (uint64)wm_prov.new_scn : 0, wm_have ? (int)(wm_prov.new_scn == expected_scn) : -1))); + + /* + * PGRAC: branch-1 (S3 step-2 forensics) — storage-fallback rescue. + * + * A STALE verdict here means the master is holding a RETAINED + * stale resident (a kept Past Image) as the would-be grant + * payload. When the shared-storage page already covers the + * authoritative watermark, the cluster-proven current version is + * durably readable: convert the reply to + * GRANTED_STORAGE_FALLBACK (ship no image; page_lsn carries the + * watermark — the same contract as the state=N grant above) so + * the requester proves/refreshes its copy through + * cluster_gcs_block_fallback_verify_refresh instead of aborting + * 53R93 on every hit. The verdict re-check uses the same + * gcs_block_lost_write_verdict SCN order the detector itself + * trusts (spec-2.41 §2.6). + * + * ANOMALY (shipped InvalidScn on a tracked tag) is NOT rescued: + * an unstamped resident says the master's own view is broken — + * stay fail-closed. A failed/unverifiable storage read keeps + * the fail-closed DENIED too (Rule 8.A). + */ + if (verdict == GCS_LOST_WRITE_FAIL_STALE) { + SCN storage_scn = InvalidScn; + bool storage_read_ok = false; + MemoryContext probe_cxt = CurrentMemoryContext; + + /* + * PGRAC: the storage probe is the only disk I/O on this + * self-check path and smgrread can ereport(ERROR) (short + * read on a concurrent truncate/drop, real I/O failure). + * An uncaught throw here would leak the ship image — a + * live_sge borrow is a raw pin outside ResourceOwner + * tracking — and drop the reply after produce_reply already + * applied the PCM transition, wedging the requester. Catch + * locally and fall through to the fail-closed DENIED arm, + * which releases the image and still replies (Rule 8.A). + */ + PG_TRY(); + { + storage_read_ok + = cluster_bufmgr_read_storage_scn_for_gcs(req->tag, &storage_scn); + } + PG_CATCH(); + { + ErrorData *edata; + + MemoryContextSwitchTo(probe_cxt); + edata = CopyErrorData(); + FlushErrorState(); + ereport(LOG, (errmsg_internal( + "cluster_gcs_block: master-direct storage-fallback probe " + "failed; keeping fail-closed DENIED_LOST_WRITE: %s", + edata->message != NULL ? edata->message : "(no message)"))); + FreeErrorData(edata); + storage_read_ok = false; + storage_scn = InvalidScn; + } + PG_END_TRY(); + + /* Test hook: pretend storage is unverifiable so the rescue + * refuses and the original fail-closed DENIED ships. */ + CLUSTER_INJECTION_POINT("cluster-gcs-block-master-direct-fallback-storage-stale"); + if (cluster_injection_should_skip( + "cluster-gcs-block-master-direct-fallback-storage-stale")) + storage_scn = InvalidScn; + + if (storage_read_ok + && gcs_block_lost_write_verdict(expected_scn, storage_scn) + == GCS_LOST_WRITE_PASS) { + ereport( + LOG, + (errmsg_internal( + "cluster_gcs_block: master-direct stale ship rescued to " + "GRANTED_STORAGE_FALLBACK: tag spc=%u db=%u rel=%u block=%u " + "fork=%d expected pi_watermark_scn=" UINT64_FORMAT + " shipped pd_block_scn=" UINT64_FORMAT + " storage pd_block_scn=" UINT64_FORMAT + " requester=%d request_id=" UINT64_FORMAT " epoch=" UINT64_FORMAT, + req->tag.spcOid, req->tag.dbOid, req->tag.relNumber, req->tag.blockNum, + (int)req->tag.forkNum, (uint64)expected_scn, (uint64)shipped_scn, + (uint64)storage_scn, req->sender_node, req->request_id, req->epoch))); + status = GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK; + page_lsn = (XLogRecPtr)expected_scn; + gcs_block_release_ship_image(block_payload_release_cb, + block_payload_release_arg); + block_payload = NULL; + block_payload_lkey = 0; + block_payload_release_cb = NULL; + block_payload_release_arg = NULL; + if (ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64( + &ClusterGcsBlock->lost_write_master_direct_storage_fallback_count, 1); + goto build_and_send_reply; + } + } + status = GCS_BLOCK_REPLY_DENIED_LOST_WRITE; page_lsn = InvalidXLogRecPtr; gcs_block_release_ship_image(block_payload_release_cb, block_payload_release_arg); @@ -8770,6 +8890,15 @@ cluster_gcs_get_lost_write_avoid_count(void) return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->lost_write_avoid_count) : 0; } +/* PGRAC: branch-1 master-direct storage-fallback rescue accessor. */ +uint64 +cluster_gcs_get_lost_write_master_direct_storage_fallback_count(void) +{ + return ClusterGcsBlock ? pg_atomic_read_u64( + &ClusterGcsBlock->lost_write_master_direct_storage_fallback_count) + : 0; +} + /* PGRAC: spec-2.41 D7 — SCN detector + redo-coverage observability accessors. */ uint64 cluster_gcs_get_lost_write_invalidscn_failclosed_count(void) diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 1946fc73e1..c040c0fe98 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -661,6 +661,26 @@ static ClusterInjectPoint cluster_injection_points[] = { * (deferred behavioral trigger), activated here for spec-2.41 §6 D. */ { .name = "cluster-gcs-block-stale-ship" }, + /* + * branch-1 master-direct storage-fallback (S3 step-2 forensics verdict: + * a master-retained Past Image served as the master-direct grant payload). + * + * cluster-gcs-block-stale-ship-resident: + * Fires ONLY at the master-direct self-check. SKIP forces the + * SHIPPED pd_block_scn to (valid watermark - 1) — §2.6 branch 1 + * STALE with a real shared-storage version left intact — so the + * ship converts to GRANTED_STORAGE_FALLBACK when storage covers the + * watermark (lost_write_master_direct_storage_fallback_count grows) + * instead of failing the requester with 53R93. + * + * cluster-gcs-block-master-direct-fallback-storage-stale: + * Fires inside the fallback rescue probe. SKIP forces the storage + * pd_block_scn read to InvalidScn so the rescue is REFUSED and the + * original fail-closed DENIED_LOST_WRITE ships — proves the rescue + * never converts when shared storage cannot be proven current. + */ + { .name = "cluster-gcs-block-stale-ship-resident" }, + { .name = "cluster-gcs-block-master-direct-fallback-storage-stale" }, /* * spec-2.38 D14 — SI Broadcaster fault injection. * diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index e65ae353a8..777b85586d 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -6993,6 +6993,36 @@ cluster_bufmgr_probe_block_for_gcs(BufferTag tag) return valid; } +/* + * Read the shared-storage version for a GCS lost-write rescue probe. This is + * deliberately independent of the local buffer table: a BM_VALID mirror is + * residency only and must not be consulted as the storage-version carrier (the + * resident copy is exactly what the caller has just proven stale). The + * physical read happens without any PCM/GRD lock held by the caller. + * + * Returns false (with *out_page_scn = InvalidScn) when the page fails + * verification; the caller must then keep its fail-closed verdict. + */ +bool +cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) +{ + PGAlignedBlock scratch; + SMgrRelation reln; + + if (out_page_scn != NULL) + *out_page_scn = InvalidScn; + + reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); + smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); + if (!PageIsVerifiedExtended((Page) scratch.data, tag.blockNum, + PIV_LOG_WARNING | PIV_REPORT_STAT)) + return false; + + if (out_page_scn != NULL) + *out_page_scn = ((PageHeader) scratch.data)->pd_block_scn; + return true; +} + /* * Copy the 8KB block bytes for `tag` into *dst, flushing WAL up to the * page's LSN before reading the bytes (HC82 I-WAL-before-ship). Sets diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index b7db575530..913d670c77 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1827,6 +1827,7 @@ StaticAssertDecl(GCS_BLOCK_DATA_SIZE == BLCKSZ, #include "access/xlogdefs.h" /* XLogRecPtr */ #include "cluster/cluster_pcm_lock.h" /* PcmLockMode for invalidate helper */ extern bool cluster_bufmgr_probe_block_for_gcs(BufferTag tag); +extern bool cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn); extern bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst); extern bool cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page_lsn, void **out_page_addr, @@ -2334,6 +2335,8 @@ extern uint64 cluster_gcs_get_lost_write_detected_count(void); extern uint64 cluster_gcs_get_lost_write_avoid_count(void); /* PGRAC: spec-2.41 D7 — SCN detector + redo-coverage observability accessors. */ extern uint64 cluster_gcs_get_lost_write_invalidscn_failclosed_count(void); +/* PGRAC: branch-1 master-direct storage-fallback rescue accessor. */ +extern uint64 cluster_gcs_get_lost_write_master_direct_storage_fallback_count(void); extern uint64 cluster_gcs_get_lost_write_not_scn_tracked_skip_count(void); extern uint64 cluster_gcs_get_redo_coverage_required_lsn_zero_count(void); extern uint64 cluster_gcs_get_redo_coverage_gate_block_count(void); diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 1a4ef9e203..0651600fe3 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,7 +57,7 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '181', 'pg_stat_cluster_injections returns 181 rows (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-writer-cached-x-stall + cluster-pcm-restore-aba-window + cluster-pcm-grant-finalize-window + cluster-pcm-grant-finalize-deliver-invalidate + cluster-pcm-drop-prepin-window + cluster-pcm-restore-aba-force-round; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '183', 'pg_stat_cluster_injections returns 183 rows (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-writer-cached-x-stall + cluster-pcm-restore-aba-window + cluster-pcm-grant-finalize-window + cluster-pcm-grant-finalize-deliver-invalidate + cluster-pcm-drop-prepin-window + cluster-pcm-restore-aba-force-round; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -85,8 +85,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-done-drop,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-fallback-refresh-stale,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-gcs-block-yield-notify-drop,cluster-gcs-xfer-copy-drop-window,cluster-ges-master-work-queue-full,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-drop-prepin-window,cluster-pcm-grant-finalize-deliver-invalidate,cluster-pcm-grant-finalize-window,cluster-pcm-release-pre,cluster-pcm-restore-aba-force-round,cluster-pcm-restore-aba-window,cluster-pcm-writer-cached-x-stall,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '178 injection point names match the full registry (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-done-drop,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-fallback-refresh-stale,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-master-direct-fallback-storage-stale,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-stale-ship-resident,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-gcs-block-yield-notify-drop,cluster-gcs-xfer-copy-drop-window,cluster-ges-master-work-queue-full,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-drop-prepin-window,cluster-pcm-grant-finalize-deliver-invalidate,cluster-pcm-grant-finalize-window,cluster-pcm-release-pre,cluster-pcm-restore-aba-force-round,cluster-pcm-restore-aba-window,cluster-pcm-writer-cached-x-stall,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '180 injection point names match the full registry (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 079a1ca54a..5d37f2dd97 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '181', - 'all 181 injection points have a .fault_type entry (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '183', + 'all 183 injection points have a .fault_type entry (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '181', - 'all 181 injection points have a .hits entry (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '183', + 'all 183 injection points have a .hits entry (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index cdb839d37b..28f1736087 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '181', 'M1 181 injection points (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '183', 'M1 183 injection points (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '362', - 'M5 inject category has 181×2 = 362 sub-keys (.fault_type + .hits; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + ) eq '366', + 'M5 inject category has 183×2 = 366 sub-keys (.fault_type + .hits; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index ca1802e426..6e46e46968 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -65,12 +65,12 @@ sub gcs_value { $node->start; -# L1 — pg_cluster_state.gcs surface has 111 keys (round-4c fallback-scn + spec-7.2 D6 hist). +# L1 — pg_cluster_state.gcs surface has 112 keys (branch-1 +1 master-direct rescue; round-4c fallback-scn + spec-7.2 D6 hist). is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L1 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows; round-6 +1 xfer_stale_deny; ownership-gen ruling② +2 invalidate_busy_sent/received) (spec-7.2 D6)'); + '112', + 'L1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows; round-6 +1 xfer_stale_deny; ownership-gen ruling② +2 invalidate_busy_sent/received) (spec-7.2 D6)'); # L2 — api_state = "active" after postmaster phase 1 init. diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index 9d70ee1fd1..54e13d39cb 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -8,7 +8,7 @@ # # L1 ClusterPair startup — both postmasters healthy + tier1 connected # L2 fresh baseline gcs counters on both nodes (block_* counters = 0) -# L3 pg_cluster_state.gcs category has 111 keys (spec-7.2 D6+flip) (cumulative through +# L3 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) (cumulative through # spec-6.13 direct-land observability) # L4 4 NEW wait events registered in pg_stat_cluster_wait_events: # ClusterGCSBlockShipWait, ClusterGCSBlockRequestDispatch, @@ -114,19 +114,19 @@ sub gcs_int_value { # ============================================================ -# L3: pg_cluster_state.gcs category has 111 keys (spec-7.2 D6+flip) +# L3: pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) # (cumulative GCS surface through spec-6.13 direct-land observability). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node0 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '112', + 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node1 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '112', + 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 8feb55eb21..3f4f462fd9 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 9 NEW reliability counters all 0 on both nodes -# L3 pg_cluster_state.gcs has 111 keys (cumulative through spec-7.2 D6 hist) +# L3 pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; cumulative through spec-7.2 D6 hist) # L4 2 NEW wait events registered (ClusterGCSBlockRetransmitWait + # ClusterGCSBlockEpochStaleRetry) # L5 CLUSTER_WAIT_EVENTS_COUNT = 120 (spec-7.2 +2) @@ -111,18 +111,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs category has 111 keys (cumulative through spec-7.2 D6 hist). +# L3: pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; cumulative through spec-7.2 D6 hist). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node0 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); + '112', + 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node1 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); + '112', + 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); # ============================================================ diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index eac1a63d71..e18093b8b0 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 7 NEW counters all 0 + catversion >= 202605420 -# L3 pg_cluster_state.gcs category has 111 keys (spec-7.2 D6+flip) (cumulative through spec-6.13) +# L3 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) (cumulative through spec-6.13) # L4 cross-node forward path: node A read first → master_holder = A; # force same tag on node B via test-only injection → master # chooses forward path → A direct-ships to B → block_forward_sent @@ -109,18 +109,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 111 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node0 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '112', + 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node1 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '112', + 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index e8fce4eb01..3c494a3f01 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -12,7 +12,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 6 NEW spec-2.36 counters all 0 -# L3 pg_cluster_state.gcs has 111 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) +# L3 pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) # L4 catversion lower-bound >= 202605430; wait event count == 123 # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader @@ -107,18 +107,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 111 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node0 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '112', + 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L3 node1 pg_cluster_state.gcs category has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '112', + 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index 37a9148e1d..a08eaf6945 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -130,8 +130,8 @@ sub gcs_int is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - "L4 node$i pg_cluster_state.gcs has 111 keys (round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)"); + '112', + "L4 node$i pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)"); } diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index 4c532ccbfb..b1223f72f1 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -20,7 +20,7 @@ # L7 SQLSTATE 53R93 ERRCODE_CLUSTER_LOST_WRITE_DETECTED literal- # encodable in PG SQL (catalog 形式 verification) # L8 GUC switch back to 'error' SHOW returns 'error' -# L9 pg_cluster_state.gcs category has 111 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a) +# L9 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) (cumulative through spec-6.14a) # L10 Reply status enum value 12 (DENIED_LOST_WRITE) is新增的 # 最大 value (baseline workload must not trigger lost-write) # L11 spec-2.41 D / P1-C — behavioral lost-write inject: a @@ -108,8 +108,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L2 pg_cluster_state.gcs category has 111 keys (gcs-race-fix-2 +6 rows) (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); + '112', + 'L2 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; gcs-race-fix-2 +6 rows) (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); # ============================================================ @@ -199,8 +199,8 @@ sub gcs_int is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '111', - 'L9 node1 pg_cluster_state.gcs has 111 keys (gcs-race-fix-2 +6 rows) (cross-node parity)'); + '112', + 'L9 node1 pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; gcs-race-fix-2 +6 rows) (cross-node parity)'); # ============================================================ diff --git a/src/test/cluster_tap/t/401_gcs_master_direct_storage_fallback_2node.pl b/src/test/cluster_tap/t/401_gcs_master_direct_storage_fallback_2node.pl new file mode 100644 index 0000000000..5291a433d3 --- /dev/null +++ b/src/test/cluster_tap/t/401_gcs_master_direct_storage_fallback_2node.pl @@ -0,0 +1,404 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 401_gcs_master_direct_storage_fallback_2node.pl +# branch-1 (S3 step-2 forensics) — master-direct STALE ship rescue. +# +# The S3 step-2 stop-capture proved the "branch 1" verdict: a master +# that retained a stale resident copy (a kept Past Image) served it as +# the master-direct grant payload; the SCN self-check fail-closed +# DENIED_LOST_WRITE and every hit cost the requester a 53R93 client +# abort — even though shared storage already held a version covering +# the watermark. The rescue: when the master-direct self-check says +# STALE, probe the shared-storage page pd_block_scn; if it covers the +# authoritative pi_watermark_scn, convert the reply to +# GRANTED_STORAGE_FALLBACK (ship no image; page_lsn carries the +# watermark — the state=N grant contract) so the requester +# proves/refreshes its copy via cluster_gcs_block_fallback_verify_ +# refresh. Storage unprovable → keep DENIED_LOST_WRITE (Rule 8.A). +# +# L1 ClusterPair startup baseline (both postmasters healthy) +# L2 NEW counter lost_write_master_direct_storage_fallback_count +# present (= 0) on both nodes; gcs category has 112 keys +# L3 GREEN leg: warmed cross-node heap block + CHECKPOINT (storage +# current), then cluster-gcs-block-stale-ship-resident forces a +# STALE master-direct ship → rescue converts it to +# GRANTED_STORAGE_FALLBACK: the NEW counter grows, the producer +# LOG carries BOTH the STALE verdict on the master-direct ship +# and the rescue line (path-fire proof: the inject exists ONLY at +# the master-direct self-check), and the write completes. +# L4 fail-closed leg: same STALE ship but cluster-gcs-block-master- +# direct-fallback-storage-stale forces the storage probe to +# InvalidScn → rescue REFUSED → DENIED_LOST_WRITE → requester +# 53R93; lost_write_detected_count grows; the rescue counter +# does NOT grow. +# L5 disarm + restart: cluster stays usable (no corruption). +# +# Spec: spec-2.41-cross-node-block-version-scn-lost-write.md §2.6 + +# S3 step-2 forensics bundle (RACvsRAC bench/results/ +# s3-step2-forensics-20260714-112856) branch-1 verdict. +# +# Author: SqlRush +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + + +sub gcs_int +{ + my ($node, $key) = @_; + + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state + WHERE category='gcs' AND key='$key'}); + return defined($v) && $v ne '' ? int($v) : 0; +} + + +# ============================================================ +# L1: ClusterPair startup (same true-2-node harness as t/116). +# ============================================================ +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'gcs_md_storage_fallback', + quorum_voting_disks => 3, + shared_data => 1, + data_port_span => 2, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 1024', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 10', + # The master-direct GRANTED serve for a remote READ against a + # master-local X-held quiescent block runs through the spec-6.12a + # X->S self-downgrade (scache_downgraded_fall_through); without it + # the read takes the one-shot READ_IMAGE path and the master-direct + # self-check never fires in a 2-node pair. + 'cluster.read_scache = on', + ]); +$pair->start_pair; + +usleep(3_000_000); + +is($pair->node0->safe_psql('postgres', 'SELECT 1'), + '1', 'L1 node0 postmaster alive'); +is($pair->node1->safe_psql('postgres', 'SELECT 1'), + '1', 'L1 node1 postmaster alive'); + + +# ============================================================ +# L2: NEW counter surface + gcs key count. +# ============================================================ +for my $node ($pair->node0, $pair->node1) +{ + my $name = $node->name; + + is($node->safe_psql('postgres', + q{SELECT count(*) FROM pg_cluster_state + WHERE category='gcs' + AND key='lost_write_master_direct_storage_fallback_count'}), + '1', "L2 $name NEW rescue counter key present"); + is(gcs_int($node, 'lost_write_master_direct_storage_fallback_count'), 0, + "L2 $name rescue counter = 0 at startup"); +} + +is($pair->node0->safe_psql( + 'postgres', + q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), + '112', + 'L2 pg_cluster_state.gcs category has 112 keys (branch-1 +1 rescue counter)'); + + +# ============================================================ +# Shared driving machinery (t/116 pattern: mastership is fixed by tag +# hash and unknowable a priori — drive both directions each round and +# assert on the SUM of both nodes' counters). +# ============================================================ + +# Readiness for globalized DDL. +ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'peers connected (voting quorum ready for globalized DDL)'); + +# Identical DDL order on both fresh nodes → identical relfilenode → the +# heap block routes to the same shared-storage file (t/116 seq pattern). +$pair->node0->safe_psql('postgres', 'CREATE TABLE lw_b1 (v int)'); +$pair->node1->safe_psql('postgres', 'CREATE TABLE lw_b1 (v int)'); +$pair->node0->safe_psql('postgres', 'INSERT INTO lw_b1 VALUES (0)'); + +# Retrying SQL: a fresh 2-node TAP cluster runs at LOW xids, so a +# cross-node scan of a not-yet-hinted foreign tuple can transiently hit +# "cluster TT slot recycled" (retryable per its own errhint; a TAP-only +# low-xid artifact, see the serve-stall lane). The own-xid read after +# every write below (t/394 hint recipe) makes this rare; retry bounds it. +sub retry_safe_sql +{ + my ($node, $sql) = @_; + my ($rc, $out, $err); + for my $attempt (1 .. 5) + { + ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 30); + return $out if $rc == 0; + last unless defined $err && $err =~ /TT slot recycled/i; + usleep(200_000); + } + die "retry_safe_sql exhausted on '$sql': " . ($err // 'unknown'); +} + +# Warm up (NO inject): cross-node UPDATE ping-pong installs the heap page +# on each node with a valid pd_block_scn and advances the master's +# pi_watermark_scn (grant_x / take_x_after_transfer feeds), so the forced +# (watermark-1) shipped page trips the STALE branch rather than SKIP. +# The writer VACUUMs its own table right after every write (t/394 +# ITL-quiesce recipe): own-xid ITL cleanout + dead-version prune, so the +# peer's next scan never has to resolve a foreign ITL xid through the TT +# (a fresh low-xid TAP cluster hits "TT slot recycled" otherwise). +sub write_and_quiesce +{ + my ($node) = @_; + retry_safe_sql($node, 'UPDATE lw_b1 SET v = v + 1'); + retry_safe_sql($node, 'VACUUM lw_b1'); + return; +} +$pair->node0->safe_psql('postgres', 'VACUUM lw_b1'); +for (1 .. 4) +{ + write_and_quiesce($pair->node0); + write_and_quiesce($pair->node1); +} + +sub arm_points +{ + my ($val) = @_; + for my $node ($pair->node0, $pair->node1) + { + $node->safe_psql('postgres', + "ALTER SYSTEM SET cluster.injection_points = '$val'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + } + return; +} + +sub checkpoint_both +{ + # Flush every dirty block so the shared-storage version covers the + # watermark for whatever block ships next (the rescue precondition). + $pair->node0->safe_psql('postgres', 'CHECKPOINT'); + $pair->node1->safe_psql('postgres', 'CHECKPOINT'); + return; +} + +my $saw_53r93 = 0; +my $err_53r93 = ''; +sub note_53r93 +{ + my ($err) = @_; + if (defined $err && $err =~ /53R93|lost write detected|stale storage-fallback/i) + { + $saw_53r93 = 1; + $err_53r93 = $err; + } + return; +} + +# Writer leg: take X via UPDATE, then quiesce (commit + ITL cleanout) so +# the block is servable; both statements tolerant. +sub try_write +{ + my ($node) = @_; + my ($rc, $out, $err) = + $node->psql('postgres', 'UPDATE lw_b1 SET v = v + 1', timeout => 30); + note_53r93($err); + ($rc, $out, $err) = $node->psql('postgres', 'VACUUM lw_b1', timeout => 30); + note_53r93($err); + return; +} + +# Reader leg: a remote N->S read against the writer's X-held quiescent +# block — the master-direct GRANTED trigger (X->S self-downgrade). +sub try_read +{ + my ($node) = @_; + my ($rc, $out, $err) = + $node->psql('postgres', 'SELECT count(*) FROM lw_b1', timeout => 30); + note_53r93($err); + return; +} + +sub sum_counter +{ + my ($key) = @_; + return gcs_int($pair->node0, $key) + gcs_int($pair->node1, $key); +} + + +# ============================================================ +# L3: GREEN leg — STALE master-direct ship rescued via storage. +# ============================================================ +my $resc_before = sum_counter('lost_write_master_direct_storage_fallback_count'); +my $det_l3_before = sum_counter('lost_write_detected_count'); + +# Mastership is hash-fixed and unknowable a priori: drive BOTH +# writer/reader assignments each round — only the direction where the +# WRITER masters the tag serves master-direct (the other direction takes +# the holder-forward read). Writer takes X + quiesces + CHECKPOINT +# (storage current — the rescue precondition), then the peer's remote +# read triggers the X->S downgrade GRANTED with the forced-stale ship. +my $resc_after = $resc_before; +for my $round (1 .. 8) +{ + last if $resc_after > $resc_before; + try_write($pair->node0); + checkpoint_both(); + arm_points('cluster-gcs-block-stale-ship-resident:skip'); + try_read($pair->node1); + try_write($pair->node1); + checkpoint_both(); + arm_points('cluster-gcs-block-stale-ship-resident:skip'); + try_read($pair->node0); + usleep(500_000); + $resc_after = sum_counter('lost_write_master_direct_storage_fallback_count'); +} + +diag("L3 rescue counter $resc_before->$resc_after; saw_53r93=$saw_53r93"); + +# Path diagnosis: which serve path did the rounds take, and did the +# resident inject site ever fire? +for my $node ($pair->node0, $pair->node1) +{ + my $inj = $node->safe_psql('postgres', + q{SELECT key || '=' || value FROM pg_cluster_state + WHERE category='inject' AND key LIKE '%stale-ship%' + ORDER BY key}); + $inj =~ s/\n/ /g; + diag($node->name . " inject: $inj"); + my $gcs = $node->safe_psql('postgres', + q{SELECT key || '=' || value FROM pg_cluster_state + WHERE category='gcs' + AND (key LIKE '%lost_write%' OR key LIKE '%fallback%' + OR key LIKE '%read_image%' OR key LIKE '%ship%' + OR key LIKE '%grant%') + AND value <> '0' + ORDER BY key}); + $gcs =~ s/\n/ /g; + diag($node->name . " gcs nonzero: $gcs"); +} + +cmp_ok($resc_after, '>', $resc_before, + 'L3 STALE master-direct ship rescued: ' + . "lost_write_master_direct_storage_fallback_count grew ($resc_before → $resc_after)"); + +# Path-fire proof (t/398 lesson): the producer LOG must show the STALE +# verdict ON THE MASTER-DIRECT SHIP — the inject exists only at that +# self-check, so a green here cannot come from holder-forward. +my $stale_line = ''; +my $rescue_line = ''; +for my $node ($pair->node0, $pair->node1) +{ + my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile); + $stale_line = $1 + if $stale_line eq '' + && $log =~ /(cluster_gcs_block: lost-write verdict STALE on master-direct ship[^\n]*)/; + $rescue_line = $1 + if $rescue_line eq '' + && $log =~ /(cluster_gcs_block: master-direct stale ship rescued[^\n]*)/; +} +isnt($stale_line, '', + 'L3b producer LOG shows the STALE verdict on the master-direct ship (path fired)'); +isnt($rescue_line, '', + 'L3c producer LOG shows the GRANTED_STORAGE_FALLBACK rescue line'); +like($rescue_line, qr/storage pd_block_scn=\d+/, + 'L3c2 rescue LOG carries the storage-probe pd_block_scn') + if $rescue_line ne ''; + +# The rescue leg must not have produced fail-closed denials of its own +# beyond what later legs will drive; record for the L4 delta instead. +my $det_l3_after = sum_counter('lost_write_detected_count'); +diag("L3 lost_write_detected_count $det_l3_before->$det_l3_after"); + +# Cluster stays usable right after the rescue (write completes cleanly). +arm_points(''); +is(retry_safe_sql($pair->node0, 'UPDATE lw_b1 SET v = v + 1 RETURNING v > 0'), + 't', 'L3d write path stays usable after the rescue'); + +# Hard gates for the GREEN-leg claims (branch-1 review P1-3): the rescued +# leg must be 53R93-free, and both nodes must agree on the data end value +# — a rescue that let the requester consume a stale storage version would +# diverge the two readbacks. +is($saw_53r93, 0, 'L3e rescued leg surfaced zero 53R93 client aborts') + or diag("53R93 err: $err_53r93"); +# Quiesce first (owner-side ITL cleanout, the t/394 recipe used by the +# warm-up above): without it the cross-node readback can trip the known +# low-xid "TT slot recycled" TAP artifact past the retry budget. +retry_safe_sql($pair->node0, 'VACUUM lw_b1'); +my $v_n0 = retry_safe_sql($pair->node0, 'SELECT v FROM lw_b1'); +my $v_n1 = retry_safe_sql($pair->node1, 'SELECT v FROM lw_b1'); +is($v_n1, $v_n0, "L3f cross-node readback agrees after rescue (v=$v_n0)"); +cmp_ok($v_n0, '>=', 9, + 'L3g end value covers the 8 warm-up increments plus L3d'); + + +# ============================================================ +# L4: fail-closed leg — storage probe unprovable → DENIED_LOST_WRITE. +# ============================================================ +my $det_before = sum_counter('lost_write_detected_count'); +my $resc_l4_before = sum_counter('lost_write_master_direct_storage_fallback_count'); +$saw_53r93 = 0; +$err_53r93 = ''; + +my $det_after = $det_before; +for my $round (1 .. 8) +{ + last if $det_after > $det_before; + try_write($pair->node0); + checkpoint_both(); + arm_points('cluster-gcs-block-stale-ship-resident:skip,' + . 'cluster-gcs-block-master-direct-fallback-storage-stale:skip'); + try_read($pair->node1); + try_write($pair->node1); + checkpoint_both(); + arm_points('cluster-gcs-block-stale-ship-resident:skip,' + . 'cluster-gcs-block-master-direct-fallback-storage-stale:skip'); + try_read($pair->node0); + usleep(500_000); + $det_after = sum_counter('lost_write_detected_count'); +} + +diag("L4 detected $det_before->$det_after; saw_53r93=$saw_53r93"); + +cmp_ok($det_after, '>', $det_before, + 'L4 storage-unprovable rescue refused: lost_write_detected_count grew ' + . "($det_before → $det_after) — fail-closed retained"); +ok($saw_53r93, + 'L4b requester saw 53R93 / lost-write (rescue must not mask fail-closed)'); +is(sum_counter('lost_write_master_direct_storage_fallback_count'), $resc_l4_before, + 'L4c rescue counter did NOT grow while storage was unprovable'); + + +# ============================================================ +# L5: disarm + restart, cluster stays usable. +# ============================================================ +arm_points(''); +$pair->node0->stop('fast'); +$pair->node1->stop('fast'); +$pair->node0->start; +$pair->node1->start; +usleep(3_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'L5 cluster restarted injection-free after both legs'); +is(retry_safe_sql($pair->node0, 'SELECT count(*) FROM lw_b1'), + '1', 'L5 table still readable after both legs (no corruption)'); + + +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 73f80b2ba6..3032aa5bd5 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1482,6 +1482,12 @@ cluster_gcs_get_lost_write_not_scn_tracked_skip_count(void) { return 0; } +/* branch-1 stub: master-direct storage-fallback rescue accessor. */ +uint64 +cluster_gcs_get_lost_write_master_direct_storage_fallback_count(void) +{ + return 0; +} uint64 cluster_gcs_get_redo_coverage_required_lsn_zero_count(void) {