From b4c3da00d43d0ddce7b860a96102c3916640e04b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 22:34:14 +0000 Subject: [PATCH 01/12] =?UTF-8?q?fix(config,spac):=20close=202=20Blockers?= =?UTF-8?q?=20=E2=80=94=20db=20reset=20table=20coverage=20+=20SPAC=20write?= =?UTF-8?q?=20race?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit db reset: resetAllDatabases() truncated only 45 of 75 registered repositories, silently leaving 30 tables (all SPAC, XBRL, observation provenance, section16, form144, offering, dead-letter, family-tier, etc.) fully populated after a "reset" — producing a corrupt mix of stale derived rows whose referenced canonical entities had been truncated. Add the 30 missing deleteAll() calls and a source-parity test (resetAllDatabases.test.ts) that fails if a DefaultDI repo token is ever added without a matching reset, mirroring form-wiring.test.ts. SPAC write race: every SpacReportWriter.record* method is an unsynchronised read-derive-write over a CIK's spac/spac_deal/spac_history rows, but the form tasks map over filings with concurrencyLimit > 1, so two same-CIK filings interleave their recompute+rebuild+snapshot cycles — lost-updating the derived row and forking the history chain (two open rows). Add a module-scoped, refcounted per-CIK AsyncMutex (writers are constructed fresh per call, so an instance field would not serialise across them) wrapping all five record* methods, mirroring the resolver per-key mutex. Adds a regression test that asserts the critical sections never overlap. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/config/resetAllDatabases.test.ts | 33 +++++ src/config/resetAllDatabases.ts | 70 ++++++++++ src/storage/spac/SpacReportWriter.test.ts | 64 +++++++++ src/storage/spac/SpacReportWriter.ts | 155 ++++++++++++++-------- 4 files changed, 268 insertions(+), 54 deletions(-) create mode 100644 src/config/resetAllDatabases.test.ts diff --git a/src/config/resetAllDatabases.test.ts b/src/config/resetAllDatabases.test.ts new file mode 100644 index 00000000..d0631b94 --- /dev/null +++ b/src/config/resetAllDatabases.test.ts @@ -0,0 +1,33 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from "bun:test"; +import { readFileSync } from "node:fs"; + +/** + * Drift guard for `sec db reset --confirm`. `resetAllDatabases()` truncates a + * hand-maintained list of repository tokens; if a new table is registered in + * DefaultDI but its `deleteAll()` is forgotten here, a "reset" silently leaves + * that table fully populated (orphan rows + dangling cross-tier references). + * + * This mirrors `form-wiring.test.ts`: it pins the data/wiring consistency at the + * source level so a forgotten table fails CI instead of corrupting a "clean" DB. + */ +function repositoryTokens(relativePath: string): Set { + const source = readFileSync(new URL(relativePath, import.meta.url), "utf8"); + const tokens = source.match(/[A-Z0-9_]+_REPOSITORY_TOKEN/g) ?? []; + return new Set(tokens); +} + +describe("resetAllDatabases token coverage", () => { + it("truncates every repository token registered in DefaultDI", () => { + const registered = repositoryTokens("./DefaultDI.ts"); + const reset = repositoryTokens("./resetAllDatabases.ts"); + + const missing = [...registered].filter((token) => !reset.has(token)).sort(); + expect(missing).toEqual([]); + }); +}); diff --git a/src/config/resetAllDatabases.ts b/src/config/resetAllDatabases.ts index 8f9ddcd8..8009c988 100644 --- a/src/config/resetAllDatabases.ts +++ b/src/config/resetAllDatabases.ts @@ -60,6 +60,42 @@ import { PERSON_OBSERVATION_REPOSITORY_TOKEN } from "../storage/observation/Pers import { COMPONENT_VERSION_REPOSITORY_TOKEN } from "../storage/versioning/ComponentVersionSchema"; import { EXTRACTOR_RUN_REPOSITORY_TOKEN } from "../storage/versioning/ExtractorRunSchema"; import { VERSION_EVENT_REPOSITORY_TOKEN } from "../storage/versioning/VersionEventSchema"; +import { BENEFICIAL_OWNERSHIP_REPOSITORY_TOKEN } from "../storage/beneficial-ownership/BeneficialOwnershipSchema"; +import { + CANONICAL_SPONSOR_FAMILY_ALIAS_REPOSITORY_TOKEN, + CANONICAL_UNDERWRITER_FAMILY_ALIAS_REPOSITORY_TOKEN, +} from "../storage/canonical/CanonicalAliasSchemas"; +import { CANONICAL_SPONSOR_FAMILY_REPOSITORY_TOKEN } from "../storage/canonical/CanonicalSponsorFamilySchema"; +import { CANONICAL_UNDERWRITER_FAMILY_REPOSITORY_TOKEN } from "../storage/canonical/CanonicalUnderwriterFamilySchema"; +import { SPAC_SPONSOR_LINK_REPOSITORY_TOKEN } from "../storage/canonical/SpacSponsorLinkSchema"; +import { SPONSOR_FAMILY_MEMBERSHIP_REPOSITORY_TOKEN } from "../storage/canonical/SponsorFamilyMembershipSchema"; +import { UNDERWRITER_FAMILY_MEMBERSHIP_REPOSITORY_TOKEN } from "../storage/canonical/UnderwriterFamilyMembershipSchema"; +import { UNDERWRITER_LINK_REPOSITORY_TOKEN } from "../storage/canonical/UnderwriterLinkSchema"; +import { S1_CLASSIFICATION_REPOSITORY_TOKEN } from "../storage/classification/S1ClassificationSchema"; +import { EXTRACTION_DEAD_LETTER_REPOSITORY_TOKEN } from "../storage/dead-letter/ExtractionDeadLetterSchema"; +import { + FORM144_ACQUISITION_REPOSITORY_TOKEN, + FORM144_FILING_REPOSITORY_TOKEN, + FORM144_RECENT_SALE_REPOSITORY_TOKEN, +} from "../storage/form144/Form144Schema"; +import { ISSUER_TICKER_REPOSITORY_TOKEN } from "../storage/offering/IssuerTickerSchema"; +import { OFFERING_TERMS_REPOSITORY_TOKEN } from "../storage/offering/OfferingTermsSchema"; +import { SPAC_UNIT_TERMS_REPOSITORY_TOKEN } from "../storage/offering/SpacUnitTermsSchema"; +import { OBSERVATION_PROVENANCE_REPOSITORY_TOKEN } from "../storage/provenance/ObservationProvenanceSchema"; +import { RELATED_PARTY_TRANSACTION_REPOSITORY_TOKEN } from "../storage/related-party/RelatedPartyTransactionSchema"; +import { + SECTION16_FILING_REPOSITORY_TOKEN, + SECTION16_HOLDING_REPOSITORY_TOKEN, + SECTION16_TRANSACTION_REPOSITORY_TOKEN, +} from "../storage/section16/Section16Schema"; +import { SPAC_DEAL_REPOSITORY_TOKEN } from "../storage/spac/SpacDealSchema"; +import { SPAC_EVENT_REPOSITORY_TOKEN } from "../storage/spac/SpacEventSchema"; +import { SPAC_HISTORY_REPOSITORY_TOKEN } from "../storage/spac/SpacHistorySchema"; +import { SPAC_MERGER_EXTRACTION_REPOSITORY_TOKEN } from "../storage/spac/SpacMergerExtractionSchema"; +import { SPAC_REDEMPTION_EXTRACTION_REPOSITORY_TOKEN } from "../storage/spac/SpacRedemptionExtractionSchema"; +import { SPAC_REPOSITORY_TOKEN } from "../storage/spac/SpacSchema"; +import { USE_OF_PROCEEDS_REPOSITORY_TOKEN } from "../storage/use-of-proceeds/UseOfProceedsSchema"; +import { XBRL_FACT_REPOSITORY_TOKEN } from "../storage/xbrl/XbrlFactSchema"; /** * Truncates every registered repository. Used by `sec db reset --confirm` @@ -114,4 +150,38 @@ export async function resetAllDatabases(): Promise { await globalServiceRegistry.get(CANONICAL_PERSON_ALIAS_REPOSITORY_TOKEN).deleteAll(); await globalServiceRegistry.get(CANONICAL_COMPANY_ALIAS_REPOSITORY_TOKEN).deleteAll(); await globalServiceRegistry.get(FORM_8K_EVENT_REPOSITORY_TOKEN).deleteAll(); + // Observation provenance + AI-extracted offering / ownership / related-party tiers. + await globalServiceRegistry.get(OBSERVATION_PROVENANCE_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(BENEFICIAL_OWNERSHIP_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(RELATED_PARTY_TRANSACTION_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(EXTRACTION_DEAD_LETTER_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(S1_CLASSIFICATION_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(ISSUER_TICKER_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(OFFERING_TERMS_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPAC_UNIT_TERMS_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(USE_OF_PROCEEDS_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(XBRL_FACT_REPOSITORY_TOKEN).deleteAll(); + // SPAC lifecycle: derived `spac` row + append-only deal/event/extraction tables. + await globalServiceRegistry.get(SPAC_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPAC_DEAL_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPAC_EVENT_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPAC_HISTORY_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPAC_MERGER_EXTRACTION_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPAC_REDEMPTION_EXTRACTION_REPOSITORY_TOKEN).deleteAll(); + // Section 16 (Forms 3/4/5) and Form 144 detail tables. + await globalServiceRegistry.get(SECTION16_FILING_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SECTION16_TRANSACTION_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SECTION16_HOLDING_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(FORM144_FILING_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(FORM144_ACQUISITION_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(FORM144_RECENT_SALE_REPOSITORY_TOKEN).deleteAll(); + // Family-tier canonical / alias / membership / link tables (sponsor + underwriter). + await globalServiceRegistry.get(CANONICAL_SPONSOR_FAMILY_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(CANONICAL_SPONSOR_FAMILY_ALIAS_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPONSOR_FAMILY_MEMBERSHIP_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(CANONICAL_UNDERWRITER_FAMILY_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(CANONICAL_UNDERWRITER_FAMILY_ALIAS_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(UNDERWRITER_FAMILY_MEMBERSHIP_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(UNDERWRITER_LINK_REPOSITORY_TOKEN).deleteAll(); + await globalServiceRegistry.get(SPAC_SPONSOR_LINK_REPOSITORY_TOKEN).deleteAll(); } diff --git a/src/storage/spac/SpacReportWriter.test.ts b/src/storage/spac/SpacReportWriter.test.ts index 5714c994..aa24c680 100644 --- a/src/storage/spac/SpacReportWriter.test.ts +++ b/src/storage/spac/SpacReportWriter.test.ts @@ -11,6 +11,7 @@ import { setupAllDatabases } from "../../config/setupAllDatabases"; import { SpacRepo } from "./SpacRepo"; import { SpacReportWriter } from "./SpacReportWriter"; import { SpacMergerExtractionRepo } from "./SpacMergerExtractionRepo"; +import type { SpacHistory } from "./SpacHistorySchema"; import { CHANGE_LOG_REPOSITORY_TOKEN } from "../change-tracking/ChangeLogSchema"; describe("SpacReportWriter", () => { @@ -328,4 +329,67 @@ describe("SpacReportWriter", () => { expect(row?.target_name).toBe("Acme Target Inc."); // still correlated expect(row?.status).toBe("deal_announced"); // no proxy event -> not "proxy" }); + + it("serialises concurrent same-CIK writes (no interleaved read-derive-write)", async () => { + // Two filings for the same SPAC processed concurrently (as the form tasks + // do with concurrencyLimit > 1). Each record* method is an unsynchronised + // read-derive-write over the CIK's rows; if two overlap they lost-update the + // derived row / fork the history chain. The repo here makes its writes + // observably slow and tracks how many record* critical sections are in + // flight at once: the per-CIK lock must keep that at 1. + let inFlight = 0; + let maxInFlight = 0; + class InstrumentedSpacRepo extends SpacRepo { + async saveHistory(history: SpacHistory): Promise { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + try { + await new Promise((resolve) => setTimeout(resolve, 10)); + return await super.saveHistory(history); + } finally { + inFlight -= 1; + } + } + } + const slowRepo = new InstrumentedSpacRepo(); + // Distinct writer instances sharing the storage — the lock map is + // module-scoped, so it must serialise across instances, not just within one. + const w1 = new SpacReportWriter(slowRepo); + const w2 = new SpacReportWriter(slowRepo); + + await Promise.all([ + w1.recordRegistration({ + cik: 99, + accession_number: "99-reg", + filing_date: "2020-12-01", + form: "S-1", + primary_document: "s1.htm", + spac_name: "Race SPAC", + spac_sic: 6770, + }), + w2.recordIpo({ + cik: 99, + accession_number: "99-ipo", + filing_date: "2021-01-15", + form: "424B4", + primary_document: "424.htm", + ipo_proceeds: 150_000_000, + trust_amount: 150_000_000, + spac_tickers: ["RACE.U"], + }), + ]); + + // The two critical sections never overlapped. + expect(maxInFlight).toBe(1); + // Exactly one open history row, and the derived row reflects both events. + const history = await repo.getHistory(99); + expect(history.filter((h) => h.valid_to == null).length).toBe(1); + const events = await repo.getEvents(99); + expect(events.some((e) => e.event_type === "registration")).toBe(true); + expect(events.some((e) => e.event_type === "ipo")).toBe(true); + const row = await repo.getSpac(99); + expect(row?.registration_date).toBe("2020-12-01"); + expect(row?.ipo_date).toBe("2021-01-15"); + expect(row?.ipo_proceeds).toBe(150_000_000); + }); }); diff --git a/src/storage/spac/SpacReportWriter.ts b/src/storage/spac/SpacReportWriter.ts index 50ff32c6..dbf25b3b 100644 --- a/src/storage/spac/SpacReportWriter.ts +++ b/src/storage/spac/SpacReportWriter.ts @@ -14,6 +14,42 @@ import type { Spac } from "./SpacSchema"; import type { SpacEvent, SpacEventType } from "./SpacEventSchema"; import type { SpacHistory } from "./SpacHistorySchema"; import { CHANGE_LOG_REPOSITORY_TOKEN } from "../change-tracking/ChangeLogSchema"; +import { AsyncMutex } from "../../util/AsyncMutex"; + +/** + * Per-CIK write serialisation. Each public `record*` method runs a + * read-derive-write cycle over the CIK's spac / spac_deal / spac_history rows + * (append event → recompute deals → rebuild row → snapshot history). Two + * filings for the SAME CIK can be processed concurrently — the form tasks map + * over filings with `concurrencyLimit > 1` — so without a lock their cycles + * interleave across `await` boundaries and either lost-update the derived row + * or fork the history chain (two open rows). + * + * The map is module-scoped because every caller constructs a fresh + * `SpacReportWriter` (an instance field would not serialise across writers), + * and is refcounted / evicted at zero to stay bounded — mirroring the resolver + * per-key mutex. Single-process only: multi-process callers still rely on the + * append-only event PK `(cik, accession_number, event_type)` for idempotency. + */ +const cikWriteMutexes = new Map(); + +function withCikLock(cik: number, fn: () => Promise): Promise { + let entry = cikWriteMutexes.get(cik); + if (entry === undefined) { + entry = { mutex: new AsyncMutex(), refs: 0 }; + cikWriteMutexes.set(cik, entry); + } + entry.refs += 1; + const held = entry; + return held.mutex.lock(fn).finally(() => { + held.refs -= 1; + // Same-identity check guards against a caller recreating the entry between + // the decrement and the delete. + if (held.refs === 0 && cikWriteMutexes.get(cik) === held) { + cikWriteMutexes.delete(cik); + } + }); +} interface RecordRegistrationArgs { readonly cik: number; @@ -80,59 +116,66 @@ export class SpacReportWriter { } async recordRegistration(args: RecordRegistrationArgs): Promise { - await this.appendEvent({ - cik: args.cik, - accession_number: args.accession_number, - event_type: "registration", - event_date: args.filing_date, - form: args.form, - primary_document: args.primary_document, - }); - await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, { - spac_name: args.spac_name, - spac_sic: args.spac_sic, + await withCikLock(args.cik, async () => { + await this.appendEvent({ + cik: args.cik, + accession_number: args.accession_number, + event_type: "registration", + event_date: args.filing_date, + form: args.form, + primary_document: args.primary_document, + }); + await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, { + spac_name: args.spac_name, + spac_sic: args.spac_sic, + }); }); } async recordIpo(args: RecordIpoArgs): Promise { - await this.appendEvent({ - cik: args.cik, - accession_number: args.accession_number, - event_type: "ipo", - event_date: args.filing_date, - form: args.form, - primary_document: args.primary_document, - amount: args.ipo_proceeds, - }); - await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, { - ipo_proceeds: args.ipo_proceeds, - trust_amount: args.trust_amount, - spac_tickers: - args.spac_tickers && args.spac_tickers.length > 0 - ? JSON.stringify(args.spac_tickers) - : null, + await withCikLock(args.cik, async () => { + await this.appendEvent({ + cik: args.cik, + accession_number: args.accession_number, + event_type: "ipo", + event_date: args.filing_date, + form: args.form, + primary_document: args.primary_document, + amount: args.ipo_proceeds, + }); + await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, { + ipo_proceeds: args.ipo_proceeds, + trust_amount: args.trust_amount, + spac_tickers: + args.spac_tickers && args.spac_tickers.length > 0 + ? JSON.stringify(args.spac_tickers) + : null, + }); }); } /** * Record de-SPAC milestone events mapped from 8-K item codes: append each * event (idempotent by PK), recompute the deal set from the full event - * stream (merge-preserving §4b-owned columns), then rebuild the row. + * stream + stored extractions (target/pipe/redemption derived by + * correlation, not merge-preserved), then rebuild the row. */ async recordDealMilestones(args: RecordDealMilestonesArgs): Promise { if (args.events.length === 0) return; - for (const e of args.events) { - await this.appendEvent({ - cik: args.cik, - accession_number: args.accession_number, - event_type: e.event_type, - event_date: e.event_date, - form: args.form, - primary_document: args.primary_document, - }); - } - await this.recomputeAndSaveDeals(args.cik); - await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, {}); + await withCikLock(args.cik, async () => { + for (const e of args.events) { + await this.appendEvent({ + cik: args.cik, + accession_number: args.accession_number, + event_type: e.event_type, + event_date: e.event_date, + form: args.form, + primary_document: args.primary_document, + }); + } + await this.recomputeAndSaveDeals(args.cik); + await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, {}); + }); } /** @@ -143,18 +186,20 @@ export class SpacReportWriter { * this runs. */ async recordMergerProxy(args: RecordMergerProxyArgs): Promise { - if (args.emitProxyEvent) { - await this.appendEvent({ - cik: args.cik, - accession_number: args.accession_number, - event_type: "proxy", - event_date: args.filing_date, - form: args.form, - primary_document: args.primary_document, - }); - } - await this.recomputeAndSaveDeals(args.cik); - await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, {}); + await withCikLock(args.cik, async () => { + if (args.emitProxyEvent) { + await this.appendEvent({ + cik: args.cik, + accession_number: args.accession_number, + event_type: "proxy", + event_date: args.filing_date, + form: args.form, + primary_document: args.primary_document, + }); + } + await this.recomputeAndSaveDeals(args.cik); + await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, {}); + }); } /** @@ -171,8 +216,10 @@ export class SpacReportWriter { readonly filing_date: string; readonly form: string; }): Promise { - await this.recomputeAndSaveDeals(args.cik); - await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, {}); + await withCikLock(args.cik, async () => { + await this.recomputeAndSaveDeals(args.cik); + await this.rebuild(args.cik, args.filing_date, `${args.form}:${args.accession_number}`, {}); + }); } /** From e7fc9976c356b0ec984c92eefaa8e48e72be1b00 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 22:38:32 +0000 Subject: [PATCH 02/12] fix(observation): enforce UNIQUE natural key + recover from the insert race The observation natural key (accession_number, extractor_id, observation_index) was registered as a NON-unique index, so the schema's "is UNIQUE" docstring was a lie at the storage layer and upsertByNaturalKey (query-then-insert, no mutex) let two concurrent inserts for one logical mention both write a row. - Move the natural-key tuple from indexes to uniqueIndexes in DefaultDI and TestingDI (both observation tables), matching the canonical-tier wiring. - upsertByNaturalKey now recovers from a lost insert race: on a UNIQUE violation it re-reads the winning row and merges onto it, so both callers converge on one observation (mirrors the resolver's create-then-requery pattern). - Relocate isUniqueConstraintError from resolver/ to util/ (it is a pure, backend-agnostic helper) so storage can use it without a storage->resolver backward dependency; update the 6 resolver imports. - Test storage now mirrors the unique index and a new test pins the same-natural-key concurrent-insert convergence (fails under the old config). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/config/DefaultDI.ts | 11 +++- src/config/TestingDI.ts | 29 ++++++--- src/resolver/CompanyResolver.race.test.ts | 2 +- src/resolver/CompanyResolver.ts | 2 +- src/resolver/FamilyResolver.ts | 2 +- src/resolver/PersonResolver.race.test.ts | 2 +- src/resolver/PersonResolver.ts | 2 +- .../observation/CompanyObservationRepo.ts | 56 ++++++++++------- .../observation/PersonObservationRepo.test.ts | 37 ++++++++++-- .../observation/PersonObservationRepo.ts | 60 ++++++++++++------- .../isUniqueConstraintError.test.ts | 0 .../isUniqueConstraintError.ts | 0 12 files changed, 143 insertions(+), 60 deletions(-) rename src/{resolver => util}/isUniqueConstraintError.test.ts (100%) rename src/{resolver => util}/isUniqueConstraintError.ts (100%) diff --git a/src/config/DefaultDI.ts b/src/config/DefaultDI.ts index 6a2a6503..5efe0584 100644 --- a/src/config/DefaultDI.ts +++ b/src/config/DefaultDI.ts @@ -701,7 +701,12 @@ export const DefaultDI = () => { "person_observations", PersonObservationSchema, PersonObservationPrimaryKeyNames, - [["accession_number"], ["accession_number", "extractor_id", "observation_index"]] + [["accession_number"]], + // The natural key is UNIQUE (one row per mention). Enforcing it at the + // storage layer closes the find-or-insert race in upsertByNaturalKey + // (two concurrent inserts both saw no row) the same way the resolver + // relies on (resolver_version, cik) — the repo recovers by re-querying. + [["accession_number", "extractor_id", "observation_index"]] ) ); globalServiceRegistry.registerInstance( @@ -710,7 +715,9 @@ export const DefaultDI = () => { "company_observations", CompanyObservationSchema, CompanyObservationPrimaryKeyNames, - [["accession_number"], ["accession_number", "extractor_id", "observation_index"]] + [["accession_number"]], + // The natural key is UNIQUE — see person_observations above. + [["accession_number", "extractor_id", "observation_index"]] ) ); globalServiceRegistry.registerInstance( diff --git a/src/config/TestingDI.ts b/src/config/TestingDI.ts index e246c4b1..f3f6233d 100644 --- a/src/config/TestingDI.ts +++ b/src/config/TestingDI.ts @@ -644,17 +644,30 @@ export function resetDependencyInjectionsForTesting() { // Initialize Observation / Canonical / Resolver repositories globalServiceRegistry.registerInstance( PERSON_OBSERVATION_REPOSITORY_TOKEN, - new InMemoryTabularStorage(PersonObservationSchema, PersonObservationPrimaryKeyNames, [ - ["accession_number"], - ["accession_number", "extractor_id", "observation_index"], - ]) + new InMemoryTabularStorage( + PersonObservationSchema, + PersonObservationPrimaryKeyNames, + [["accession_number"]], + undefined, + undefined, + undefined, + // Natural key is UNIQUE — must mirror DefaultDI so the upsert race + // recovery (and the test that exercises it) sees the same constraint. + [["accession_number", "extractor_id", "observation_index"]] + ) ); globalServiceRegistry.registerInstance( COMPANY_OBSERVATION_REPOSITORY_TOKEN, - new InMemoryTabularStorage(CompanyObservationSchema, CompanyObservationPrimaryKeyNames, [ - ["accession_number"], - ["accession_number", "extractor_id", "observation_index"], - ]) + new InMemoryTabularStorage( + CompanyObservationSchema, + CompanyObservationPrimaryKeyNames, + [["accession_number"]], + undefined, + undefined, + undefined, + // Natural key is UNIQUE — see person_observations above. + [["accession_number", "extractor_id", "observation_index"]] + ) ); globalServiceRegistry.registerInstance( OBSERVATION_PROVENANCE_REPOSITORY_TOKEN, diff --git a/src/resolver/CompanyResolver.race.test.ts b/src/resolver/CompanyResolver.race.test.ts index b8d04f5c..5e41e67a 100644 --- a/src/resolver/CompanyResolver.race.test.ts +++ b/src/resolver/CompanyResolver.race.test.ts @@ -20,7 +20,7 @@ import { } from "../storage/canonical/CanonicalAliasSchemas"; import type { CompanyObservation } from "../storage/observation/CompanyObservationSchema"; import { CompanyResolver } from "./CompanyResolver"; -import { isUniqueConstraintError } from "./isUniqueConstraintError"; +import { isUniqueConstraintError } from "../util/isUniqueConstraintError"; type ErrorShape = "sqlite" | "pg"; type CompanyKey = "cik" | "crd"; diff --git a/src/resolver/CompanyResolver.ts b/src/resolver/CompanyResolver.ts index a2ef02f8..d1897a63 100644 --- a/src/resolver/CompanyResolver.ts +++ b/src/resolver/CompanyResolver.ts @@ -10,7 +10,7 @@ import type { CanonicalCompanyAliasRepo } from "../storage/canonical/CanonicalCo import type { CompanyObservation } from "../storage/observation/CompanyObservationSchema"; import type { CanonicalCompany } from "../storage/canonical/CanonicalCompanySchema"; import { AsyncMutex } from "../util/AsyncMutex"; -import { isUniqueConstraintError } from "./isUniqueConstraintError"; +import { isUniqueConstraintError } from "../util/isUniqueConstraintError"; interface CompanyResolverOptions { canonicalCompanyRepo: CanonicalCompanyRepo; diff --git a/src/resolver/FamilyResolver.ts b/src/resolver/FamilyResolver.ts index f2bf7318..5119ed0a 100644 --- a/src/resolver/FamilyResolver.ts +++ b/src/resolver/FamilyResolver.ts @@ -6,7 +6,7 @@ import { normalizeCompanyName } from "../storage/company/CompanyNormalization"; import { AsyncMutex } from "../util/AsyncMutex"; -import { isUniqueConstraintError } from "./isUniqueConstraintError"; +import { isUniqueConstraintError } from "../util/isUniqueConstraintError"; /** * The single source of truth for a family natural key (sponsor or underwriter). diff --git a/src/resolver/PersonResolver.race.test.ts b/src/resolver/PersonResolver.race.test.ts index 2be92c1d..40527de3 100644 --- a/src/resolver/PersonResolver.race.test.ts +++ b/src/resolver/PersonResolver.race.test.ts @@ -20,7 +20,7 @@ import { } from "../storage/canonical/CanonicalAliasSchemas"; import type { PersonObservation } from "../storage/observation/PersonObservationSchema"; import { PersonResolver } from "./PersonResolver"; -import { isUniqueConstraintError } from "./isUniqueConstraintError"; +import { isUniqueConstraintError } from "../util/isUniqueConstraintError"; type ErrorShape = "sqlite" | "pg"; diff --git a/src/resolver/PersonResolver.ts b/src/resolver/PersonResolver.ts index 10f76f91..33e74117 100644 --- a/src/resolver/PersonResolver.ts +++ b/src/resolver/PersonResolver.ts @@ -10,7 +10,7 @@ import type { CanonicalPersonAliasRepo } from "../storage/canonical/CanonicalPer import type { PersonObservation } from "../storage/observation/PersonObservationSchema"; import type { CanonicalPerson } from "../storage/canonical/CanonicalPersonSchema"; import { AsyncMutex } from "../util/AsyncMutex"; -import { isUniqueConstraintError } from "./isUniqueConstraintError"; +import { isUniqueConstraintError } from "../util/isUniqueConstraintError"; interface PersonResolverOptions { canonicalPersonRepo: CanonicalPersonRepo; diff --git a/src/storage/observation/CompanyObservationRepo.ts b/src/storage/observation/CompanyObservationRepo.ts index c9c9a4e0..09737745 100644 --- a/src/storage/observation/CompanyObservationRepo.ts +++ b/src/storage/observation/CompanyObservationRepo.ts @@ -6,6 +6,7 @@ import { globalServiceRegistry } from "workglow"; import type { SearchCriteria } from "workglow"; +import { isUniqueConstraintError } from "../../util/isUniqueConstraintError"; import { COMPANY_OBSERVATION_REPOSITORY_TOKEN, type CompanyObservation, @@ -69,27 +70,42 @@ export class CompanyObservationRepo { } async upsertByNaturalKey(draft: CompanyObservationDraft): Promise { - const matches = await this.repo.query({ - accession_number: draft.accession_number, - extractor_id: draft.extractor_id, - observation_index: draft.observation_index, - }); - const existing = matches?.[0]; - if (existing) { - const merged: CompanyObservation = { - ...existing, - ...this.applyNullDefaults(draft), - observation_id: existing.observation_id, - }; - await this.repo.put(merged); - return merged; - } - // See PersonObservationRepo for the rationale on dropping the - // process-wide mutex: the backend's auto-generated PK closes the - // TOCTOU race on its own. - return await this.repo.put( - this.applyNullDefaults(draft) as Parameters[0] + const existing = await this.getByNaturalKey( + draft.accession_number, + draft.extractor_id, + draft.observation_index ); + if (existing) return this.mergeOnto(existing, draft); + + try { + return await this.repo.put( + this.applyNullDefaults(draft) as Parameters[0] + ); + } catch (err) { + // Concurrent insert of the same natural key — the UNIQUE index rejects + // the duplicate; re-read the winner and merge. See PersonObservationRepo. + if (!isUniqueConstraintError(err)) throw err; + const winner = await this.getByNaturalKey( + draft.accession_number, + draft.extractor_id, + draft.observation_index + ); + if (!winner) throw err; + return this.mergeOnto(winner, draft); + } + } + + private async mergeOnto( + existing: CompanyObservation, + draft: CompanyObservationDraft + ): Promise { + const merged: CompanyObservation = { + ...existing, + ...this.applyNullDefaults(draft), + observation_id: existing.observation_id, + }; + await this.repo.put(merged); + return merged; } async getByNaturalKey( diff --git a/src/storage/observation/PersonObservationRepo.test.ts b/src/storage/observation/PersonObservationRepo.test.ts index ab0b739b..7d8aeb72 100644 --- a/src/storage/observation/PersonObservationRepo.test.ts +++ b/src/storage/observation/PersonObservationRepo.test.ts @@ -26,10 +26,16 @@ describe("PersonObservationRepo", () => { typeof PersonObservationSchema, typeof PersonObservationPrimaryKeyNames, PersonObservation - >(PersonObservationSchema, PersonObservationPrimaryKeyNames, [ - ["accession_number"], - ["accession_number", "extractor_id", "observation_index"], - ]); + >( + PersonObservationSchema, + PersonObservationPrimaryKeyNames, + [["accession_number"]], + undefined, + undefined, + undefined, + // Mirror DefaultDI/TestingDI: the natural key is UNIQUE. + [["accession_number", "extractor_id", "observation_index"]] + ); repo = new PersonObservationRepo({ personObservationRepository: storage }); }); @@ -175,6 +181,29 @@ describe("PersonObservationRepo", () => { expect(await storage.size()).toBe(2); }); + it("concurrent inserts on the SAME natural key converge to one row", async () => { + // Two extractions racing the same (accession, extractor_id, index): both + // query an empty store, both attempt to insert. The UNIQUE index rejects + // the loser, whose upsert recovers by re-reading and merging onto the + // winner — so exactly one row survives, not two forked observations. + const claim = { + accession_number: "0001-25-000009", + extractor_id: "D", + extractor_version: "1.0.0", + observation_index: 0, + last_name: "Race", + normalized_last: "race", + created_at: "2026-05-22T00:00:00.000Z", + }; + const [a, b] = await Promise.all([ + repo.upsertByNaturalKey(claim), + repo.upsertByNaturalKey(claim), + ]); + expect(a.observation_id).toBe(b.observation_id); + expect(await storage.size()).toBe(1); + expect(await repo.count()).toBe(1); + }); + it("concurrent update-of-existing keeps original id while parallel insert gets a new id", async () => { const seeded = await repo.upsertByNaturalKey({ accession_number: "0001-25-000001", diff --git a/src/storage/observation/PersonObservationRepo.ts b/src/storage/observation/PersonObservationRepo.ts index 352182c1..4b9a50fe 100644 --- a/src/storage/observation/PersonObservationRepo.ts +++ b/src/storage/observation/PersonObservationRepo.ts @@ -6,6 +6,7 @@ import { globalServiceRegistry } from "workglow"; import type { SearchCriteria } from "workglow"; +import { isUniqueConstraintError } from "../../util/isUniqueConstraintError"; import { PERSON_OBSERVATION_REPOSITORY_TOKEN, type PersonObservation, @@ -81,30 +82,47 @@ export class PersonObservationRepo { } async upsertByNaturalKey(draft: PersonObservationDraft): Promise { - const matches = await this.repo.query({ - accession_number: draft.accession_number, - extractor_id: draft.extractor_id, - observation_index: draft.observation_index, - }); - const existing = matches?.[0]; - if (existing) { - const merged: PersonObservation = { - ...existing, - ...this.applyNullDefaults(draft), - observation_id: existing.observation_id, - }; - await this.repo.put(merged); - return merged; - } + const existing = await this.getByNaturalKey( + draft.accession_number, + draft.extractor_id, + draft.observation_index + ); + if (existing) return this.mergeOnto(existing, draft); + // No prior row for this natural key: hand the storage backend a draft // without `observation_id` and let the auto-generated PK assign one. - // This closes the TOCTOU race the previous size()+1 path had — the - // backend's monotonic counter / AUTOINCREMENT serialises ID - // assignment, so the previous process-wide AsyncMutex is unnecessary. // `put()` returns the persisted row including the assigned key. - return await this.repo.put( - this.applyNullDefaults(draft) as Parameters[0] - ); + try { + return await this.repo.put( + this.applyNullDefaults(draft) as Parameters[0] + ); + } catch (err) { + // A concurrent caller inserted the same natural key between our query and + // this put. The (accession_number, extractor_id, observation_index) + // UNIQUE index rejects the duplicate; re-read the winner and merge onto + // it so both callers converge on one row instead of forking two. + if (!isUniqueConstraintError(err)) throw err; + const winner = await this.getByNaturalKey( + draft.accession_number, + draft.extractor_id, + draft.observation_index + ); + if (!winner) throw err; + return this.mergeOnto(winner, draft); + } + } + + private async mergeOnto( + existing: PersonObservation, + draft: PersonObservationDraft + ): Promise { + const merged: PersonObservation = { + ...existing, + ...this.applyNullDefaults(draft), + observation_id: existing.observation_id, + }; + await this.repo.put(merged); + return merged; } async getByNaturalKey( diff --git a/src/resolver/isUniqueConstraintError.test.ts b/src/util/isUniqueConstraintError.test.ts similarity index 100% rename from src/resolver/isUniqueConstraintError.test.ts rename to src/util/isUniqueConstraintError.test.ts diff --git a/src/resolver/isUniqueConstraintError.ts b/src/util/isUniqueConstraintError.ts similarity index 100% rename from src/resolver/isUniqueConstraintError.ts rename to src/util/isUniqueConstraintError.ts From 7c05d6d11b0df97910286b3fa2e7afdab1205cfe Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 22:53:14 +0000 Subject: [PATCH 03/12] fix(util,fetch,spac,task,forms): correctness batch from subsystem review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - parseDate: the yyyyMMdd branch scrambled fields (e.g. "20230415" -> year 15); fix the regex + route it through the year-first branch, correct the dd-MM-yyyy comment to match the actual MM-dd-yyyy behavior, and add the first tests. - Form 144: wire in the dead `firstOf` guard at the two access sites so a repeated securitiesInformation/broker element can't null the proposed-sale block. - SecFetchJob: a per-attempt timeout was classified non-retriable ("timed out" didn't match the "timeout" pattern), so stalled fetches failed on the first timeout; key retry off the timeout controller directly and broaden the message pattern. Also give combineSignals a cleanup() so its fallback path stops leaking an abort listener per attempt onto the long-lived signal. - sectionRunner: a "
-partial" dead letter was never resolvable and lingered on the version-gated retry worklist forever; reconcile it (record on drop, resolve on a clean run). - spacRollup: stop summing redemption-typed events on top of the deal column (the deal column is the sole source) — closes a latent double-count. - spacDealGrouping: same-filing_date merger/redemption ties now resolve deterministically (form precedence then accession) instead of by the backend's unspecified row order. - RetryDeadLettersTask: isolate each accession so one hard parse/store error no longer abandons the rest of the sweep; report a failed count. - fetchAndStoreSubmission: rethrow TaskAbortedError instead of recording the interrupted CIK as a failure (mirrors the facts sibling). - bootstrap skip-set: only treat successfully-processed CIKs as done so a transient failure is retried on the next non-force bootstrap. - StoreCikLastUpdatedTask: progress is over rows, not a batch counter divided by the row count (the bar never advanced). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/cli/groups/extractor.ts | 4 +- src/fetch/SecFetchJob.ts | 48 +++++++++++++---- .../forms/insider-trading/Form_144.storage.ts | 7 ++- .../s1/sectionRunner.test.ts | 52 +++++++++++++++++++ .../s1/sectionRunner.ts | 38 +++++++++----- src/storage/processing/processedCikSet.ts | 18 +++---- src/storage/spac/spacDealGrouping.ts | 31 ++++++++++- src/storage/spac/spacRollup.test.ts | 6 ++- src/storage/spac/spacRollup.ts | 17 +++--- src/task/forms/RetryDeadLettersTask.ts | 34 +++++++++--- src/task/index/StoreCikLastUpdatedTask.ts | 6 ++- .../submissions/fetchAndStoreSubmission.ts | 6 ++- src/util/parseDate.test.ts | 43 +++++++++++++++ src/util/parseDate.ts | 10 ++-- 14 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 src/util/parseDate.test.ts diff --git a/src/cli/groups/extractor.ts b/src/cli/groups/extractor.ts index 4be6df12..1cb3f156 100644 --- a/src/cli/groups/extractor.ts +++ b/src/cli/groups/extractor.ts @@ -56,9 +56,11 @@ export function addExtractorCommands(program: Command): void { const out = (await withCli(new RetryDeadLettersTask()).run({ extractorId })) as { eligibleAccessions: string[]; reprocessed: number; + failed: number; }; + const failedSuffix = out.failed > 0 ? `, ${out.failed} failed` : ""; console.log( - `reprocessed ${out.reprocessed} filing(s) from ${out.eligibleAccessions.length} eligible` + `reprocessed ${out.reprocessed} filing(s) from ${out.eligibleAccessions.length} eligible${failedSuffix}` ); }); }); diff --git a/src/fetch/SecFetchJob.ts b/src/fetch/SecFetchJob.ts index 6babf0bb..78821ec1 100644 --- a/src/fetch/SecFetchJob.ts +++ b/src/fetch/SecFetchJob.ts @@ -47,7 +47,7 @@ interface MaybeHttpError { /** Error code / message heuristics shared with consumers classifying fetch failures. */ export const NETWORK_ERRNO_PATTERN = /^E(CONNRESET|TIMEDOUT|PIPE|AI_AGAIN|NOTFOUND|HOSTUNREACH|NETUNREACH)$/; -export const NETWORK_MESSAGE_PATTERN = /network|timeout|fetch failed|socket hang up/i; +export const NETWORK_MESSAGE_PATTERN = /network|timeout|timed out|fetch failed|socket hang up/i; function getStatus(error: MaybeHttpError): number | undefined { return error.status ?? error.statusCode ?? error.httpStatus ?? error.response?.status; @@ -159,25 +159,47 @@ function backoffDelay(attempt: number): number { return Math.floor(base * (0.5 + Math.random() * 0.5)); } -function combineSignals(signals: Array): AbortSignal { +/** + * Combine abort signals into one. Returns the combined signal plus a `cleanup` + * the caller must invoke (e.g. in `finally`): the fallback path attaches abort + * listeners to the source signals (including the long-lived `context.signal`), + * which otherwise accumulate one per attempt for the lifetime of the job/ + * workflow. `cleanup` is a no-op when no listeners were attached. + */ +function combineSignals(signals: Array): { + signal: AbortSignal; + cleanup: () => void; +} { + const noop = () => {}; const live = signals.filter((s): s is AbortSignal => Boolean(s)); - if (live.length === 0) return new AbortController().signal; - if (live.length === 1) return live[0]; + if (live.length === 0) return { signal: new AbortController().signal, cleanup: noop }; + if (live.length === 1) return { signal: live[0], cleanup: noop }; if ( typeof (AbortSignal as unknown as { any?: (s: AbortSignal[]) => AbortSignal }).any === "function" ) { - return (AbortSignal as unknown as { any: (s: AbortSignal[]) => AbortSignal }).any(live); + return { + signal: (AbortSignal as unknown as { any: (s: AbortSignal[]) => AbortSignal }).any(live), + cleanup: noop, + }; } const controller = new AbortController(); + const attached: Array<[AbortSignal, () => void]> = []; for (const s of live) { if (s.aborted) { controller.abort(s.reason); break; } - s.addEventListener("abort", () => controller.abort(s.reason), { once: true }); + const onAbort = () => controller.abort(s.reason); + attached.push([s, onAbort]); + s.addEventListener("abort", onAbort); } - return controller.signal; + return { + signal: controller.signal, + cleanup: () => { + for (const [s, onAbort] of attached) s.removeEventListener("abort", onAbort); + }, + }; } export class SecFetchJob< @@ -209,19 +231,27 @@ export class SecFetchJob< DEFAULT_TIMEOUT_MS ) : undefined; - const signal = combineSignals([context.signal, timeoutController?.signal]); + const { signal, cleanup } = combineSignals([context.signal, timeoutController?.signal]); try { return (await super.execute(input, { ...context, signal })) as Output; } catch (error) { lastError = error; if (context.signal.aborted) throw error; - if (!isRetriableError(error) || attempt === MAX_FETCH_ATTEMPTS - 1) throw error; + // A per-attempt timeout is a transient condition the retry loop exists + // to absorb, but the abort surfaces as a bare Error/AbortError whose + // shape isRetriableError can't recognise — so key off the timeout + // controller directly rather than the (lossy) error message. + const timedOut = timeoutController?.signal.aborted === true; + if ((!timedOut && !isRetriableError(error)) || attempt === MAX_FETCH_ATTEMPTS - 1) { + throw error; + } const retryAfter = getRetryAfterMs(error as MaybeHttpError); const delay = retryAfter ?? backoffDelay(attempt); await sleepWithAbort(delay, context.signal); } finally { if (timeoutHandle !== undefined) clearTimeout(timeoutHandle); + cleanup(); } } throw lastError; diff --git a/src/sec/forms/insider-trading/Form_144.storage.ts b/src/sec/forms/insider-trading/Form_144.storage.ts index 61f552fd..2512d3fe 100644 --- a/src/sec/forms/insider-trading/Form_144.storage.ts +++ b/src/sec/forms/insider-trading/Form_144.storage.ts @@ -140,8 +140,11 @@ export async function processForm144({ const addressRepo = new AddressRepo(); const repo = new Form144Repo(); - const securitiesInfo = formData.securitiesInformation; - const broker = securitiesInfo?.brokerOrMarketmakerDetails; + // firstOf guards the single-element-becomes-array case: if a filing repeats + // securitiesInformation/broker, fast-xml-parser yields an array and reading a + // field off it would silently null the whole proposed-sale block. + const securitiesInfo = firstOf(formData.securitiesInformation); + const broker = firstOf(securitiesInfo?.brokerOrMarketmakerDetails); const relationships = (issuerInfo?.relationshipsToIssuer?.relationshipToIssuer ?? []) .map((r) => str(r)) .filter((r): r is string => r !== null); diff --git a/src/sec/forms/registration-statements/s1/sectionRunner.test.ts b/src/sec/forms/registration-statements/s1/sectionRunner.test.ts index bf014960..1d91f8c9 100644 --- a/src/sec/forms/registration-statements/s1/sectionRunner.test.ts +++ b/src/sec/forms/registration-statements/s1/sectionRunner.test.ts @@ -92,4 +92,56 @@ describe("makeRunSection confidenceFloor", () => { expect(persisted).toBe(1); expect(resolved).toEqual(["merger"]); }); + + it("records a
-partial dead letter when some rows fail span verification", async () => { + const { repo, letters, resolved } = stubDeadLetters(); + const runSection = makeRunSection({ + deadLetters: repo, + extractor_id: "merger-proxy", + extractor_version: "1.0.0", + accession_number: "acc-3", + }); + await runSection({ + sectionName: "merger", + text: "keep drop", + emptyDetail: "empty", + lowConfidenceDetail: "low", + unverifiedPartialDetail: "$N of $T dropped", + extract: async () => [ + { confidence: 0.9, value: 1 }, + { confidence: 0.9, value: 2 }, + ], + verifyRow: (_text, r) => r.value === 1, // drop value:2 + persist: async () => 1, + }); + expect(resolved).toEqual(["merger"]); // base section resolved (survivors persisted) + expect(letters).toContainEqual({ + section_name: "merger-partial", + reason_code: "UNVERIFIED_SOURCE_SPAN", + }); + }); + + it("resolves a stale
-partial on a clean re-run (no drops)", async () => { + const { repo, letters, resolved } = stubDeadLetters(); + const runSection = makeRunSection({ + deadLetters: repo, + extractor_id: "merger-proxy", + extractor_version: "1.0.0", + accession_number: "acc-4", + }); + await runSection({ + sectionName: "merger", + text: "all good", + emptyDetail: "empty", + lowConfidenceDetail: "low", + unverifiedPartialDetail: "$N of $T dropped", + extract: async () => [{ confidence: 0.9, value: 1 }], + verifyRow: () => true, // nothing dropped + persist: async () => 1, + }); + // No new -partial recorded, and the base + sibling -partial are resolved so + // a previously-pending -partial cannot linger forever on the worklist. + expect(letters).toEqual([]); + expect(resolved).toEqual(["merger", "merger-partial"]); + }); }); diff --git a/src/sec/forms/registration-statements/s1/sectionRunner.ts b/src/sec/forms/registration-statements/s1/sectionRunner.ts index b9234119..26736f1b 100644 --- a/src/sec/forms/registration-statements/s1/sectionRunner.ts +++ b/src/sec/forms/registration-statements/s1/sectionRunner.ts @@ -130,21 +130,31 @@ export function makeRunSection(opts: { const wrote = await sargs.persist(rows); if (sargs.invalidWriteDetail !== undefined && wrote === 0) { await record("MODEL_INVALID_OUTPUT", sargs.invalidWriteDetail); - } else { - await deadLetters.markResolved(extractor_id, accession_number, sargs.sectionName); + return; } - if (droppedUnverified > 0 && sargs.unverifiedPartialDetail !== undefined) { - await deadLetters.record({ - extractor_id, - accession_number, - section_name: `${sargs.sectionName}-partial`, - reason_code: "UNVERIFIED_SOURCE_SPAN", - detail: sargs.unverifiedPartialDetail - .replace(/\$N/g, String(droppedUnverified)) - .replace(/\$T/g, String(confident.length)), - failed_extractor_version: extractor_version, - source_run_id: null, - }); + await deadLetters.markResolved(extractor_id, accession_number, sargs.sectionName); + // Reconcile the sibling `-partial` triage entry (only sections that can + // emit one carry unverifiedPartialDetail). Record it when THIS run dropped + // unverified rows; otherwise resolve any `-partial` left pending by a + // prior run, so a now-clean filing stops lingering forever on the + // version-gated retry worklist (markResolved no-ops when none exists). + if (sargs.unverifiedPartialDetail !== undefined) { + const partialSection = `${sargs.sectionName}-partial`; + if (droppedUnverified > 0) { + await deadLetters.record({ + extractor_id, + accession_number, + section_name: partialSection, + reason_code: "UNVERIFIED_SOURCE_SPAN", + detail: sargs.unverifiedPartialDetail + .replace(/\$N/g, String(droppedUnverified)) + .replace(/\$T/g, String(confident.length)), + failed_extractor_version: extractor_version, + source_run_id: null, + }); + } else { + await deadLetters.markResolved(extractor_id, accession_number, partialSection); + } } } catch (e) { await record( diff --git a/src/storage/processing/processedCikSet.ts b/src/storage/processing/processedCikSet.ts index aa253117..a5efd4f4 100644 --- a/src/storage/processing/processedCikSet.ts +++ b/src/storage/processing/processedCikSet.ts @@ -7,22 +7,22 @@ import type { ITabularStorage } from "workglow"; /** - * Streams every row in a `processed_*` repository and returns the set of - * `cik` values. Peak memory is bounded to `pageSize` rows + the final - * set (~24 bytes/entry — ~25 MB for a 1M-CIK corpus). + * Streams a `processed_*` repository and returns the set of `cik` values whose + * last processing **succeeded**. Peak memory is bounded to `pageSize` rows + + * the final set (~24 bytes/entry — ~25 MB for a 1M-CIK corpus). * - * Used by `BootstrapSubmissionsTask` and `BootstrapCompanyFactsTask` to - * compute the unprocessed-CIK set difference. The previous - * `await repo.getAll()` materialised every row + every column into RAM - * just to discard everything except `cik`; this avoids that intermediate. + * Used by `BootstrapSubmissionsTask` and `BootstrapCompanyFactsTask` to compute + * the unprocessed-CIK set difference. Failure rows (`success: false`, e.g. a + * transient FETCH_ERROR / STORE_ERROR) are excluded so a flaky pass is retried + * on the next non-`--force` bootstrap instead of being skipped forever. */ -export async function streamProcessedCikSet( +export async function streamProcessedCikSet( repo: Pick, "records">, pageSize: number = 5000 ): Promise> { const seen = new Set(); for await (const row of repo.records(pageSize)) { - seen.add(row.cik); + if (row.success) seen.add(row.cik); } return seen; } diff --git a/src/storage/spac/spacDealGrouping.ts b/src/storage/spac/spacDealGrouping.ts index 4a0c10f1..e0f6cd1e 100644 --- a/src/storage/spac/spacDealGrouping.ts +++ b/src/storage/spac/spacDealGrouping.ts @@ -9,6 +9,19 @@ import type { SpacEvent, SpacEventType } from "./SpacEventSchema"; import type { SpacMergerExtraction } from "./SpacMergerExtractionSchema"; import type { SpacRedemptionExtraction } from "./SpacRedemptionExtractionSchema"; +/** + * Supersession precedence for merger-proxy forms when two land on the same + * filing_date: revised (DEFR / PRER) supersedes definitive (DEFM) supersedes + * preliminary (PREM). Used only as a same-date tiebreak; filing_date still + * dominates. + */ +function mergerFormRank(form: string): number { + const f = form.toUpperCase(); + if (f.startsWith("DEFR") || f.startsWith("PRER")) return 2; + if (f.startsWith("DEFM")) return 1; + return 0; +} + /** Event types that shape a business-combination attempt. */ const DEAL_RELEVANT_EVENT_TYPES: readonly SpacEventType[] = [ "definitive_agreement", @@ -161,7 +174,16 @@ export function deriveDeals( (m) => (lower == null || m.filing_date >= lower) && (upper == null || m.filing_date < upper) ) - .sort((a, b) => a.filing_date.localeCompare(b.filing_date)); + // Deterministic supersession: by filing_date, then by form precedence + // (definitive > preliminary, revised > definitive — see CLAUDE.md), then + // by accession so ties resolve identically on every backend rather than + // depending on getByCik()'s unspecified row order. + .sort( + (a, b) => + a.filing_date.localeCompare(b.filing_date) || + mergerFormRank(a.form) - mergerFormRank(b.form) || + a.accession_number.localeCompare(b.accession_number) + ); // Latest non-null wins per field; earlier non-nulls survive when later is null. for (const m of matched) { if (m.target_name != null) d.target_name = m.target_name; @@ -190,7 +212,12 @@ export function deriveDeals( (r) => (lower == null || r.filing_date >= lower) && (upper == null || r.filing_date < upper) ) - .sort((a, b) => a.filing_date.localeCompare(b.filing_date)); + // accession is the deterministic tiebreak for same-date 8-Ks. + .sort( + (a, b) => + a.filing_date.localeCompare(b.filing_date) || + a.accession_number.localeCompare(b.accession_number) + ); // Latest non-null wins per field; earlier non-nulls survive when a later filing omits them. for (const r of matched) { if (r.redemption_amount != null) d.redemption_amount = r.redemption_amount; diff --git a/src/storage/spac/spacRollup.test.ts b/src/storage/spac/spacRollup.test.ts index 9e3b8b5f..6e384119 100644 --- a/src/storage/spac/spacRollup.test.ts +++ b/src/storage/spac/spacRollup.test.ts @@ -144,19 +144,21 @@ describe("buildSpacRow", () => { expect(row.status).toBe("liquidated"); }); - it("sums redemptions across a standalone event and a deal", () => { + it("sums redemptions only from the deal column (events are not double-counted)", () => { const row = buildSpacRow({ existing: undefined, cik: 1, deals: [deal({ deal_index: 0, outcome: "completed", redemption_amount: 50_000_000, outcome_date: "2022-05-01" })], events: [ ev({ event_type: "ipo", event_date: "2021-01-15" }), + // A stray redemption-typed event must NOT add on top of the deal column + // that deriveDeals already correlated the redemption onto. ev({ event_type: "redemption", event_date: "2022-01-01", amount: 10_000_000 }), ], patch: {}, filingDate: "2022-05-01", }); - expect(row.total_redemption_amount).toBe(60_000_000); + expect(row.total_redemption_amount).toBe(50_000_000); }); it("stale patch (older filing_date) does not overwrite merged scalar fields", () => { diff --git a/src/storage/spac/spacRollup.ts b/src/storage/spac/spacRollup.ts index 1f367a23..cfab3c27 100644 --- a/src/storage/spac/spacRollup.ts +++ b/src/storage/spac/spacRollup.ts @@ -80,7 +80,14 @@ function deriveStatus( return "registered"; } -function sumRedemptions(deals: readonly SpacDeal[], events: readonly SpacEvent[]): number | null { +/** + * The per-deal `redemption_amount` column (derived by `deriveDeals` correlating + * each redemption extraction onto exactly one deal) is the SOLE source of the + * rolled-up total, so each realized redemption is counted once. We deliberately + * do NOT also sum `redemption`-typed events: `recordRedemption` appends none, + * and summing both would double-count an extraction that is already on a deal. + */ +function sumRedemptions(deals: readonly SpacDeal[]): number | null { let sum = 0; let seen = false; for (const d of deals) { @@ -89,12 +96,6 @@ function sumRedemptions(deals: readonly SpacDeal[], events: readonly SpacEvent[] seen = true; } } - for (const e of events) { - if (e.event_type === "redemption" && e.amount != null) { - sum += e.amount; - seen = true; - } - } return seen ? sum : null; } @@ -169,7 +170,7 @@ export function buildSpacRow(input: BuildSpacRowInput): Spac { ipo_proceeds: pick("ipo_proceeds"), trust_amount: pick("trust_amount"), pipe_amount: active?.pipe_amount ?? null, - total_redemption_amount: sumRedemptions(deals, events), + total_redemption_amount: sumRedemptions(deals), registration_date: minEventDate(events, "registration"), ipo_date: minEventDate(events, "ipo"), unit_split_date: minEventDate(events, "unit_split"), diff --git a/src/task/forms/RetryDeadLettersTask.ts b/src/task/forms/RetryDeadLettersTask.ts index 38a2c026..f324693c 100644 --- a/src/task/forms/RetryDeadLettersTask.ts +++ b/src/task/forms/RetryDeadLettersTask.ts @@ -5,7 +5,14 @@ */ import { Static, Type } from "typebox"; -import { globalServiceRegistry, IExecuteContext, Task, TaskError, Workflow } from "workglow"; +import { + globalServiceRegistry, + IExecuteContext, + Task, + TaskAbortedError, + TaskError, + Workflow, +} from "workglow"; import { ExtractionDeadLetterRepo } from "../../storage/dead-letter/ExtractionDeadLetterRepo"; import { COMPONENT_VERSION_REPOSITORY_TOKEN } from "../../storage/versioning/ComponentVersionSchema"; import { VersionRegistry } from "../../storage/versioning/VersionRegistry"; @@ -23,6 +30,7 @@ const OutputSchema = () => Type.Object({ eligibleAccessions: Type.Array(Type.String()), reprocessed: Type.Number(), + failed: Type.Number(), }); type RetryDeadLettersTaskOutput = Static>; @@ -58,16 +66,28 @@ export class RetryDeadLettersTask extends Task< const accessions = [...new Set(eligible.map((e) => e.accession_number))]; if (input.dryRun) { - return { eligibleAccessions: accessions, reprocessed: 0 }; + return { eligibleAccessions: accessions, reprocessed: 0, failed: 0 }; } let reprocessed = 0; + let failed = 0; for (const accessionNumber of accessions) { - const wf = context.own(new Workflow()); - wf.pipe(new ProcessAccessionDocFormTask()); - await wf.run({ accessionNumber }); - reprocessed++; + if (context.signal?.aborted) throw new TaskAbortedError(); + // Isolate each accession: ProcessAccessionDocFormTask rethrows hard + // parse/store errors, and a recovery sweep must grind through the whole + // worklist rather than abandon every later accession on one bad filing. + try { + const wf = context.own(new Workflow()); + wf.pipe(new ProcessAccessionDocFormTask()); + await wf.run({ accessionNumber }); + reprocessed++; + } catch (e) { + if (e instanceof TaskAbortedError) throw e; + failed++; + const message = e instanceof Error ? e.message : String(e); + console.warn(`retry-dead-letters: ${accessionNumber} failed to reprocess: ${message}`); + } } - return { eligibleAccessions: accessions, reprocessed }; + return { eligibleAccessions: accessions, reprocessed, failed }; } } diff --git a/src/task/index/StoreCikLastUpdatedTask.ts b/src/task/index/StoreCikLastUpdatedTask.ts index 6d1621b6..9c07bd1f 100644 --- a/src/task/index/StoreCikLastUpdatedTask.ts +++ b/src/task/index/StoreCikLastUpdatedTask.ts @@ -57,7 +57,6 @@ export class StoreCikLastUpdatedTask extends Task< const length = updateList.length; let progress = 0; - let index = 0; const batchSize = 1000; const batches = Math.ceil(length / batchSize); for (let i = 0; i < batches; i++) { @@ -73,7 +72,10 @@ export class StoreCikLastUpdatedTask extends Task< })); await cikLastUpdateRepo.putBulk(batch); - const newProgress = Math.round((index++ / length) * 1000) / 10; + // Progress is over ROWS processed, not batches — index++ / length (the + // old form) divided a batch counter by the row count and never advanced. + const processed = Math.min((i + 1) * batchSize, length); + const newProgress = Math.round((processed / length) * 1000) / 10; if (newProgress > progress) { context.updateProgress(newProgress); progress = newProgress; diff --git a/src/task/submissions/fetchAndStoreSubmission.ts b/src/task/submissions/fetchAndStoreSubmission.ts index 3025a849..a59a4b1b 100644 --- a/src/task/submissions/fetchAndStoreSubmission.ts +++ b/src/task/submissions/fetchAndStoreSubmission.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IExecuteContext, pipe } from "workglow"; +import { IExecuteContext, TaskAbortedError, pipe } from "workglow"; import { FetchSubmissionsTask } from "./FetchSubmissionsTask"; import { processUpdateProcessing, StoreSubmissionsTask } from "./StoreSubmissionsTask"; @@ -16,6 +16,10 @@ export async function fetchAndStoreSubmission( try { await pipeline.run(input); } catch (e) { + // A cancellation is not a per-CIK failure: propagate it so the map stops, + // and do NOT record a spurious failure row that would mislabel the + // interrupted CIK (and re-fetch it forever). Mirrors the facts sibling. + if (e instanceof TaskAbortedError) throw e; const message = e instanceof Error ? e.message : String(e); console.warn(`Failed to fetch/store submission for CIK ${input.cik}: ${message}`); await processUpdateProcessing(input.cik, false); diff --git a/src/util/parseDate.test.ts b/src/util/parseDate.test.ts new file mode 100644 index 00000000..9ec707a7 --- /dev/null +++ b/src/util/parseDate.test.ts @@ -0,0 +1,43 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from "bun:test"; +import { parseDate, secDate } from "./parseDate"; + +describe("parseDate", () => { + it("parses yyyy-MM-dd", () => { + expect(parseDate("2023-04-05")).toEqual({ year: 2023, month: "04", day: "05" }); + }); + + it("parses yyyy/MM/dd", () => { + expect(parseDate("2023/04/05")).toEqual({ year: 2023, month: "04", day: "05" }); + }); + + it("parses MM/dd/yyyy", () => { + expect(parseDate("04/05/2023")).toEqual({ year: 2023, month: "04", day: "05" }); + }); + + it("parses the compact yyyyMMdd form (EDGAR index filenames)", () => { + // Regression: the old regex/branch scrambled this to {year:5, month:'2023'}. + expect(parseDate("20230405")).toEqual({ year: 2023, month: "04", day: "05" }); + expect(parseDate("20231231")).toEqual({ year: 2023, month: "12", day: "31" }); + }); + + it("throws on an unrecognised format", () => { + expect(() => parseDate("not-a-date")).toThrow("Invalid date format"); + }); +}); + +describe("secDate", () => { + it("round-trips the compact form to YYYY-MM-DD", () => { + expect(secDate("20230405")).toBe("2023-04-05"); + }); + + it("formats a Date", () => { + // Local-time constructor + secDate's local getters => timezone-independent. + expect(secDate(new Date(2023, 3, 5))).toBe("2023-04-05"); + }); +}); diff --git a/src/util/parseDate.ts b/src/util/parseDate.ts index 549b8008..d7d0a3f0 100644 --- a/src/util/parseDate.ts +++ b/src/util/parseDate.ts @@ -17,9 +17,9 @@ export function parseDate(dateStr: string): { year: number; month: string; day: const regexes = [ /^(\d{4})-(\d{1,2})-(\d{1,2})$/, // yyyy-MM-dd /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/, // MM/dd/yyyy - /^(\d{1,2})-(\d{1,2})-(\d{4})$/, // dd-MM-yyyy + /^(\d{1,2})-(\d{1,2})-(\d{4})$/, // MM-dd-yyyy /^(\d{4})\/(\d{1,2})\/(\d{1,2})$/, // yyyy/MM/dd - /^(\d{4})(\d{1,2})(\d{1,2})$/, // yyyyMMdd + /^(\d{4})(\d{2})(\d{2})$/, // yyyyMMdd (EDGAR index filenames) ]; for (const regex of regexes) { @@ -27,13 +27,13 @@ export function parseDate(dateStr: string): { year: number; month: string; day: if (match) { let year: number, month: number, day: number; - if (regex === regexes[0] || regex === regexes[3]) { - // yyyy-MM-dd or yyyy/MM/dd + if (regex === regexes[0] || regex === regexes[3] || regex === regexes[4]) { + // year-first: yyyy-MM-dd, yyyy/MM/dd, or yyyyMMdd year = parseInt(match[1], 10); month = parseInt(match[2], 10); day = parseInt(match[3], 10); } else { - // MM/dd/yyyy or dd-MM-yyyy + // month-first: MM/dd/yyyy or MM-dd-yyyy year = parseInt(match[3], 10); month = parseInt(match[1], 10); day = parseInt(match[2], 10); From a5d1380aa7cdfbd9245a571e03ecbdf00edd50d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 23:02:24 +0000 Subject: [PATCH 04/12] fix(cli): error-as-UX + teardown-safe exits + honest global flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - query: wrap every action in runCommand so a bad --format or repo failure renders as a clean "x " with exit code 1 instead of an uncaught stack-trace dump (verified end to end). - resolve/canonical/sponsor-family/underwriter-family: stop calling process.exit(1) mid-command — it bypassed the sec.ts finally that stops the fetch queue and closes the DB/PG pool. resolve now runs under runCommand and throws; the others set process.exitCode + return. Also give `sec resolve --kind person --all` the same per-observation try/catch the company branch has, so one bad row can't abort the batch. - version coverage resolver: throw instead of process.exit, and honor --format json (the resolver branch always printed human text). - init: the first-run wizard pointed at non-existent `sec bootstrap cik` / `bootstrap index`; point at the real `bootstrap download ciks` / `bootstrap ingest cik-names`. - GlobalOptions: drop --json / --verbose / --no-color, which were parsed and advertised in --help but never consumed (only --dry-run is wired); update the tests that pinned the dead flags. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/cli/GlobalOptions.test.ts | 54 +++--------- src/cli/GlobalOptions.ts | 16 ++-- src/cli/cli.integration.test.ts | 3 - src/cli/groups/canonical.ts | 24 ++++-- src/cli/groups/init.ts | 6 +- src/cli/groups/query.ts | 54 +++++++----- src/cli/groups/resolve.ts | 135 ++++++++++++++++-------------- src/cli/groups/version.ts | 9 +- src/commands/sponsorFamily.ts | 6 +- src/commands/underwriterFamily.ts | 9 +- 10 files changed, 157 insertions(+), 159 deletions(-) diff --git a/src/cli/GlobalOptions.test.ts b/src/cli/GlobalOptions.test.ts index 0ee985b4..1320f793 100644 --- a/src/cli/GlobalOptions.test.ts +++ b/src/cli/GlobalOptions.test.ts @@ -11,13 +11,17 @@ function createProgram(): Command { describe("GlobalOptions", () => { describe("applyGlobalOptions", () => { - it("adds all flags to help text", () => { + it("adds --dry-run to help text", () => { const program = createProgram(); const help = program.helpInformation(); - expect(help).toContain("--json"); - expect(help).toContain("--verbose"); expect(help).toContain("--dry-run"); - expect(help).toContain("--no-color"); + }); + + it("does not advertise the removed (never-consumed) flags", () => { + const help = createProgram().helpInformation(); + expect(help).not.toContain("--json"); + expect(help).not.toContain("--verbose"); + expect(help).not.toContain("--no-color"); }); it("returns the program for chaining", () => { @@ -27,29 +31,11 @@ describe("GlobalOptions", () => { }); }); - describe("parseGlobalOptions defaults", () => { - it("returns correct defaults when no flags are set", () => { + describe("parseGlobalOptions", () => { + it("defaults dryRun to false", () => { const program = createProgram(); program.parse([], { from: "user" }); - const opts = parseGlobalOptions(program); - expect(opts.json).toBe(false); - expect(opts.verbose).toBe(false); - expect(opts.dryRun).toBe(false); - expect(opts.color).toBe(true); - }); - }); - - describe("parseGlobalOptions with flags", () => { - it("parses --json", () => { - const program = createProgram(); - program.parse(["--json"], { from: "user" }); - expect(parseGlobalOptions(program).json).toBe(true); - }); - - it("parses --verbose", () => { - const program = createProgram(); - program.parse(["--verbose"], { from: "user" }); - expect(parseGlobalOptions(program).verbose).toBe(true); + expect(parseGlobalOptions(program).dryRun).toBe(false); }); it("parses --dry-run", () => { @@ -57,23 +43,5 @@ describe("GlobalOptions", () => { program.parse(["--dry-run"], { from: "user" }); expect(parseGlobalOptions(program).dryRun).toBe(true); }); - - it("parses --no-color", () => { - const program = createProgram(); - program.parse(["--no-color"], { from: "user" }); - expect(parseGlobalOptions(program).color).toBe(false); - }); - - it("parses all flags together", () => { - const program = createProgram(); - program.parse(["--json", "--verbose", "--dry-run", "--no-color"], { - from: "user", - }); - const opts = parseGlobalOptions(program); - expect(opts.json).toBe(true); - expect(opts.verbose).toBe(true); - expect(opts.dryRun).toBe(true); - expect(opts.color).toBe(false); - }); }); }); diff --git a/src/cli/GlobalOptions.ts b/src/cli/GlobalOptions.ts index d728e4fc..92f9c145 100644 --- a/src/cli/GlobalOptions.ts +++ b/src/cli/GlobalOptions.ts @@ -1,27 +1,21 @@ import type { Command } from "commander"; export interface GlobalOptions { - readonly json: boolean; - readonly verbose: boolean; readonly dryRun: boolean; - readonly color: boolean; } +// Only `--dry-run` is wired (it gates writes via SEC_DRY_RUN). Previous +// `--json` / `--verbose` / `--no-color` flags were parsed but never consumed — +// read commands carry their own `--format`, and output is plain text — so they +// were removed rather than advertised in --help while doing nothing. export function applyGlobalOptions(program: Command): Command { - return program - .option("--json", "Force JSON output", false) - .option("--verbose", "Show detailed logs", false) - .option("--dry-run", "Show what would happen without changes", false) - .option("--no-color", "Disable colored output"); + return program.option("--dry-run", "Show what would happen without changes", false); } export function parseGlobalOptions(cmd: Command): GlobalOptions { const opts = cmd.opts(); return { - json: opts.json ?? false, - verbose: opts.verbose ?? false, dryRun: opts.dryRun ?? false, - color: opts.color ?? true, }; } diff --git a/src/cli/cli.integration.test.ts b/src/cli/cli.integration.test.ts index 10c1732c..dc13d01e 100644 --- a/src/cli/cli.integration.test.ts +++ b/src/cli/cli.integration.test.ts @@ -35,10 +35,7 @@ describe("CLI v2 integration", () => { it("should show global options", async () => { const output = await runCli("--help"); - expect(output).toContain("--json"); - expect(output).toContain("--verbose"); expect(output).toContain("--dry-run"); - expect(output).toContain("--no-color"); }); it("should show version 2.0.0", async () => { diff --git a/src/cli/groups/canonical.ts b/src/cli/groups/canonical.ts index a778f05d..c4b8b79d 100644 --- a/src/cli/groups/canonical.ts +++ b/src/cli/groups/canonical.ts @@ -95,11 +95,13 @@ export function addCanonicalCommands(program: Command): void { intoId = await resolveCanonicalPersonRef(into, canonRepo); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } if (fromId === intoId) { console.error("error: cannot alias an id to itself"); - process.exit(1); + process.exitCode = 1; + return; } const aliasRepo = new CanonicalPersonAliasRepo(); try { @@ -112,7 +114,8 @@ export function addCanonicalCommands(program: Command): void { console.log(`aliased ${row.alias_canonical_id} → ${row.target_canonical_id}`); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } }); @@ -125,7 +128,8 @@ export function addCanonicalCommands(program: Command): void { fromId = await resolveCanonicalPersonRef(from, canonRepo); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } const aliasRepo = new CanonicalPersonAliasRepo(); await aliasRepo.remove(fromId); @@ -167,11 +171,13 @@ export function addCanonicalCommands(program: Command): void { intoId = await resolveCanonicalCompanyRef(into, canonRepo); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } if (fromId === intoId) { console.error("error: cannot alias an id to itself"); - process.exit(1); + process.exitCode = 1; + return; } const aliasRepo = new CanonicalCompanyAliasRepo(); try { @@ -184,7 +190,8 @@ export function addCanonicalCommands(program: Command): void { console.log(`aliased ${row.alias_canonical_id} → ${row.target_canonical_id}`); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } }); @@ -197,7 +204,8 @@ export function addCanonicalCommands(program: Command): void { fromId = await resolveCanonicalCompanyRef(from, canonRepo); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } const aliasRepo = new CanonicalCompanyAliasRepo(); await aliasRepo.remove(fromId); diff --git a/src/cli/groups/init.ts b/src/cli/groups/init.ts index fdcdfb68..1b05e788 100644 --- a/src/cli/groups/init.ts +++ b/src/cli/groups/init.ts @@ -164,9 +164,9 @@ export function addInitCommand(parent: Command): void { console.log("Database tables created."); console.log("\nSetup complete! Next steps:"); - console.log(" sec db status — verify database connection"); - console.log(" sec bootstrap cik — download CIK name lookup"); - console.log(" sec bootstrap index — download filing indexes"); + console.log(" sec db status — verify database connection"); + console.log(" sec bootstrap download ciks — download the CIK name lookup"); + console.log(" sec bootstrap ingest cik-names — load CIK names into the database"); } finally { rl.close(); } diff --git a/src/cli/groups/query.ts b/src/cli/groups/query.ts index 60e962fc..01cdc080 100644 --- a/src/cli/groups/query.ts +++ b/src/cli/groups/query.ts @@ -1,5 +1,6 @@ import type { Command } from "commander"; import { parseIntOption } from "../GlobalOptions"; +import { runCommand } from "../runCommand"; import { renderTable } from "../output/TableRenderer"; import { queryCiks } from "../queries/CikQuery"; import { queryCrowdfunding } from "../queries/CrowdfundingQuery"; @@ -21,6 +22,19 @@ function validateFormat(value: string): OutputFormat { return value as OutputFormat; } +/** + * Wraps a Commander action so a thrown error (bad --format, repo failure) + * renders as a clean `x ` with exit code 1 — via runCommand — instead + * of an uncaught stack-trace dump, matching the rest of the CLI surface. + */ +function wrapAction( + fn: (...args: A) => Promise +): (...args: A) => Promise { + return async (...args: A): Promise => { + await runCommand(() => fn(...args)); + }; +} + export function addQueryCommands(program: Command): void { const query = program.command("query").description("Query stored SEC data"); @@ -31,7 +45,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (name: string, options: Record) => { + .action(wrapAction(async (name: string, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -62,7 +76,7 @@ export function addQueryCommands(program: Command): void { "\nThe cik_names table is empty. Run `sec bootstrap ingest cik-names` to populate it." ); } - }); + })); query .command("entities [search]") @@ -74,7 +88,7 @@ export function addQueryCommands(program: Command): void { .option("--offset ", "Offset results", parseIntOption, 0) .option("--sort ", "Sort by field") .option("--format ", "Output format (table, json, csv)", "table") - .action(async (search: string | undefined, options: Record) => { + .action(wrapAction(async (search: string | undefined, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -104,7 +118,7 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); query .command("filings [search]") @@ -116,7 +130,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (search: string | undefined, options: Record) => { + .action(wrapAction(async (search: string | undefined, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -147,7 +161,7 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); query .command("offerings [search]") @@ -160,7 +174,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (search: string | undefined, options: Record) => { + .action(wrapAction(async (search: string | undefined, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -191,7 +205,7 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); query .command("crowdfunding [search]") @@ -203,7 +217,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (search: string | undefined, options: Record) => { + .action(wrapAction(async (search: string | undefined, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -233,7 +247,7 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); query .command("reg-a [search]") @@ -245,7 +259,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (search: string | undefined, options: Record) => { + .action(wrapAction(async (search: string | undefined, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -277,7 +291,7 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); query .command("reg-a-summary [cik]") @@ -285,7 +299,7 @@ export function addQueryCommands(program: Command): void { "Roll up Regulation A offerings: counts by status/tier, plus the latest aggregate offering amount when a CIK is given" ) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (cik: string | undefined, options: Record) => { + .action(wrapAction(async (cik: string | undefined, options: Record) => { const format = validateFormat(options.format as string); const summary = await summarizeRegA(cik ? parseIntOption(cik) : undefined); @@ -317,7 +331,7 @@ export function addQueryCommands(program: Command): void { { format } ) ); - }); + })); query .command("facts ") @@ -328,7 +342,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (cik: string, options: Record) => { + .action(wrapAction(async (cik: string, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -359,7 +373,7 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); query .command("xbrl ") @@ -369,7 +383,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (accession: string, options: Record) => { + .action(wrapAction(async (accession: string, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -406,7 +420,7 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); query .command("persons [search]") @@ -416,7 +430,7 @@ export function addQueryCommands(program: Command): void { .option("--limit ", "Limit results", parseIntOption, 25) .option("--offset ", "Offset results", parseIntOption, 0) .option("--format ", "Output format (table, json, csv)", "table") - .action(async (search: string | undefined, options: Record) => { + .action(wrapAction(async (search: string | undefined, options: Record) => { const limit = options.limit as number; const offset = options.offset as number; const format = validateFormat(options.format as string); @@ -444,5 +458,5 @@ export function addQueryCommands(program: Command): void { limit, }) ); - }); + })); } diff --git a/src/cli/groups/resolve.ts b/src/cli/groups/resolve.ts index b36c26c3..62a979b5 100644 --- a/src/cli/groups/resolve.ts +++ b/src/cli/groups/resolve.ts @@ -17,6 +17,7 @@ import { PersonResolver } from "../../resolver/PersonResolver"; import { CompanyResolver } from "../../resolver/CompanyResolver"; import { RESOLVER_IDS, isFamilyResolverId, type ResolverId } from "../../resolver/resolverIds"; import { isValidSemver } from "../../storage/versioning/VersionRegistry"; +import { runCommand } from "../runCommand"; export function addResolveCommands(program: Command): void { const cmd = program.command("resolve"); @@ -26,72 +27,78 @@ export function addResolveCommands(program: Command): void { .requiredOption("--resolver-version ", "target resolver semver") .option("--all", "process all observations of the kind", false) .action(async (opts: { kind: string; resolverVersion: string; all: boolean }) => { - if (!RESOLVER_IDS.includes(opts.kind as ResolverId)) { - console.error(`error: --kind must be one of ${RESOLVER_IDS.join("|")}`); - process.exit(1); - } - // Family-tier resolvers run inline during S-1 extraction (keyed off the - // sponsor/underwriter common name), not as a batch pass over observations. - // Refuse here rather than silently running the company resolver and writing - // mislabeled company identity-link rows. - if (isFamilyResolverId(opts.kind)) { - console.error( - `error: 'sec resolve' does not support family resolver kind '${opts.kind}'; ` + - `family resolution happens inline during S-1 extraction` - ); - process.exit(1); - } - if (!opts.all) { - console.error("error: --all is required (no other mode supported in v1)"); - process.exit(1); - } - if (!isValidSemver(opts.resolverVersion)) { - console.error( - `error: --resolver-version must be a valid semver (got '${opts.resolverVersion}')` - ); - process.exit(1); - } - - if (opts.kind === "person") { - const obsRepo = new PersonObservationRepo(); - const canonRepo = new CanonicalPersonRepo(); - const aliasRepo = new CanonicalPersonAliasRepo(); - const linkRepo = new PersonIdentityLinkRepo(); - const resolver = new PersonResolver({ - canonicalPersonRepo: canonRepo, - canonicalPersonAliasRepo: aliasRepo, - activeResolverVersion: opts.resolverVersion, - }); - const all = await obsRepo.listAll(); - let count = 0; - for (const obs of all) { - const id = await resolver.resolve(obs); - await linkRepo.upsert(obs.observation_id, opts.resolverVersion, id); - count++; + // runCommand: a validation throw renders a clean error + sets exit code + // 1 without bypassing the top-level queue/DB teardown (process.exit would). + await runCommand(async () => { + if (!RESOLVER_IDS.includes(opts.kind as ResolverId)) { + throw new Error(`--kind must be one of ${RESOLVER_IDS.join("|")}`); + } + // Family-tier resolvers run inline during S-1 extraction (keyed off the + // sponsor/underwriter common name), not as a batch pass over + // observations. Refuse here rather than silently running the company + // resolver and writing mislabeled company identity-link rows. + if (isFamilyResolverId(opts.kind)) { + throw new Error( + `'sec resolve' does not support family resolver kind '${opts.kind}'; ` + + `family resolution happens inline during S-1 extraction` + ); + } + if (!opts.all) { + throw new Error("--all is required (no other mode supported in v1)"); } - console.log(`resolved ${count} person observation(s) at v${opts.resolverVersion}`); - } else { - const obsRepo = new CompanyObservationRepo(); - const canonRepo = new CanonicalCompanyRepo(); - const aliasRepo = new CanonicalCompanyAliasRepo(); - const linkRepo = new CompanyIdentityLinkRepo(); - const resolver = new CompanyResolver({ - canonicalCompanyRepo: canonRepo, - canonicalCompanyAliasRepo: aliasRepo, - activeResolverVersion: opts.resolverVersion, - }); - const all = await obsRepo.listAll(); - let count = 0; - for (const obs of all) { - try { - const id = await resolver.resolve(obs); - await linkRepo.upsert(obs.observation_id, opts.resolverVersion, id); - count++; - } catch (e) { - console.error(`skipping observation ${obs.observation_id}: ${(e as Error).message}`); + if (!isValidSemver(opts.resolverVersion)) { + throw new Error( + `--resolver-version must be a valid semver (got '${opts.resolverVersion}')` + ); + } + + if (opts.kind === "person") { + const obsRepo = new PersonObservationRepo(); + const canonRepo = new CanonicalPersonRepo(); + const aliasRepo = new CanonicalPersonAliasRepo(); + const linkRepo = new PersonIdentityLinkRepo(); + const resolver = new PersonResolver({ + canonicalPersonRepo: canonRepo, + canonicalPersonAliasRepo: aliasRepo, + activeResolverVersion: opts.resolverVersion, + }); + const all = await obsRepo.listAll(); + let count = 0; + for (const obs of all) { + // Isolate per-observation failures so one bad row can't abort the + // whole batch (mirrors the company branch below). + try { + const id = await resolver.resolve(obs); + await linkRepo.upsert(obs.observation_id, opts.resolverVersion, id); + count++; + } catch (e) { + console.error(`skipping observation ${obs.observation_id}: ${(e as Error).message}`); + } + } + console.log(`resolved ${count} person observation(s) at v${opts.resolverVersion}`); + } else { + const obsRepo = new CompanyObservationRepo(); + const canonRepo = new CanonicalCompanyRepo(); + const aliasRepo = new CanonicalCompanyAliasRepo(); + const linkRepo = new CompanyIdentityLinkRepo(); + const resolver = new CompanyResolver({ + canonicalCompanyRepo: canonRepo, + canonicalCompanyAliasRepo: aliasRepo, + activeResolverVersion: opts.resolverVersion, + }); + const all = await obsRepo.listAll(); + let count = 0; + for (const obs of all) { + try { + const id = await resolver.resolve(obs); + await linkRepo.upsert(obs.observation_id, opts.resolverVersion, id); + count++; + } catch (e) { + console.error(`skipping observation ${obs.observation_id}: ${(e as Error).message}`); + } } + console.log(`resolved ${count} company observation(s) at v${opts.resolverVersion}`); } - console.log(`resolved ${count} company observation(s) at v${opts.resolverVersion}`); - } + }); }); } diff --git a/src/cli/groups/version.ts b/src/cli/groups/version.ts index a2a547ff..170f4762 100644 --- a/src/cli/groups/version.ts +++ b/src/cli/groups/version.ts @@ -192,10 +192,15 @@ export function addVersionCommands(program: Command): void { ); const slot = await getActiveSlot(versionRegistry, "resolver", id); if (!slot) { - console.error(`No active slot for resolver:${id}`); - process.exit(1); + // Throw (not process.exit) so runCommand renders the error and sets + // the exit code without bypassing the top-level teardown. + throw new Error(`No active slot for resolver:${id}`); } const result = await computeResolverCoverage(id as ResolverId, slot.semver); + if (options.format === "json") { + console.log(JSON.stringify(result, null, 2)); + return; + } console.log( `resolver:${result.kind}@${result.resolver_version}: ${result.numerator}/${result.denominator} (${(result.fraction * 100).toFixed(1)}%)` ); diff --git a/src/commands/sponsorFamily.ts b/src/commands/sponsorFamily.ts index 6a2db001..b1f0af2b 100644 --- a/src/commands/sponsorFamily.ts +++ b/src/commands/sponsorFamily.ts @@ -71,7 +71,8 @@ export function registerSponsorFamilyCommands(program: Command): void { ); if (!from || !into) { console.error("error: both family names must already exist"); - process.exit(1); + process.exitCode = 1; + return; } try { await new CanonicalSponsorFamilyAliasRepo().add( @@ -83,7 +84,8 @@ export function registerSponsorFamilyCommands(program: Command): void { console.log(`aliased '${fromName}' -> '${intoName}'`); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } } ); diff --git a/src/commands/underwriterFamily.ts b/src/commands/underwriterFamily.ts index 7f02fe6c..da6e1946 100644 --- a/src/commands/underwriterFamily.ts +++ b/src/commands/underwriterFamily.ts @@ -74,7 +74,8 @@ export function registerUnderwriterFamilyCommands(program: Command): void { ); if (!from || !into) { console.error("error: both family names must already exist"); - process.exit(1); + process.exitCode = 1; + return; } try { await new CanonicalUnderwriterFamilyAliasRepo().add( @@ -86,7 +87,8 @@ export function registerUnderwriterFamilyCommands(program: Command): void { console.log(`aliased '${fromName}' -> '${intoName}'`); } catch (e) { console.error(`error: ${(e as Error).message}`); - process.exit(1); + process.exitCode = 1; + return; } } ); @@ -103,7 +105,8 @@ export function registerUnderwriterFamilyCommands(program: Command): void { ); if (!family) { console.error(`error: no underwriter-family found for '${name}'`); - process.exit(1); + process.exitCode = 1; + return; } await new CanonicalUnderwriterFamilyAliasRepo().remove( family.canonical_underwriter_family_id From fd878711b439855f4b6835e94a1790bdb06b2c93 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 23:51:30 +0000 Subject: [PATCH 05/12] fix(forms,storage): Form D as_of out-of-order guard + collision-proof Crowdfunding history PK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Form D mutable-current row: the InvestmentOffering row (keyed cik+file_number) had no out-of-order guard and processFormD dropped filing_date entirely, so a back-catalog older D / D-A overwrote the current row with stale industry_group / security-type flags / date_of_first_sale — the regression the temporal contract forbids and that every sibling mutable-current repo (RegA/Portal/Crowdfunding) already guards. Add an `as_of` column, thread filing_date through processFormD -> processOffering, and skip the mutable write when the incoming filing is older than the row's anchor (an undated filing is treated as stale against a dated row). The append-only history is always written. Each Form D fully restates the offering, so no field-merge is needed — only the date guard. New regression test. Crowdfunding history PK: history rows were keyed (cik, valid_from) where valid_from = new Date().toISOString() (ms resolution), so two snapshots for one CIK landing in the same millisecond — realistic under a batch re-process that writes a closed-immediately snapshot per filing — collided and the second silently overwrote the first, dropping a history version. Add the source `accession_number` to the row and the PK (meaningful provenance + a guaranteed cross-filing disambiguator), threaded via saveCrowdfundingWithHistory's options; Form C passes it. New regression test (fails under the old 2-tuple PK). No DB migration needed (no persistent database yet). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- .../forms/exempt-offerings/Form_C.storage.ts | 1 + .../exempt-offerings/Form_D.pipeline.test.ts | 2 + .../forms/exempt-offerings/Form_D.storage.ts | 31 +++++++++-- src/sec/forms/exempt-offerings/Form_D.test.ts | 52 +++++++++++++++++++ .../InvestmentOfferingSchema.ts | 9 ++++ .../portal/CrowdfundingHistorySchema.ts | 10 +++- .../portal/CrowdfundingTemporalRepo.test.ts | 51 ++++++++++++++++++ .../portal/CrowdfundingTemporalRepo.ts | 11 +++- 8 files changed, 159 insertions(+), 8 deletions(-) diff --git a/src/sec/forms/exempt-offerings/Form_C.storage.ts b/src/sec/forms/exempt-offerings/Form_C.storage.ts index febe3e1a..f48f7c85 100644 --- a/src/sec/forms/exempt-offerings/Form_C.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_C.storage.ts @@ -474,6 +474,7 @@ export async function processFormC({ await temporalRepo.saveCrowdfundingWithHistory(crowdfunding, `Form ${submissionType}`, { skipMutableUpdate: isStale, + accessionNumber: accession_number, }); // Issuers: index 0 (issuer), 1+ (co-issuers) diff --git a/src/sec/forms/exempt-offerings/Form_D.pipeline.test.ts b/src/sec/forms/exempt-offerings/Form_D.pipeline.test.ts index 1433b524..f1e9c536 100644 --- a/src/sec/forms/exempt-offerings/Form_D.pipeline.test.ts +++ b/src/sec/forms/exempt-offerings/Form_D.pipeline.test.ts @@ -68,6 +68,7 @@ describe("Form_D pipeline", () => { cik, file_number: fileNumber, accession_number: accession, + filing_date: "", primary_doc: file, formD, }); @@ -160,6 +161,7 @@ describe("Form_D pipeline", () => { cik, file_number: fileNumber, accession_number: accession, + filing_date: "", primary_doc: file, formD, }); diff --git a/src/sec/forms/exempt-offerings/Form_D.storage.ts b/src/sec/forms/exempt-offerings/Form_D.storage.ts index 91837e43..35826e0d 100644 --- a/src/sec/forms/exempt-offerings/Form_D.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_D.storage.ts @@ -77,6 +77,7 @@ async function processOffering( cik: number, file_number: string, accession_number: string, + filing_date: string, offering: OfferingData, ctx: FormDStorageContext ): Promise { @@ -109,6 +110,7 @@ async function processOffering( offering.businessCombinationTransaction.isBusinessCombinationTransaction === "true", is_other_type: typesOfSecOff.isOtherType === "true" ? true : null, description_of_other: typesOfSecOff.descriptionOfOtherType || null, + as_of: null, }; const investmentOfferingHistory: InvestmentOfferingHistory = { @@ -127,10 +129,27 @@ async function processOffering( non_accredited_count: offering.investors.numberNonAccreditedInvestors || null, }; - await investmentOfferingRepo.saveInvestmentOfferingWithHistory( - investmentOffering, - investmentOfferingHistory - ); + // History is per-(cik, file_number, accession) and always recorded — it is + // the append-only time series. The mutable current row must reflect the + // latest filing BY FILING DATE, not by processing order: skip an out-of-order + // older D / D-A so a back-catalog replay can't regress industry_group / + // security-type flags / date_of_first_sale. An undated incoming filing ("") + // cannot be ordered against a dated row and is treated as stale; when the + // existing row is also undated or absent it still applies. (Each Form D fully + // restates the offering, so no field-merge is needed — only the date guard.) + await investmentOfferingRepo.saveInvestmentOfferingHistory(investmentOfferingHistory); + + const existing = await investmentOfferingRepo.getInvestmentOffering(cik, file_number); + const isStale = + existing?.as_of != null && + existing.as_of !== "" && + (filing_date === "" || filing_date < existing.as_of); + if (!isStale) { + await investmentOfferingRepo.saveInvestmentOffering({ + ...investmentOffering, + as_of: filing_date || existing?.as_of || null, + }); + } if (offering.salesCompensationList.recipient) { let salesIndex = 100; @@ -423,12 +442,14 @@ export async function processFormD({ cik, file_number, accession_number, + filing_date, primary_doc, formD, }: { cik: number; file_number: string; accession_number: string; + filing_date: string; primary_doc: string; formD: FormD; }): Promise { @@ -501,7 +522,7 @@ export async function processFormD({ } // Sales compensation: indices 100–199 (handled inside processOffering) - await processOffering(cik, file_number, accession_number, formD.offeringData, ctx); + await processOffering(cik, file_number, accession_number, filing_date, formD.offeringData, ctx); // Related persons: indices 200–299 let relatedPersonIndex = 200; diff --git a/src/sec/forms/exempt-offerings/Form_D.test.ts b/src/sec/forms/exempt-offerings/Form_D.test.ts index 77c03fdf..22c5f5d7 100644 --- a/src/sec/forms/exempt-offerings/Form_D.test.ts +++ b/src/sec/forms/exempt-offerings/Form_D.test.ts @@ -73,6 +73,7 @@ describe("Form_D comprehensive storage test", () => { cik, file_number: fileNumber, accession_number: accessionNumber, + filing_date: "", primary_doc: file, formD, }); @@ -140,6 +141,7 @@ describe("Form_D comprehensive storage test", () => { cik, file_number: `file-${accessionNumber}`, accession_number: accessionNumber, + filing_date: "", primary_doc: "000175724818000002-primary_doc.xml", formD, }); @@ -167,6 +169,7 @@ describe("Form_D comprehensive storage test", () => { cik, file_number: `file-${accessionNumber}`, accession_number: accessionNumber, + filing_date: "", primary_doc: "000192959422000001-primary_doc.xml", formD, }); @@ -191,6 +194,52 @@ describe("Form_D comprehensive storage test", () => { expect(history?.total_amount_sold).toBe(900000); }); + it("does not regress the mutable offering row on an out-of-order older replay", async () => { + const xmlContent = readFileSync( + join(__dirname, "mock_data", "form-d", "000192959422000001-primary_doc.xml"), + "utf-8" + ); + const formD = await Form_D.parse("D", xmlContent); + const cik = parseInt(formD.primaryIssuer.cik); + const fileNumber = `file-stale-${cik}`; + + // The newer filing (by filing date) establishes the current row. + await processFormD({ + cik, + file_number: fileNumber, + accession_number: "acc-new", + filing_date: "2023-06-01", + primary_doc: "x.xml", + formD, + }); + + // A back-catalog older D/A for the same offering carrying different data + // is processed AFTER it (out of order). It must not clobber the row. + const olderFormD = structuredClone(formD); + olderFormD.offeringData.industryGroup.industryGroupType = "Pooled Investment Fund"; + await processFormD({ + cik, + file_number: fileNumber, + accession_number: "acc-old", + filing_date: "2023-01-01", + primary_doc: "x.xml", + formD: olderFormD, + }); + + // Mutable row still reflects the newer filing (industry_group not regressed). + const offering = await investmentOfferingRepo.getInvestmentOffering(cik, fileNumber); + expect(offering?.industry_group).toBe("Commercial"); + expect(offering?.as_of).toBe("2023-06-01"); + + // Both filings remain in the append-only history. + const histories = + await investmentOfferingRepo.getInvestmentOfferingHistoriesByCikAndFileNumber( + cik, + fileNumber + ); + expect(histories.map((h) => h.accession_number).sort()).toEqual(["acc-new", "acc-old"]); + }); + it("should store signature data correctly", async () => { const xmlContent = readFileSync( join(__dirname, "mock_data", "form-d", "000192959422000001-primary_doc.xml"), @@ -205,6 +254,7 @@ describe("Form_D comprehensive storage test", () => { cik, file_number: `file-${accessionNumber}`, accession_number: accessionNumber, + filing_date: "", primary_doc: "000192959422000001-primary_doc.xml", formD, }); @@ -233,6 +283,7 @@ describe("Form_D comprehensive storage test", () => { cik, file_number: `file-${accessionNumber}`, accession_number: accessionNumber, + filing_date: "", primary_doc: "000192959422000001-primary_doc.xml", formD, }); @@ -273,6 +324,7 @@ describe("Form_D comprehensive storage test", () => { cik, file_number: `file-${accessionNumber}`, accession_number: accessionNumber, + filing_date: "", primary_doc: "000175724718000001-primary_doc.xml", formD, }); diff --git a/src/storage/investment-offering/InvestmentOfferingSchema.ts b/src/storage/investment-offering/InvestmentOfferingSchema.ts index 81c5ecc2..20ddb90c 100644 --- a/src/storage/investment-offering/InvestmentOfferingSchema.ts +++ b/src/storage/investment-offering/InvestmentOfferingSchema.ts @@ -90,6 +90,15 @@ export const InvestmentOfferingSchema = Type.Object({ description: "Description of other security types if is_other_type is true", }) ), + as_of: TypeNullable( + Type.String({ + format: "date", + description: + "Filing date of the latest Form D/D-A reflected in this mutable row. " + + "Writes guard against out-of-order processing with it (skip a stale " + + "older filing); null when the filing date is unknown.", + }) + ), }); export type InvestmentOffering = Static; diff --git a/src/storage/portal/CrowdfundingHistorySchema.ts b/src/storage/portal/CrowdfundingHistorySchema.ts index b9f8aaea..e7a7c893 100644 --- a/src/storage/portal/CrowdfundingHistorySchema.ts +++ b/src/storage/portal/CrowdfundingHistorySchema.ts @@ -27,6 +27,14 @@ export const CrowdfundingHistorySchema = Type.Object({ description: "When this version ceased to be valid (null = current)", }) ), + accession_number: Type.String({ + maxLength: 25, + description: + "Accession of the filing that produced this version. Part of the PK so " + + "two snapshots for one CIK landing in the same wall-clock millisecond " + + "(e.g. a batch re-process) get distinct rows instead of colliding on " + + "(cik, valid_from) and silently dropping a history version.", + }), file_number: Type.String({ maxLength: 10, description: "File number", @@ -103,7 +111,7 @@ export type CrowdfundingHistory = Static; /** * Crowdfunding History repository storage type and primary key definitions */ -export const CrowdfundingHistoryPrimaryKeyNames = ["cik", "valid_from"] as const; +export const CrowdfundingHistoryPrimaryKeyNames = ["cik", "valid_from", "accession_number"] as const; export type CrowdfundingHistoryRepositoryStorage = ITabularStorage< typeof CrowdfundingHistorySchema, typeof CrowdfundingHistoryPrimaryKeyNames, diff --git a/src/storage/portal/CrowdfundingTemporalRepo.test.ts b/src/storage/portal/CrowdfundingTemporalRepo.test.ts index 0ca0299e..a8cfa97f 100644 --- a/src/storage/portal/CrowdfundingTemporalRepo.test.ts +++ b/src/storage/portal/CrowdfundingTemporalRepo.test.ts @@ -74,6 +74,56 @@ describe("CrowdfundingTemporalRepo", () => { }); }); + test("same-millisecond stale snapshots for one CIK don't collide (distinct accessions)", async () => { + const base: Crowdfunding = { + cik: 77777, + file_number: "020-77777", + filing_date: "2024-01-01", + name: "Same MS Co", + legal_status: "Corporation", + state_jurisdiction: "DE", + date_incorporation: "2020-01-01", + url: "http://example.com", + portal_cik: 11111, + status: "annual-report", + }; + + // Freeze the wall clock so both stale-replay snapshots get an identical + // valid_from — reproducing a batch re-process landing two filings of one + // CIK in the same millisecond. + const RealDate = Date; + const FIXED = RealDate.parse("2026-06-01T00:00:00.000Z"); + class FakeDate extends RealDate { + constructor(...args: ConstructorParameters | []) { + if (args.length === 0) super(FIXED); + else super(...(args as ConstructorParameters)); + } + static now(): number { + return FIXED; + } + } + globalThis.Date = FakeDate as DateConstructor; + try { + await temporalRepo.saveCrowdfundingWithHistory(base, "Form C-AR", { + skipMutableUpdate: true, + accessionNumber: "acc-A", + }); + await temporalRepo.saveCrowdfundingWithHistory(base, "Form C-TR", { + skipMutableUpdate: true, + accessionNumber: "acc-B", + }); + } finally { + globalThis.Date = RealDate; + } + + // Both snapshots survive. Without accession_number in the PK they would + // collide on (cik, valid_from) and the second would silently overwrite the + // first, losing a history version. + const rows = (await mockHistoryRepo.query({ cik: base.cik })) ?? []; + expect(rows.length).toBe(2); + expect(rows.map((r) => r.accession_number).sort()).toEqual(["acc-A", "acc-B"]); + }); + test("should save new crowdfunding entity with history and change log", async () => { const crowdfunding: Crowdfunding = { cik: 12345, @@ -309,6 +359,7 @@ describe("CrowdfundingTemporalRepo", () => { const earlier: CrowdfundingHistory = { cik, file_number: fileNumber, + accession_number: "acc-earlier", filing_date: "2024-01-01", name: "Earlier Snapshot", legal_status: "Corporation", diff --git a/src/storage/portal/CrowdfundingTemporalRepo.ts b/src/storage/portal/CrowdfundingTemporalRepo.ts index 65769ea7..6b0f624f 100644 --- a/src/storage/portal/CrowdfundingTemporalRepo.ts +++ b/src/storage/portal/CrowdfundingTemporalRepo.ts @@ -53,16 +53,20 @@ export class CrowdfundingTemporalRepo { async saveCrowdfundingWithHistory( crowdfunding: Crowdfunding, changeSource: string, - options?: { skipMutableUpdate?: boolean } | undefined, + options?: { skipMutableUpdate?: boolean; accessionNumber?: string } | undefined, batchId?: string ): Promise { const changeDate = new Date().toISOString(); + // Part of the history PK so same-millisecond snapshots for one CIK don't + // collide and drop a version. "" when unknown (callers should pass it). + const accession_number = options?.accessionNumber ?? ""; if (options?.skipMutableUpdate === true) { // Snapshot the older filing as a closed history row; do not touch the // mutable row, do not emit a ChangeLog entry. const history: CrowdfundingHistory = { ...crowdfunding, + accession_number, valid_from: changeDate, valid_to: changeDate, change_source: changeSource, @@ -95,6 +99,7 @@ export class CrowdfundingTemporalRepo { // Create new history record const history: CrowdfundingHistory = { ...crowdfunding, + accession_number, valid_from: changeDate, valid_to: null, change_source: changeSource, @@ -126,6 +131,7 @@ export class CrowdfundingTemporalRepo { // New entity - create initial history record const history: CrowdfundingHistory = { ...crowdfunding, + accession_number, valid_from: changeDate, valid_to: null, change_source: changeSource, @@ -208,7 +214,8 @@ export class CrowdfundingTemporalRepo { const validRecord = dominant ?? matches[0]; if (validRecord) { - const { change_source, change_date, valid_from, valid_to, ...crowdfunding } = validRecord; + const { change_source, change_date, valid_from, valid_to, accession_number, ...crowdfunding } = + validRecord; return crowdfunding as Crowdfunding; } From c513cf2374d406662a23f515f88ae8d6c41d6a28 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 25 Jun 2026 01:04:37 +0000 Subject: [PATCH 06/12] fix(resolver,task): reap phantom observations on re-extraction + idempotent junctions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observation rows are upserted by positional (accession, extractor_id, observation_index), so a re-extraction that yields FEWER entities — or reclassifies one company->person (a different table at the same index) — overwrites the live rows but never deletes the now-stale high-index / wrong-kind rows. Those orphans stay joined to canonical entities through their identity links, manufacturing entities the filing no longer describes (and a later `sec resolve` would re-link them). This violated the documented "replays are idempotent and order-safe" invariant for the observation/canonical tier. - ProcessAccessionDocFormTask captures runStart before extraction and, after a successful run, calls reapStaleObservations(accession, extractor_id, runStart): any observation for that filing+extractor not refreshed this run (created_at < runStart) is removed along with its identity links (all resolver versions), provenance, and — via each link — its address/phone junction contributions. Re-observed rows keep their stable observation_id, so the beneficial-ownership / related-party / spac_merger_extraction FKs that reference it stay valid. Centralized in the dispatcher, so no form-storage module changes. - EntityObserver now decrements an observation's prior junction contribution before re-recording, so a replay nets out to the same co-occurrence count instead of a blind +1 (the entangled junction-idempotency Major). - New repo primitives: junction removeObservation (decrement/delete); identity link listForObservation + deleteForObservation; provenance deleteForObservation; observation listByAccessionAndExtractor + deleteByObservationId. - Tests: a smaller re-extraction leaves no orphan observations/links/junctions; a company->person reclassification removes the stale-kind orphan; junction counts stay at 1 across repeated replays. All verified to fail when the respective fix is neutered. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/resolver/EntityObserver.ts | 72 +++++++++ src/resolver/reapStaleObservations.test.ts | 140 ++++++++++++++++++ src/resolver/reapStaleObservations.ts | 134 +++++++++++++++++ .../canonical/CanonicalCompanyAddressRepo.ts | 15 ++ .../canonical/CanonicalCompanyPhoneRepo.ts | 15 ++ .../canonical/CanonicalPersonAddressRepo.ts | 20 +++ .../canonical/CanonicalPersonPhoneRepo.ts | 15 ++ .../canonical/CompanyIdentityLinkRepo.ts | 14 ++ .../canonical/PersonIdentityLinkRepo.ts | 14 ++ .../observation/CompanyObservationRepo.ts | 12 ++ .../observation/PersonObservationRepo.ts | 12 ++ .../provenance/ObservationProvenanceRepo.ts | 5 + src/task/forms/ProcessAccessionDocFormTask.ts | 20 +++ 13 files changed, 488 insertions(+) create mode 100644 src/resolver/reapStaleObservations.test.ts create mode 100644 src/resolver/reapStaleObservations.ts diff --git a/src/resolver/EntityObserver.ts b/src/resolver/EntityObserver.ts index 59ec6c2c..af7148cc 100644 --- a/src/resolver/EntityObserver.ts +++ b/src/resolver/EntityObserver.ts @@ -76,6 +76,11 @@ export class EntityObserver { async observePerson( claim: PersonClaim ): Promise<{ canonical_person_id: string; observation_id: number }> { + // Re-observing this natural key would blindly +1 the address/phone + // co-occurrence counts. Remove the prior observation's contribution first + // so a replay nets out to the same count (idempotent) instead of inflating. + await this.removePriorPersonJunctions(claim); + // Normalize name parts into normalized fields const fullName = [claim.first_name, claim.middle_name, claim.last_name, claim.suffix] .filter(Boolean) @@ -145,6 +150,9 @@ export class EntityObserver { async observeCompany( claim: CompanyClaim ): Promise<{ canonical_company_id: string; observation_id: number }> { + // Idempotent replay: drop the prior contribution before re-recording. + await this.removePriorCompanyJunctions(claim); + const normalized_name = claim.name ? normalizeCompanyName(claim.name) : null; const now = new Date().toISOString(); @@ -197,4 +205,68 @@ export class EntityObserver { return { canonical_company_id, observation_id: upserted.observation_id }; } + + /** + * If an observation already exists for this natural key, decrement its + * address/phone junction contribution at the active resolver version (using + * its *prior* address/phone + its current link's canonical id) so the + * subsequent re-record nets out instead of double-counting. No-op on first + * sight or when the prior observation has no link at the active version. + */ + private async removePriorPersonJunctions(claim: PersonClaim): Promise { + const prior = await this.opts.personObservationRepo.getByNaturalKey( + claim.accession_number, + claim.extractor_id, + claim.observation_index + ); + if (!prior) return; + const link = await this.opts.personIdentityLinkRepo.getForObservation( + prior.observation_id, + this.opts.activeResolverPersonVersion + ); + if (!link) return; + if (prior.raw_address_id) { + await this.opts.canonicalPersonAddressRepo.removeObservation({ + canonical_person_id: link.canonical_person_id, + address_hash_id: prior.raw_address_id, + resolver_version: this.opts.activeResolverPersonVersion, + }); + } + if (prior.raw_phone_id) { + await this.opts.canonicalPersonPhoneRepo.removeObservation({ + canonical_person_id: link.canonical_person_id, + international_number: prior.raw_phone_id, + resolver_version: this.opts.activeResolverPersonVersion, + }); + } + } + + /** Company counterpart of {@link removePriorPersonJunctions}. */ + private async removePriorCompanyJunctions(claim: CompanyClaim): Promise { + const prior = await this.opts.companyObservationRepo.getByNaturalKey( + claim.accession_number, + claim.extractor_id, + claim.observation_index + ); + if (!prior) return; + const link = await this.opts.companyIdentityLinkRepo.getForObservation( + prior.observation_id, + this.opts.activeResolverCompanyVersion + ); + if (!link) return; + if (prior.raw_address_id) { + await this.opts.canonicalCompanyAddressRepo.removeObservation({ + canonical_company_id: link.canonical_company_id, + address_hash_id: prior.raw_address_id, + resolver_version: this.opts.activeResolverCompanyVersion, + }); + } + if (prior.raw_phone_id) { + await this.opts.canonicalCompanyPhoneRepo.removeObservation({ + canonical_company_id: link.canonical_company_id, + international_number: prior.raw_phone_id, + resolver_version: this.opts.activeResolverCompanyVersion, + }); + } + } } diff --git a/src/resolver/reapStaleObservations.test.ts b/src/resolver/reapStaleObservations.test.ts new file mode 100644 index 00000000..586cd182 --- /dev/null +++ b/src/resolver/reapStaleObservations.test.ts @@ -0,0 +1,140 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { beforeEach, describe, expect, it } from "bun:test"; +import { resetDependencyInjectionsForTesting } from "../config/TestingDI"; +import { buildEntityObserver } from "./buildEntityObserver"; +import { reapStaleObservations } from "./reapStaleObservations"; +import { PersonObservationRepo } from "../storage/observation/PersonObservationRepo"; +import { CompanyObservationRepo } from "../storage/observation/CompanyObservationRepo"; +import { PersonIdentityLinkRepo } from "../storage/canonical/PersonIdentityLinkRepo"; +import { CompanyIdentityLinkRepo } from "../storage/canonical/CompanyIdentityLinkRepo"; +import { CanonicalPersonAddressRepo } from "../storage/canonical/CanonicalPersonAddressRepo"; + +const V = "1.0.0"; +const ACC = "0001-25-000001"; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +function observer() { + return buildEntityObserver({ activeResolverPersonVersion: V, activeResolverCompanyVersion: V }); +} + +// Distinct CIKs so each resolves to its own canonical via the resolver CIK +// fast-path (a bare surname with no first name normalizes to all-null name +// fields, which would otherwise collapse them onto the shared issuer_cik). +function personClaim(index: number, addr: string) { + return { + accession_number: ACC, + extractor_id: "S-1", + extractor_version: "1.0.0", + observation_index: index, + cik: 1000 + index, + source_filing_issuer_cik: 999, + last_name: `Director${index}`, + address_id: addr, + }; +} + +describe("reapStaleObservations", () => { + beforeEach(() => { + resetDependencyInjectionsForTesting(); + }); + + it("reaps orphan observations a smaller re-extraction leaves behind, with their links + junctions", async () => { + const obs = observer(); + const personObs = new PersonObservationRepo(); + const links = new PersonIdentityLinkRepo(); + const addr = new CanonicalPersonAddressRepo(); + + // v1 extraction: three directors at indices 0,1,2. + await obs.observePerson(personClaim(0, "addr0")); + await obs.observePerson(personClaim(1, "addr1")); + const orphan = await obs.observePerson(personClaim(2, "addr2")); + + await sleep(5); + const runStart = new Date().toISOString(); + + // v2 re-extraction yields only two directors (indices 0,1). + await obs.observePerson(personClaim(0, "addr0")); + await obs.observePerson(personClaim(1, "addr1")); + + const { reaped } = await reapStaleObservations({ + accession_number: ACC, + extractor_id: "S-1", + before: runStart, + }); + + expect(reaped).toBe(1); + + // Orphan index 2 is gone; the live rows survive (with stable ids). + const remaining = await personObs.listByAccessionAndExtractor(ACC, "S-1"); + expect(remaining.map((o) => o.observation_index).sort()).toEqual([0, 1]); + + // The orphan's identity link is gone (no phantom canonical linkage). + expect(await links.listForObservation(orphan.observation_id)).toEqual([]); + + // The orphan's address junction is gone; the live ones remain at count 1. + expect(await addr.listForCanonical(orphan.canonical_person_id, V)).toEqual([]); + }); + + it("keeps junction counts idempotent across replays (no blind +1)", async () => { + const obs = observer(); + const addr = new CanonicalPersonAddressRepo(); + + const first = await obs.observePerson(personClaim(0, "addr0")); + // Re-observe the SAME natural key three more times (a replay loop). + await obs.observePerson(personClaim(0, "addr0")); + await obs.observePerson(personClaim(0, "addr0")); + await obs.observePerson(personClaim(0, "addr0")); + + const rows = await addr.listForCanonical(first.canonical_person_id, V); + expect(rows.length).toBe(1); + // Without the prior-contribution decrement this would be 4. + expect(rows[0].observation_count).toBe(1); + }); + + it("removes the stale-kind orphan when a reporting owner is reclassified company->person", async () => { + const obs = observer(); + const personObs = new PersonObservationRepo(); + const companyObs = new CompanyObservationRepo(); + const companyLinks = new CompanyIdentityLinkRepo(); + + // v1 classifies the owner at index 5 as a COMPANY. + const company = await obs.observeCompany({ + accession_number: ACC, + extractor_id: "ownership", + extractor_version: "1.0.0", + observation_index: 5, + name: "Smith Family Trust LLC", + }); + + await sleep(5); + const runStart = new Date().toISOString(); + + // v2 reclassifies the same index as a PERSON (different table). + await obs.observePerson({ + accession_number: ACC, + extractor_id: "ownership", + extractor_version: "1.0.0", + observation_index: 5, + source_filing_issuer_cik: 999, + last_name: "Smith Family Trust", + }); + + await reapStaleObservations({ + accession_number: ACC, + extractor_id: "ownership", + before: runStart, + }); + + // The stale company observation (and its link) at index 5 is gone... + expect(await companyObs.listByAccessionAndExtractor(ACC, "ownership")).toEqual([]); + expect(await companyLinks.listForObservation(company.observation_id)).toEqual([]); + // ...while the new person observation at the same index survives. + const people = await personObs.listByAccessionAndExtractor(ACC, "ownership"); + expect(people.map((o) => o.observation_index)).toEqual([5]); + }); +}); diff --git a/src/resolver/reapStaleObservations.ts b/src/resolver/reapStaleObservations.ts new file mode 100644 index 00000000..022f36df --- /dev/null +++ b/src/resolver/reapStaleObservations.ts @@ -0,0 +1,134 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { PersonObservationRepo } from "../storage/observation/PersonObservationRepo"; +import { CompanyObservationRepo } from "../storage/observation/CompanyObservationRepo"; +import { PersonIdentityLinkRepo } from "../storage/canonical/PersonIdentityLinkRepo"; +import { CompanyIdentityLinkRepo } from "../storage/canonical/CompanyIdentityLinkRepo"; +import { CanonicalPersonAddressRepo } from "../storage/canonical/CanonicalPersonAddressRepo"; +import { CanonicalPersonPhoneRepo } from "../storage/canonical/CanonicalPersonPhoneRepo"; +import { CanonicalCompanyAddressRepo } from "../storage/canonical/CanonicalCompanyAddressRepo"; +import { CanonicalCompanyPhoneRepo } from "../storage/canonical/CanonicalCompanyPhoneRepo"; +import { ObservationProvenanceRepo } from "../storage/provenance/ObservationProvenanceRepo"; + +export interface ReapStaleObservationsArgs { + readonly accession_number: string; + readonly extractor_id: string; + /** + * Run-start timestamp (ISO 8601). Observations for this filing+extractor whose + * `created_at` is strictly older were NOT re-observed by the current run, so + * they are stale orphans from a prior (larger / differently-classified) + * extraction and are reaped. `upsertByNaturalKey` refreshes `created_at` to + * the run time on every re-observe, so surviving rows keep their stable + * `observation_id` (and the beneficial-ownership / related-party / merger FKs + * that reference it stay valid). + */ + readonly before: string; +} + +export interface ReapStaleObservationsDeps { + personObservationRepo?: PersonObservationRepo; + companyObservationRepo?: CompanyObservationRepo; + personIdentityLinkRepo?: PersonIdentityLinkRepo; + companyIdentityLinkRepo?: CompanyIdentityLinkRepo; + canonicalPersonAddressRepo?: CanonicalPersonAddressRepo; + canonicalPersonPhoneRepo?: CanonicalPersonPhoneRepo; + canonicalCompanyAddressRepo?: CanonicalCompanyAddressRepo; + canonicalCompanyPhoneRepo?: CanonicalCompanyPhoneRepo; + observationProvenanceRepo?: ObservationProvenanceRepo; +} + +/** + * Remove the phantom rows a re-extraction leaves behind. Observation rows are + * upserted by positional `(accession, extractor_id, observation_index)`, so a + * re-extraction that yields FEWER entities — or reclassifies one from company + * to person (a different table at the same index) — overwrites the live rows + * but never deletes the now-stale high-index / wrong-kind rows. Those orphans + * stay joined to canonical entities through their identity links, manufacturing + * entities the filing no longer describes. + * + * Run AFTER a successful extraction. For each observation of (accession, + * extractor_id) not refreshed this run (`created_at < before`): decrement its + * address/phone junction contributions (via its links, so the count tracks live + * observations), delete its identity links (all resolver versions) and + * provenance, then delete the observation row. Re-observed rows are untouched. + */ +export async function reapStaleObservations( + args: ReapStaleObservationsArgs, + deps: ReapStaleObservationsDeps = {} +): Promise<{ reaped: number }> { + const personObs = deps.personObservationRepo ?? new PersonObservationRepo(); + const companyObs = deps.companyObservationRepo ?? new CompanyObservationRepo(); + const personLinks = deps.personIdentityLinkRepo ?? new PersonIdentityLinkRepo(); + const companyLinks = deps.companyIdentityLinkRepo ?? new CompanyIdentityLinkRepo(); + const personAddr = deps.canonicalPersonAddressRepo ?? new CanonicalPersonAddressRepo(); + const personPhone = deps.canonicalPersonPhoneRepo ?? new CanonicalPersonPhoneRepo(); + const companyAddr = deps.canonicalCompanyAddressRepo ?? new CanonicalCompanyAddressRepo(); + const companyPhone = deps.canonicalCompanyPhoneRepo ?? new CanonicalCompanyPhoneRepo(); + const provenance = deps.observationProvenanceRepo ?? new ObservationProvenanceRepo(); + + let reaped = 0; + + const people = await personObs.listByAccessionAndExtractor( + args.accession_number, + args.extractor_id + ); + for (const o of people) { + if (o.created_at >= args.before) continue; // refreshed this run — keep + const links = await personLinks.listForObservation(o.observation_id); + for (const link of links) { + if (o.raw_address_id) { + await personAddr.removeObservation({ + canonical_person_id: link.canonical_person_id, + address_hash_id: o.raw_address_id, + resolver_version: link.resolver_version, + }); + } + if (o.raw_phone_id) { + await personPhone.removeObservation({ + canonical_person_id: link.canonical_person_id, + international_number: o.raw_phone_id, + resolver_version: link.resolver_version, + }); + } + } + await personLinks.deleteForObservation(o.observation_id); + await provenance.deleteForObservation("person", o.observation_id); + await personObs.deleteByObservationId(o.observation_id); + reaped++; + } + + const companies = await companyObs.listByAccessionAndExtractor( + args.accession_number, + args.extractor_id + ); + for (const o of companies) { + if (o.created_at >= args.before) continue; // refreshed this run — keep + const links = await companyLinks.listForObservation(o.observation_id); + for (const link of links) { + if (o.raw_address_id) { + await companyAddr.removeObservation({ + canonical_company_id: link.canonical_company_id, + address_hash_id: o.raw_address_id, + resolver_version: link.resolver_version, + }); + } + if (o.raw_phone_id) { + await companyPhone.removeObservation({ + canonical_company_id: link.canonical_company_id, + international_number: o.raw_phone_id, + resolver_version: link.resolver_version, + }); + } + } + await companyLinks.deleteForObservation(o.observation_id); + await provenance.deleteForObservation("company", o.observation_id); + await companyObs.deleteByObservationId(o.observation_id); + reaped++; + } + + return { reaped }; +} diff --git a/src/storage/canonical/CanonicalCompanyAddressRepo.ts b/src/storage/canonical/CanonicalCompanyAddressRepo.ts index f29368b2..c7ce7fd2 100644 --- a/src/storage/canonical/CanonicalCompanyAddressRepo.ts +++ b/src/storage/canonical/CanonicalCompanyAddressRepo.ts @@ -57,6 +57,21 @@ export class CanonicalCompanyAddressRepo { return fresh; } + /** Remove one observation's contribution; see CanonicalPersonAddressRepo.removeObservation. */ + async removeObservation(pk: { + canonical_company_id: string; + address_hash_id: string; + resolver_version: string; + }): Promise { + const existing = await this.repo.get(pk); + if (!existing) return; + if (existing.observation_count <= 1) { + await this.repo.delete(pk); + return; + } + await this.repo.put({ ...existing, observation_count: existing.observation_count - 1 }); + } + async listForCanonical( canonical_company_id: string, resolver_version: string diff --git a/src/storage/canonical/CanonicalCompanyPhoneRepo.ts b/src/storage/canonical/CanonicalCompanyPhoneRepo.ts index adf6f2d6..7608db61 100644 --- a/src/storage/canonical/CanonicalCompanyPhoneRepo.ts +++ b/src/storage/canonical/CanonicalCompanyPhoneRepo.ts @@ -57,6 +57,21 @@ export class CanonicalCompanyPhoneRepo { return fresh; } + /** Remove one observation's contribution; see CanonicalPersonAddressRepo.removeObservation. */ + async removeObservation(pk: { + canonical_company_id: string; + international_number: string; + resolver_version: string; + }): Promise { + const existing = await this.repo.get(pk); + if (!existing) return; + if (existing.observation_count <= 1) { + await this.repo.delete(pk); + return; + } + await this.repo.put({ ...existing, observation_count: existing.observation_count - 1 }); + } + async listForCanonical( canonical_company_id: string, resolver_version: string diff --git a/src/storage/canonical/CanonicalPersonAddressRepo.ts b/src/storage/canonical/CanonicalPersonAddressRepo.ts index 10c69361..e16f8d33 100644 --- a/src/storage/canonical/CanonicalPersonAddressRepo.ts +++ b/src/storage/canonical/CanonicalPersonAddressRepo.ts @@ -57,6 +57,26 @@ export class CanonicalPersonAddressRepo { return fresh; } + /** + * Remove one observation's contribution: decrement the co-occurrence count, + * deleting the row when it reaches zero. The inverse of {@link recordObservation}, + * used when an observation is reaped (orphan) or re-observed (idempotent replay) + * so the count tracks live observations rather than blindly accumulating. + */ + async removeObservation(pk: { + canonical_person_id: string; + address_hash_id: string; + resolver_version: string; + }): Promise { + const existing = await this.repo.get(pk); + if (!existing) return; + if (existing.observation_count <= 1) { + await this.repo.delete(pk); + return; + } + await this.repo.put({ ...existing, observation_count: existing.observation_count - 1 }); + } + async listForCanonical( canonical_person_id: string, resolver_version: string diff --git a/src/storage/canonical/CanonicalPersonPhoneRepo.ts b/src/storage/canonical/CanonicalPersonPhoneRepo.ts index 7a9c5cf7..a5a0e8fb 100644 --- a/src/storage/canonical/CanonicalPersonPhoneRepo.ts +++ b/src/storage/canonical/CanonicalPersonPhoneRepo.ts @@ -57,6 +57,21 @@ export class CanonicalPersonPhoneRepo { return fresh; } + /** Remove one observation's contribution; see CanonicalPersonAddressRepo.removeObservation. */ + async removeObservation(pk: { + canonical_person_id: string; + international_number: string; + resolver_version: string; + }): Promise { + const existing = await this.repo.get(pk); + if (!existing) return; + if (existing.observation_count <= 1) { + await this.repo.delete(pk); + return; + } + await this.repo.put({ ...existing, observation_count: existing.observation_count - 1 }); + } + async listForCanonical( canonical_person_id: string, resolver_version: string diff --git a/src/storage/canonical/CompanyIdentityLinkRepo.ts b/src/storage/canonical/CompanyIdentityLinkRepo.ts index 5538b359..a5ed0517 100644 --- a/src/storage/canonical/CompanyIdentityLinkRepo.ts +++ b/src/storage/canonical/CompanyIdentityLinkRepo.ts @@ -52,6 +52,20 @@ export class CompanyIdentityLinkRepo { return await this.repo.get({ observation_id, resolver_version }); } + /** All links for an observation across resolver versions. */ + async listForObservation(observation_id: number): Promise { + return (await this.repo.query({ observation_id })) ?? []; + } + + /** Delete every link for an observation (all resolver versions) — used when the + * underlying observation row is reaped so no link is left dangling. */ + async deleteForObservation(observation_id: number): Promise { + const rows = (await this.repo.query({ observation_id })) ?? []; + for (const r of rows) { + await this.repo.delete({ observation_id: r.observation_id, resolver_version: r.resolver_version }); + } + } + async listForCanonical( canonical_company_id: string, resolver_version: string diff --git a/src/storage/canonical/PersonIdentityLinkRepo.ts b/src/storage/canonical/PersonIdentityLinkRepo.ts index c63ff4d7..6d5cdebe 100644 --- a/src/storage/canonical/PersonIdentityLinkRepo.ts +++ b/src/storage/canonical/PersonIdentityLinkRepo.ts @@ -52,6 +52,20 @@ export class PersonIdentityLinkRepo { return await this.repo.get({ observation_id, resolver_version }); } + /** All links for an observation across resolver versions. */ + async listForObservation(observation_id: number): Promise { + return (await this.repo.query({ observation_id })) ?? []; + } + + /** Delete every link for an observation (all resolver versions) — used when the + * underlying observation row is reaped so no link is left dangling. */ + async deleteForObservation(observation_id: number): Promise { + const rows = (await this.repo.query({ observation_id })) ?? []; + for (const r of rows) { + await this.repo.delete({ observation_id: r.observation_id, resolver_version: r.resolver_version }); + } + } + async listForCanonical( canonical_person_id: string, resolver_version: string diff --git a/src/storage/observation/CompanyObservationRepo.ts b/src/storage/observation/CompanyObservationRepo.ts index 09737745..54172efa 100644 --- a/src/storage/observation/CompanyObservationRepo.ts +++ b/src/storage/observation/CompanyObservationRepo.ts @@ -130,6 +130,18 @@ export class CompanyObservationRepo { return rows.sort((a, b) => a.observation_index - b.observation_index); } + /** All observations for one filing + extractor (the reaping scope). */ + async listByAccessionAndExtractor( + accession_number: string, + extractor_id: string + ): Promise { + return (await this.repo.query({ accession_number, extractor_id })) ?? []; + } + + async deleteByObservationId(observation_id: number): Promise { + await this.repo.delete({ observation_id }); + } + async listAll(): Promise { return (await this.repo.getAll()) ?? []; } diff --git a/src/storage/observation/PersonObservationRepo.ts b/src/storage/observation/PersonObservationRepo.ts index 4b9a50fe..2677f18b 100644 --- a/src/storage/observation/PersonObservationRepo.ts +++ b/src/storage/observation/PersonObservationRepo.ts @@ -147,6 +147,18 @@ export class PersonObservationRepo { return rows.sort((a, b) => a.observation_index - b.observation_index); } + /** All observations for one filing + extractor (the reaping scope). */ + async listByAccessionAndExtractor( + accession_number: string, + extractor_id: string + ): Promise { + return (await this.repo.query({ accession_number, extractor_id })) ?? []; + } + + async deleteByObservationId(observation_id: number): Promise { + await this.repo.delete({ observation_id }); + } + async listAll(): Promise { return (await this.repo.getAll()) ?? []; } diff --git a/src/storage/provenance/ObservationProvenanceRepo.ts b/src/storage/provenance/ObservationProvenanceRepo.ts index a86b18d8..f6544739 100644 --- a/src/storage/provenance/ObservationProvenanceRepo.ts +++ b/src/storage/provenance/ObservationProvenanceRepo.ts @@ -33,6 +33,11 @@ export class ObservationProvenanceRepo { return this.storage.get({ kind, observation_id }); } + /** Delete the provenance row for a reaped observation (no-op when absent). */ + async deleteForObservation(kind: "person" | "company", observation_id: number): Promise { + await this.storage.delete({ kind, observation_id }); + } + /** * Audit helper: every row whose confidence is non-null and below the floor. * Scans all rows — not a hot path. diff --git a/src/task/forms/ProcessAccessionDocFormTask.ts b/src/task/forms/ProcessAccessionDocFormTask.ts index 1d02908a..0c776764 100644 --- a/src/task/forms/ProcessAccessionDocFormTask.ts +++ b/src/task/forms/ProcessAccessionDocFormTask.ts @@ -37,6 +37,7 @@ import { EXTRACTOR_RUN_REPOSITORY_TOKEN } from "../../storage/versioning/Extract import { formToExtractorId } from "../../storage/versioning/extractorIds"; import { getActiveSlot } from "../../storage/versioning/getActiveSlot"; import { VersionRegistry } from "../../storage/versioning/VersionRegistry"; +import { reapStaleObservations } from "../../resolver/reapStaleObservations"; import { SecFetchAccessionDocTask } from "./SecFetchAccessionDocTask"; /** @@ -290,6 +291,9 @@ export class ProcessAccessionDocFormTask extends Task< } // --- Domain 3: parse + store (hard error -> record + rethrow, unchanged) --- + // Captured before any observe so the post-run reap can tell rows this run + // refreshed (created_at >= runStart) from stale orphans of a prior run. + const runStart = new Date().toISOString(); let parseError: unknown = undefined; try { const formCls = ALL_FORMS_MAP.get(form!); @@ -411,6 +415,22 @@ export class ProcessAccessionDocFormTask extends Task< dlErr ); } + // Remove observation rows this re-extraction superseded (a smaller or + // reclassified entity set leaves stale orphans joined to canonical + // entities). Best-effort, like recordRun — a reaper hiccup must not mask + // the successful extraction. + try { + await reapStaleObservations({ + accession_number: accessionNumber, + extractor_id: extractorId, + before: runStart, + }); + } catch (reapErr) { + console.error( + `Failed to reap stale observations for ${accessionNumber}@${extractorId}:`, + reapErr + ); + } try { await runRepo.recordRun({ cik: cik!, From 30258058d4ca670aea6b081f9748b9c9a822837a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 25 Jun 2026 01:13:16 +0000 Subject: [PATCH 07/12] fix(spac): backfill merger proxies gated before their spac row existed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit processMergerProxy gates on a known SPAC and silently no-ops when the `spac` row does not yet exist (e.g. a DEFM14A ingested before its S-1). The dispatcher then records a success:true run, so the normal unprocessed-run sweep excludes the filing forever — the merger target / PIPE for that deal is dropped, and unlike the analogous redemption gate there was no recovery path. Add BackfillMergerProxiesTask + `sec spac backfill-merger-proxies`, mirroring BackfillRedemptionsTask: enumerate known-SPAC merger proxies (the 14A/14C merger + revised-proxy forms, derived from FORM_TO_EXTRACTOR_ID) from the bootstrapped filing metadata and re-process those that still lack a spac_merger_extraction row, so they extract now that a spac row exists. Skipping already-extracted proxies avoids redundant AI re-runs. Per-filing failures are isolated. Adds a selection test and documents the command in CLAUDE.md. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- CLAUDE.md | 7 ++ src/commands/spac.ts | 19 ++- .../spac/BackfillMergerProxiesTask.test.ts | 94 +++++++++++++++ src/task/spac/BackfillMergerProxiesTask.ts | 108 ++++++++++++++++++ 4 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 src/task/spac/BackfillMergerProxiesTask.test.ts create mode 100644 src/task/spac/BackfillMergerProxiesTask.ts diff --git a/CLAUDE.md b/CLAUDE.md index 7dca3e2b..0b453810 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -205,8 +205,15 @@ model via `SEC_MERGER_PROXY_MODEL` (default `claude-sonnet-4-6`) and an optional confidence floor via `SEC_MERGER_PROXY_CONFIDENCE_FLOOR` (falls back to the shared `SEC_S1_CONFIDENCE_FLOOR` when unset). +A proxy ingested before its issuer's `spac` row exists (e.g. the S-1 lands later) +hits the known-SPAC gate and no-ops — recording a successful run, so the normal +unprocessed-run sweep never revisits it. `sec spac backfill-merger-proxies` +recovers these: it re-processes known-SPAC merger proxies that still lack a +`spac_merger_extraction` row (mirroring `backfill-redemptions`). + ```bash sec fetch form DEFM14A # fetch + extract a merger proxy +sec spac backfill-merger-proxies # recover proxies gated before their spac row existed sec extractor dead-letters merger-proxy # version-fixable extraction failures sec extractor retry-dead-letters merger-proxy ``` diff --git a/src/commands/spac.ts b/src/commands/spac.ts index b1fea2e3..86633b95 100644 --- a/src/commands/spac.ts +++ b/src/commands/spac.ts @@ -11,6 +11,7 @@ import { SpacRepo } from "../storage/spac/SpacRepo"; import { SPAC_SPONSOR_LINK_REPOSITORY_TOKEN } from "../storage/canonical/SpacSponsorLinkSchema"; import { UNDERWRITER_LINK_REPOSITORY_TOKEN } from "../storage/canonical/UnderwriterLinkSchema"; import { BackfillRedemptionsTask } from "../task/spac/BackfillRedemptionsTask"; +import { BackfillMergerProxiesTask } from "../task/spac/BackfillMergerProxiesTask"; export interface SpacReport { readonly cik: number; @@ -26,7 +27,10 @@ export interface SpacReport { * Sponsor and underwriter counts are obtained by querying the link tables by issuer_cik, * since neither SpacSponsorLinkRepo nor UnderwriterLinkRepo exposes a listByIssuer method. */ -export async function assembleSpacReport(cik: number, repo: SpacRepo = new SpacRepo()): Promise { +export async function assembleSpacReport( + cik: number, + repo: SpacRepo = new SpacRepo() +): Promise { const [spac, deals, events] = await Promise.all([ repo.getSpac(cik), repo.getDeals(cik), @@ -123,4 +127,17 @@ export function registerSpacCommands(program: Command): void { }; console.log(`selected ${out.selected} filing(s); processed ${out.processed}`); }); + + spacCmd + .command("backfill-merger-proxies") + .description( + "Re-process known-SPAC merger proxies that were ingested before their spac row existed" + ) + .action(async () => { + const out = (await withCli(new BackfillMergerProxiesTask()).run({})) as { + selected: number; + processed: number; + }; + console.log(`selected ${out.selected} filing(s); processed ${out.processed}`); + }); } diff --git a/src/task/spac/BackfillMergerProxiesTask.test.ts b/src/task/spac/BackfillMergerProxiesTask.test.ts new file mode 100644 index 00000000..f8682051 --- /dev/null +++ b/src/task/spac/BackfillMergerProxiesTask.test.ts @@ -0,0 +1,94 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ +import { beforeEach, describe, expect, it } from "bun:test"; +import { globalServiceRegistry } from "workglow"; +import { resetDependencyInjectionsForTesting } from "../../config/TestingDI"; +import { setupAllDatabases } from "../../config/setupAllDatabases"; +import { FILING_REPOSITORY_TOKEN } from "../../storage/filing/FilingSchema"; +import { SpacReportWriter } from "../../storage/spac/SpacReportWriter"; +import { SpacMergerExtractionRepo } from "../../storage/spac/SpacMergerExtractionRepo"; +import { selectMergerProxyBackfillAccessions } from "./BackfillMergerProxiesTask"; + +async function seedSpac(cik: number): Promise { + await new SpacReportWriter().recordRegistration({ + cik, + accession_number: `${cik}-reg`, + filing_date: "2025-12-01", + form: "S-1", + primary_document: "s1.htm", + spac_name: "Backfill SPAC Inc.", + spac_sic: 6770, + }); +} + +async function seedFiling(opts: { + readonly cik: number; + readonly accession_number: string; + readonly form: string; +}): Promise { + const repo = globalServiceRegistry.get(FILING_REPOSITORY_TOKEN); + await repo.put({ + cik: opts.cik, + accession_number: opts.accession_number, + form: opts.form, + primary_doc: "primary.htm", + file_number: "", + filing_date: "2026-03-20", + acceptance_date: "2026-03-20T00:00:00.000Z", + report_date: "2026-03-19", + film_number: null, + primary_doc_description: null, + size: null, + is_xbrl: null, + is_inline_xbrl: null, + items: "", + act: null, + } as never); +} + +async function seedExtraction(accession_number: string, cik: number, form: string): Promise { + await new SpacMergerExtractionRepo().save({ + accession_number, + cik, + form, + filing_date: "2026-03-20", + extractor_id: "merger-proxy", + extractor_version: "1.0.0", + target_name: "Acme Target Inc.", + target_cik: null, + target_observation_id: null, + pipe_amount: null, + merger_consideration: null, + confidence: 0.9, + source_span: null, + model_id: null, + created_at: new Date().toISOString(), + }); +} + +describe("selectMergerProxyBackfillAccessions", () => { + beforeEach(async () => { + resetDependencyInjectionsForTesting(); + await setupAllDatabases(); + }); + + it("selects known-SPAC merger proxies lacking an extraction row only", async () => { + await seedSpac(5); + // Known SPAC, no extraction yet -> selected (these are the gated filings). + await seedFiling({ cik: 5, accession_number: "acc-defm", form: "DEFM14A" }); + await seedFiling({ cik: 5, accession_number: "acc-prem", form: "PREM14A" }); + // Known SPAC merger proxy that was already extracted -> skipped. + await seedFiling({ cik: 5, accession_number: "acc-done", form: "DEFM14C" }); + await seedExtraction("acc-done", 5, "DEFM14C"); + // Known SPAC, but not a merger-proxy form -> not selected. + await seedFiling({ cik: 5, accession_number: "acc-10k", form: "10-K" }); + // Merger proxy for a CIK with no spac row -> not selected. + await seedFiling({ cik: 6, accession_number: "acc-nonspac", form: "DEFM14A" }); + + const accessions = await selectMergerProxyBackfillAccessions(); + expect(accessions.sort()).toEqual(["acc-defm", "acc-prem"]); + }); +}); diff --git a/src/task/spac/BackfillMergerProxiesTask.ts b/src/task/spac/BackfillMergerProxiesTask.ts new file mode 100644 index 00000000..8d17cc75 --- /dev/null +++ b/src/task/spac/BackfillMergerProxiesTask.ts @@ -0,0 +1,108 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ +import { Static, Type } from "typebox"; +import { globalServiceRegistry, IExecuteContext, Task, Workflow } from "workglow"; +import { FILING_REPOSITORY_TOKEN } from "../../storage/filing/FilingSchema"; +import { SpacRepo } from "../../storage/spac/SpacRepo"; +import { SpacMergerExtractionRepo } from "../../storage/spac/SpacMergerExtractionRepo"; +import { FORM_TO_EXTRACTOR_ID } from "../../storage/versioning/extractorIds"; +import { ProcessAccessionDocFormTask } from "../forms/ProcessAccessionDocFormTask"; + +/** The 14A/14C merger + revised-proxy forms routed to the `merger-proxy` extractor. */ +const MERGER_PROXY_FORMS = Object.entries(FORM_TO_EXTRACTOR_ID) + .filter(([, id]) => id === "merger-proxy") + .map(([form]) => form); + +/** + * Accession numbers of known-SPAC merger proxies that have no extraction row + * yet, enumerated from the bootstrapped `filing` metadata (no network discovery). + * + * `processMergerProxy` gates on a known SPAC and silently no-ops (recording a + * `success: true` run) when the `spac` row does not yet exist — e.g. a proxy + * ingested before its S-1. Those filings are then excluded from the normal + * unprocessed-run sweep forever, so the merger target / PIPE is dropped. This + * recovers them once a `spac` row appears. Filings that already produced a + * `spac_merger_extraction` row are skipped to avoid redundant AI re-runs. + */ +export async function selectMergerProxyBackfillAccessions(): Promise { + const filingRepo = globalServiceRegistry.get(FILING_REPOSITORY_TOKEN); + const spacRepo = new SpacRepo(); + const extractions = new SpacMergerExtractionRepo(); + const out: string[] = []; + const spacs = await spacRepo.getAllSpacs(); + for (const spac of spacs) { + // Query by (form, cik) — the filings storage is indexed on ["form", "cik"], + // so this loads only the SPAC's proxies instead of scanning all its filings. + for (const form of MERGER_PROXY_FORMS) { + const filings = (await filingRepo.query({ form, cik: spac.cik })) ?? []; + for (const f of filings) { + const already = await extractions.getByAccession(f.accession_number); + if (!already) out.push(f.accession_number); + } + } + } + return out; +} + +const InputSchema = () => + Type.Object({ + dryRun: Type.Optional(Type.Boolean({ default: false })), + }); +export type BackfillMergerProxiesTaskInput = Static>; + +const OutputSchema = () => + Type.Object({ + selected: Type.Number(), + processed: Type.Number(), + }); +type BackfillMergerProxiesTaskOutput = Static>; + +/** + * Sweeps historical known-SPAC merger proxies that were never extracted (because + * the SPAC was not yet known at ingestion) and re-runs + * {@link ProcessAccessionDocFormTask} for each so the merger-proxy extractor + * runs now that a `spac` row exists. + */ +export class BackfillMergerProxiesTask extends Task< + BackfillMergerProxiesTaskInput, + BackfillMergerProxiesTaskOutput +> { + static readonly type = "BackfillMergerProxiesTask"; + static readonly category = "SEC"; + static readonly cacheable = false; + + static inputSchema() { + return InputSchema(); + } + + static outputSchema() { + return OutputSchema(); + } + + async execute( + input: BackfillMergerProxiesTaskInput, + context: IExecuteContext + ): Promise { + const accessions = await selectMergerProxyBackfillAccessions(); + if (input.dryRun) { + return { selected: accessions.length, processed: 0 }; + } + // Isolate per-filing failures: one bad proxy (fetch error, malformed body) + // must not abort the sweep over the remaining accessions. + let processed = 0; + for (const accessionNumber of accessions) { + try { + const wf = context.own(new Workflow()); + wf.pipe(new ProcessAccessionDocFormTask()); + await wf.run({ accessionNumber }); + processed++; + } catch (err) { + console.error(`backfill-merger-proxies: failed to reprocess ${accessionNumber}:`, err); + } + } + return { selected: accessions.length, processed }; + } +} From 2bb28e493a77b9eacb0ea24343b7839ddd9a401f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 25 Jun 2026 01:23:36 +0000 Subject: [PATCH 08/12] fix(xbrl): write-first replace so a failed re-extract can't wipe a filing's facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit XbrlFactRepo.replaceForAccession did deleteSearch then putBulk with no transaction, so a putBulk rejection (e.g. a schema/maxLength overflow on one row aborting the batch) after the delete left the filing with ZERO facts — and the "never throws" caller (extractAndStoreXbrl) swallowed it as a NO_XBRL success, masking the loss. Reorder to write-first: putBulk the new rows (upsert by the (accession_number, fact_index) PK), THEN delete only the rows a prior longer extract left whose fact_index the new set omits. A putBulk failure now throws before any delete, leaving the prior facts intact; the storage abstraction exposes no transaction, so the worst residual case (a failure during the stale-tail delete) is a superset the next re-extract cleans up — never data loss. The existing stale-row-clearing test still holds; a new test pins that a putBulk rejection keeps the prior facts (it fails under the old delete-first order with count 0). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/storage/xbrl/XbrlFactRepo.test.ts | 35 ++++++++++++++++++++++++++- src/storage/xbrl/XbrlFactRepo.ts | 25 ++++++++++++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/storage/xbrl/XbrlFactRepo.test.ts b/src/storage/xbrl/XbrlFactRepo.test.ts index 540a5077..9ac085be 100644 --- a/src/storage/xbrl/XbrlFactRepo.test.ts +++ b/src/storage/xbrl/XbrlFactRepo.test.ts @@ -8,8 +8,13 @@ import { beforeEach, describe, expect, it } from "bun:test"; import { resetDependencyInjectionsForTesting } from "../../config/TestingDI"; import { parseInlineXbrl } from "../../sec/xbrl/parseInlineXbrl"; import { toXbrlFactRows } from "../../sec/xbrl/toFactRows"; +import { globalServiceRegistry } from "workglow"; import { XbrlFactRepo } from "./XbrlFactRepo"; -import type { XbrlFactRow } from "./XbrlFactSchema"; +import { + XBRL_FACT_REPOSITORY_TOKEN, + type XbrlFactRow, + type XbrlFactRepositoryStorage, +} from "./XbrlFactSchema"; const ACCESSION = "0001213900-26-039320"; @@ -63,6 +68,34 @@ describe("XbrlFactRepo", () => { expect(await repo.countByAccession(ACCESSION)).toBe(1); }); + it("keeps the prior facts when a re-extract's putBulk fails (no zero-facts window)", async () => { + // Seed a complete prior extract. + await new XbrlFactRepo().replaceForAccession(ACCESSION, rowsFromInline()); + expect(await new XbrlFactRepo().countByAccession(ACCESSION)).toBe(2); + + // A re-extract whose putBulk rejects (e.g. a maxLength overflow on one row) + // must not have already deleted the prior facts. + const real = globalServiceRegistry.get(XBRL_FACT_REPOSITORY_TOKEN); + const failingPutBulk = { + putBulk: async () => { + throw new Error("simulated putBulk rejection"); + }, + query: (criteria: unknown) => (real.query as (c: unknown) => unknown)(criteria), + delete: (pk: unknown) => (real.delete as (p: unknown) => unknown)(pk), + // Delegated so the OLD delete-then-put order would genuinely wipe the prior + // facts here (and fail the count assertion below), not error on a missing method. + deleteSearch: (c: unknown) => (real.deleteSearch as (c: unknown) => unknown)(c), + } as unknown as XbrlFactRepositoryStorage; + + await expect( + new XbrlFactRepo(failingPutBulk).replaceForAccession(ACCESSION, rowsFromInline()) + ).rejects.toThrow("simulated putBulk rejection"); + + // The prior facts are still there — the old delete-then-put order would have + // wiped them before the put failed. + expect(await new XbrlFactRepo().countByAccession(ACCESSION)).toBe(2); + }); + it("queries one concept across an issuer's filings", async () => { const repo = new XbrlFactRepo(); await repo.replaceForAccession(ACCESSION, rowsFromInline()); diff --git a/src/storage/xbrl/XbrlFactRepo.ts b/src/storage/xbrl/XbrlFactRepo.ts index 9aa4023c..ab4bcece 100644 --- a/src/storage/xbrl/XbrlFactRepo.ts +++ b/src/storage/xbrl/XbrlFactRepo.ts @@ -19,13 +19,30 @@ export class XbrlFactRepo { } /** - * Idempotent re-extract: clears any facts already stored for the filing - * (a re-run may yield fewer facts, so stale high indexes must go) and bulk - * writes the new rows. + * Idempotent re-extract that never leaves the filing with zero facts. Writes + * the new rows FIRST (an upsert by the `(accession_number, fact_index)` PK), + * THEN deletes any rows a prior, longer extract left whose `fact_index` the + * new set does not include. + * + * The previous order — delete-all, then `putBulk` — wiped every fact for the + * filing if the `putBulk` then failed (e.g. a schema/maxLength rejection of one + * row aborts the batch), and the "never throws" caller masked that loss as a + * `NO_XBRL` success. Writing first means a `putBulk` failure throws before any + * delete, leaving the prior facts intact; the stale-tail delete only runs once + * the new rows are committed. There is no surrounding transaction (the storage + * abstraction exposes none), so the worst residual case — a failure during the + * stale-tail delete — leaves a superset of facts, which the next re-extract + * cleans up, rather than data loss. */ async replaceForAccession(accession_number: string, rows: readonly XbrlFactRow[]): Promise { - await this.storage.deleteSearch({ accession_number }); if (rows.length > 0) await this.storage.putBulk([...rows]); + const keep = new Set(rows.map((r) => r.fact_index)); + const existing = (await this.storage.query({ accession_number })) ?? []; + for (const r of existing) { + if (!keep.has(r.fact_index)) { + await this.storage.delete({ accession_number, fact_index: r.fact_index }); + } + } } /** All facts for a filing in extraction order. */ From 3bcf02f8a1b0efcb2ae90a74834429b1db27bf3e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Jun 2026 20:24:23 +0000 Subject: [PATCH 09/12] fix(forms,resolver): don't reap a section's observations when it failed this run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reapStaleObservations runs in ProcessAccessionDocFormTask's success branch and deletes any observation for (accession, extractor) not refreshed this run. But sectionRunner swallows a section's zero-row outcomes into a dead-letter WITHOUT aborting the filing, so the processor still returns success. A section that transiently fails on a re-extraction (network blip / rate limit / empty or truncated model response) writes nothing, and the reaper then deletes that section's still-valid prior observations (plus identity links and junction counts) as false orphans — silent data loss. Gate the reap on a new hasBlockingSectionFailure() policy: skip it whenever a section recorded a fresh dead-letter THIS run for any reason that means it yielded no rows (MODEL_INVALID_OUTPUT, MODEL_EMPTY, LOW_CONFIDENCE_ALL, a whole-section UNVERIFIED_SOURCE_SPAN). A genuinely-absent section (SECTION_NOT_FOUND) and a partial success (`-partial` marker) do not block, so the reaper still removes real orphans on a clean run. If the dead-letter check itself errors, default to NOT reaping — retaining a stale superset is recoverable; deleting valid rows is not. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/resolver/reapGate.test.ts | 81 ++++++ src/resolver/reapStaleObservations.ts | 45 +++ ...ocessAccessionDocFormTask.reapgate.test.ts | 265 ++++++++++++++++++ src/task/forms/ProcessAccessionDocFormTask.ts | 51 +++- 4 files changed, 433 insertions(+), 9 deletions(-) create mode 100644 src/resolver/reapGate.test.ts create mode 100644 src/task/forms/ProcessAccessionDocFormTask.reapgate.test.ts diff --git a/src/resolver/reapGate.test.ts b/src/resolver/reapGate.test.ts new file mode 100644 index 00000000..2ef535ab --- /dev/null +++ b/src/resolver/reapGate.test.ts @@ -0,0 +1,81 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from "bun:test"; +import { hasBlockingSectionFailure, type ReapGateDeadLetter } from "./reapStaleObservations"; + +const ACC = "0000000000-26-000001"; +const RUN_START = "2026-06-27T12:00:00.000Z"; +const FRESH = "2026-06-27T12:00:05.000Z"; // >= RUN_START +const STALE = "2026-06-26T00:00:00.000Z"; // < RUN_START + +function dl(partial: Partial): ReapGateDeadLetter { + return { + accession_number: ACC, + reason_code: "MODEL_INVALID_OUTPUT", + section_name: "management", + last_attempt_at: FRESH, + ...partial, + }; +} + +describe("hasBlockingSectionFailure", () => { + it("blocks on every fresh zero-row failure code", () => { + for (const reason_code of [ + "MODEL_INVALID_OUTPUT", + "MODEL_EMPTY", + "LOW_CONFIDENCE_ALL", + "UNVERIFIED_SOURCE_SPAN", + ]) { + expect(hasBlockingSectionFailure([dl({ reason_code })], ACC, RUN_START)).toBe(true); + } + }); + + it("does NOT block on a genuinely-absent section (SECTION_NOT_FOUND)", () => { + // Common case — most filings lack some optional section; blocking here would + // disable the reaper entirely. + expect( + hasBlockingSectionFailure([dl({ reason_code: "SECTION_NOT_FOUND" })], ACC, RUN_START) + ).toBe(false); + }); + + it("does NOT block on a partial-success marker (section persisted its rows)", () => { + expect( + hasBlockingSectionFailure( + [dl({ reason_code: "UNVERIFIED_SOURCE_SPAN", section_name: "underwriters-partial" })], + ACC, + RUN_START + ) + ).toBe(false); + }); + + it("ignores stale dead-letters from a prior run", () => { + // A persistently-absent section must not pin the reaper forever once the + // failure is no longer being re-recorded this run. + expect( + hasBlockingSectionFailure([dl({ last_attempt_at: STALE })], ACC, RUN_START) + ).toBe(false); + }); + + it("ignores dead-letters for a different accession", () => { + expect( + hasBlockingSectionFailure([dl({ accession_number: "9999999999-99-999999" })], ACC, RUN_START) + ).toBe(false); + }); + + it("blocks when ANY fresh failing section is present among benign ones", () => { + const entries = [ + dl({ reason_code: "SECTION_NOT_FOUND", section_name: "underwriting" }), + dl({ reason_code: "UNVERIFIED_SOURCE_SPAN", section_name: "management-partial" }), + dl({ reason_code: "MODEL_EMPTY", section_name: "related-party" }), + ]; + expect(hasBlockingSectionFailure(entries, ACC, RUN_START)).toBe(true); + }); + + it("returns false for an empty dead-letter set (clean run)", () => { + expect(hasBlockingSectionFailure([], ACC, RUN_START)).toBe(false); + }); +}); diff --git a/src/resolver/reapStaleObservations.ts b/src/resolver/reapStaleObservations.ts index 022f36df..ddeb3bf0 100644 --- a/src/resolver/reapStaleObservations.ts +++ b/src/resolver/reapStaleObservations.ts @@ -29,6 +29,51 @@ export interface ReapStaleObservationsArgs { readonly before: string; } +/** Minimal shape of a dead-letter the reap gate inspects. */ +export interface ReapGateDeadLetter { + readonly accession_number: string; + readonly reason_code: string; + readonly section_name: string; + readonly last_attempt_at: string; +} + +/** + * Whether the just-finished run recorded a dead-letter meaning a section FAILED + * to produce its entities — in which case the reaper must NOT run, because the + * failed section wrote no observations and its prior-run rows would be deleted + * as false orphans (silent data loss). + * + * `sectionRunner` records several reason codes without aborting the filing; only + * two are safe to reap through: + * - `SECTION_NOT_FOUND` — the section is genuinely absent from this filing, so + * its old observations really are stale and should be reaped. (Common: most + * filings lack some optional section, so blocking on this would disable the + * reaper entirely.) + * - a `
-partial` marker — the section DID persist its verified rows + * (partial success), so the reap can safely remove the dropped orphans. + * + * Every other fresh code — `MODEL_EMPTY`, `LOW_CONFIDENCE_ALL`, a whole-section + * `UNVERIFIED_SOURCE_SPAN`, `MODEL_INVALID_OUTPUT` — denotes a section that + * yielded zero rows this run for a reason that may be transient (rate limit, + * truncated/empty model response, a swallowed exception), so the reap is + * suppressed until a clean re-extraction. Only dead-letters touched THIS run + * (`last_attempt_at >= since`) count; a stale entry from a prior run does not + * pin the reaper forever. + */ +export function hasBlockingSectionFailure( + deadLetters: readonly ReapGateDeadLetter[], + accession_number: string, + since: string +): boolean { + return deadLetters.some( + (d) => + d.accession_number === accession_number && + d.last_attempt_at >= since && + d.reason_code !== "SECTION_NOT_FOUND" && + !d.section_name.endsWith("-partial") + ); +} + export interface ReapStaleObservationsDeps { personObservationRepo?: PersonObservationRepo; companyObservationRepo?: CompanyObservationRepo; diff --git a/src/task/forms/ProcessAccessionDocFormTask.reapgate.test.ts b/src/task/forms/ProcessAccessionDocFormTask.reapgate.test.ts new file mode 100644 index 00000000..416aece7 --- /dev/null +++ b/src/task/forms/ProcessAccessionDocFormTask.reapgate.test.ts @@ -0,0 +1,265 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + AiProvider, + getAiProviderRegistry, + getGlobalModelRepository, + globalServiceRegistry, +} from "workglow"; +import type { AiProviderRunFn, Capability, ModelConfig } from "workglow"; +import { resetDependencyInjectionsForTesting } from "../../config/TestingDI"; +import { setupAllDatabases } from "../../config/setupAllDatabases"; +import { SEC_RAW_DATA_FOLDER } from "../../config/tokens"; +import { CompanyObservationRepo } from "../../storage/observation/CompanyObservationRepo"; +import { FILING_REPOSITORY_TOKEN } from "../../storage/filing/FilingSchema"; +import { ExtractionDeadLetterRepo } from "../../storage/dead-letter/ExtractionDeadLetterRepo"; +import { registerFakeStructuredProvider } from "../../sec/forms/registration-statements/s1/testing/fakeStructuredProvider"; +import { ProcessAccessionDocFormTask } from "./ProcessAccessionDocFormTask"; + +// Full prospectus: Management / Ownership / Related-party sections all present, +// so an empty model response per section dead-letters MODEL_EMPTY (not absent). +const SECTION_HTML = [ + "

MANAGEMENT

", + "

Jane Roe — Director

", + "

PRINCIPAL AND SELLING STOCKHOLDERS

", + "
ACME Fund1,000,00012.5%
", + "

CERTAIN RELATIONSHIPS AND RELATED TRANSACTIONS

", + "

We pay rent to an entity controlled by our CEO.

", + "

LEGAL MATTERS

x

", +].join(""); +const HTML = "S-11" + SECTION_HTML + ""; + +// Management-only prospectus: every other section is genuinely absent +// (SECTION_NOT_FOUND, which does not block the reap), so a run where Management +// succeeds is fully clean. +const MGMT_ONLY_HTML = + "S-11" + + "

MANAGEMENT

Jane Roe — Director

LEGAL MATTERS

x

" + + ""; + +const CIK = 1018724; +const ACCESSION = "0000000000-26-000123"; +const FILE_NAME = ACCESSION + ".txt"; +// An observation row for this filing+extractor that the current run does NOT +// re-observe (high index, old timestamp) — the reaper's target. +const PHANTOM_INDEX = 99; +const OLD_CREATED_AT = "2020-01-01T00:00:00.000Z"; + +const JSON_MODE = ["text.generation", "json-mode"] as const satisfies Capability[]; + +class ThrowingStructuredProvider extends AiProvider { + override readonly name = "fake-structured"; + override readonly displayName = "Throwing Structured"; + override readonly isLocal = true; + override readonly supportsBrowser = false; + override readonly supportsServer = false; +} + +/** + * A provider whose every section generation throws — the transient-failure case + * sectionRunner swallows into a `MODEL_INVALID_OUTPUT` dead-letter without + * aborting the filing. + */ +function registerThrowingProvider(): { unregister: () => void } { + const runFn: AiProviderRunFn = async () => { + throw new Error("simulated transient model failure"); + }; + const registry = getAiProviderRegistry(); + registry.registerProvider(new ThrowingStructuredProvider([{ serves: JSON_MODE, runFn }])); + registry.registerRunFn("fake-structured", { serves: JSON_MODE, runFn }); + return { unregister: () => registry.unregisterProvider("fake-structured") }; +} + +let rawRoot: string | undefined; +let cleanup: (() => void) | undefined; + +function seedFetchCache(folder: string, html: string): void { + const relative = `accessiondocs/${CIK.toString().padStart(10, "0")}/${ACCESSION.replaceAll( + "-", + "" + )}-${FILE_NAME}`; + const filePath = path.join(folder, relative); + mkdirSync(path.dirname(filePath), { recursive: true }); + writeFileSync(filePath, html, "utf-8"); +} + +async function seedFiling(): Promise { + const repo = globalServiceRegistry.get(FILING_REPOSITORY_TOKEN); + await repo.put({ + cik: CIK, + accession_number: ACCESSION, + form: "S-1", + primary_doc: FILE_NAME, + file_number: "333-1", + filing_date: "2026-01-02", + acceptance_date: "2026-01-02T00:00:00.000Z", + report_date: null, + film_number: null, + primary_doc_description: null, + size: null, + is_xbrl: null, + is_inline_xbrl: null, + items: null, + act: null, + } as never); +} + +/** Insert a stale phantom company observation the reaper would delete. */ +async function seedPhantomObservation(): Promise { + const obs = await new CompanyObservationRepo().upsertByNaturalKey({ + accession_number: ACCESSION, + extractor_id: "S-1", + extractor_version: "1.0.0", + observation_index: PHANTOM_INDEX, + cik: 9999999, + name: "Phantom Holdings LLC", + normalized_name: "phantom holdings llc", + created_at: OLD_CREATED_AT, + }); + return obs.observation_id; +} + +describe("ProcessAccessionDocFormTask reap gate on transient section failure", () => { + beforeEach(async () => { + resetDependencyInjectionsForTesting(); + await setupAllDatabases(); + rawRoot = mkdtempSync(path.join(tmpdir(), "sec-reapgate-")); + globalServiceRegistry.registerInstance(SEC_RAW_DATA_FOLDER, rawRoot); + process.env.SEC_S1_MODEL = "fake-s1-model"; + await getGlobalModelRepository().addModel({ + model_id: "fake-s1-model", + capabilities: ["text.generation", "json-mode"], + title: "Fake", + description: "Fake", + provider: "fake-structured", + provider_config: {}, + metadata: {}, + } as any); + }); + + afterEach(async () => { + cleanup?.(); + cleanup = undefined; + if (rawRoot) { + rmSync(rawRoot, { recursive: true, force: true }); + rawRoot = undefined; + } + await getGlobalModelRepository().removeModel("fake-s1-model"); + delete process.env.SEC_S1_MODEL; + resetDependencyInjectionsForTesting(); + }); + + it("keeps prior observations when a section dead-letters MODEL_INVALID_OUTPUT this run", async () => { + cleanup = registerThrowingProvider().unregister; + await seedFiling(); + seedFetchCache(rawRoot!, HTML); + const phantomId = await seedPhantomObservation(); + + const result = await new ProcessAccessionDocFormTask().run({ + accessionNumber: ACCESSION, + cik: CIK, + form: "S-1", + fileName: FILE_NAME, + }); + // The filing still succeeds end-to-end: section failures degrade, not abort. + expect((result as { success: boolean }).success).toBe(true); + + // A section transiently failed this run. + const pending = await new ExtractionDeadLetterRepo().listPending("S-1"); + expect( + pending.some( + (d) => d.accession_number === ACCESSION && d.reason_code === "MODEL_INVALID_OUTPUT" + ) + ).toBe(true); + + // The gate suppressed the reap: the stale phantom survives rather than being + // destroyed as collateral of a transient failure. + const survivor = await new CompanyObservationRepo().getById(phantomId); + expect(survivor).toBeDefined(); + }); + + it("keeps prior observations when a section dead-letters MODEL_EMPTY this run", async () => { + // A present section whose model returns no rows records MODEL_EMPTY (not + // MODEL_INVALID_OUTPUT) — the case the first cut of the gate missed. + const { unregister } = registerFakeStructuredProvider([ + { people: [] }, + { owners: [] }, + { parties: [] }, + ]); + cleanup = unregister; + await seedFiling(); + seedFetchCache(rawRoot!, HTML); + const phantomId = await seedPhantomObservation(); + + const result = await new ProcessAccessionDocFormTask().run({ + accessionNumber: ACCESSION, + cik: CIK, + form: "S-1", + fileName: FILE_NAME, + }); + expect((result as { success: boolean }).success).toBe(true); + + const pending = await new ExtractionDeadLetterRepo().listPending("S-1"); + expect( + pending.some((d) => d.accession_number === ACCESSION && d.reason_code === "MODEL_EMPTY") + ).toBe(true); + + // The widened gate suppresses the reap for MODEL_EMPTY too. + const survivor = await new CompanyObservationRepo().getById(phantomId); + expect(survivor).toBeDefined(); + }); + + it("still reaps stale observations on a fully clean run (no blocking failure)", async () => { + // Management persists a confident person (success -> markResolved); every + // other section is absent (SECTION_NOT_FOUND), which does not block. So no + // blocking dead-letter is recorded and the reaper runs. + const { unregister } = registerFakeStructuredProvider([ + { + people: [ + { + full_name: "Jane Roe", + title: "Director", + relationship: null, + confidence: 0.9, + source_span: "Jane Roe — Director", + }, + ], + }, + ]); + cleanup = unregister; + await seedFiling(); + seedFetchCache(rawRoot!, MGMT_ONLY_HTML); + const phantomId = await seedPhantomObservation(); + + const result = await new ProcessAccessionDocFormTask().run({ + accessionNumber: ACCESSION, + cik: CIK, + form: "S-1", + fileName: FILE_NAME, + }); + expect((result as { success: boolean }).success).toBe(true); + + // No blocking failure this run. + const pending = await new ExtractionDeadLetterRepo().listPending("S-1"); + expect( + pending.some( + (d) => + d.accession_number === ACCESSION && + d.reason_code !== "SECTION_NOT_FOUND" && + !d.section_name.endsWith("-partial") + ) + ).toBe(false); + + // The reaper runs and removes the stale phantom. + const reaped = await new CompanyObservationRepo().getById(phantomId); + expect(reaped).toBeUndefined(); + }); +}); diff --git a/src/task/forms/ProcessAccessionDocFormTask.ts b/src/task/forms/ProcessAccessionDocFormTask.ts index 0c776764..bd84d0ab 100644 --- a/src/task/forms/ProcessAccessionDocFormTask.ts +++ b/src/task/forms/ProcessAccessionDocFormTask.ts @@ -37,7 +37,10 @@ import { EXTRACTOR_RUN_REPOSITORY_TOKEN } from "../../storage/versioning/Extract import { formToExtractorId } from "../../storage/versioning/extractorIds"; import { getActiveSlot } from "../../storage/versioning/getActiveSlot"; import { VersionRegistry } from "../../storage/versioning/VersionRegistry"; -import { reapStaleObservations } from "../../resolver/reapStaleObservations"; +import { + hasBlockingSectionFailure, + reapStaleObservations, +} from "../../resolver/reapStaleObservations"; import { SecFetchAccessionDocTask } from "./SecFetchAccessionDocTask"; /** @@ -419,17 +422,47 @@ export class ProcessAccessionDocFormTask extends Task< // reclassified entity set leaves stale orphans joined to canonical // entities). Best-effort, like recordRun — a reaper hiccup must not mask // the successful extraction. + // + // BUT: a narrative section that yields zero rows this run — a transient + // throw (network blip, rate limit, malformed output → MODEL_INVALID_OUTPUT), + // an empty/truncated model response (MODEL_EMPTY), all rows below the + // confidence floor (LOW_CONFIDENCE_ALL), or all rows unverifiable + // (UNVERIFIED_SOURCE_SPAN) — is swallowed into a dead-letter WITHOUT + // aborting the filing (see sectionRunner). That section writes no + // observations this run, so its prior-run rows look stale + // (created_at < runStart) and the reap would delete the last good + // extraction for a section that merely failed transiently — silent data + // loss. Skip the reap whenever any such section failure was recorded THIS + // run; the worst case is a retained superset that the next clean + // re-extraction reaps safely. (A genuinely-absent section records + // SECTION_NOT_FOUND and a partial success records a `-partial` marker — + // neither blocks; see hasBlockingSectionFailure.) + let transientSectionFailure = false; try { - await reapStaleObservations({ - accession_number: accessionNumber, - extractor_id: extractorId, - before: runStart, - }); - } catch (reapErr) { + const pending = await deadLetters.listPending(extractorId); + transientSectionFailure = hasBlockingSectionFailure(pending, accessionNumber, runStart); + } catch (dlErr) { + // If we cannot tell, default to NOT reaping — preserving stale rows is + // recoverable; deleting still-valid ones is not. console.error( - `Failed to reap stale observations for ${accessionNumber}@${extractorId}:`, - reapErr + `Failed to check section dead-letters before reap for ${accessionNumber}@${extractorId}:`, + dlErr ); + transientSectionFailure = true; + } + if (!transientSectionFailure) { + try { + await reapStaleObservations({ + accession_number: accessionNumber, + extractor_id: extractorId, + before: runStart, + }); + } catch (reapErr) { + console.error( + `Failed to reap stale observations for ${accessionNumber}@${extractorId}:`, + reapErr + ); + } } try { await runRepo.recordRun({ From 8c1d04e117cdfad64ad925d37e0710d37db4cd7e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Jun 2026 20:24:36 +0000 Subject: [PATCH 10/12] fix(util,spac): date range validation, merger-proxy lifecycle rank, abort handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three isolated correctness fixes from the review pass: - parseDate accepted any digit-count match, so "20251301" (month 13) or "2025-12-32" parsed to a bogus { month: "13" } / { day: "32" } and corrupted date ordering / as_of guards downstream. Reject out-of-range months (1-12) and days (1-31). - mergerFormRank gave PRER (preliminary-revised) the same rank as DEFR, so a same-date PRER tiebreak outranked the definitive DEFM. Rank by the actual filing lifecycle PREM < PRER < DEFM < DEFR so a definitive proxy supersedes a preliminary-revised one (only a definitive-revised DEFR outranks DEFM). - BackfillMergerProxiesTask / BackfillRedemptionsTask swallowed every per-filing error in their sweep, including TaskAbortedError — so a cooperative cancellation kept grinding through the whole worklist. Check context.signal before each filing and rethrow TaskAbortedError, mirroring RetryDeadLettersTask. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- src/storage/spac/spacDealGrouping.test.ts | 32 +++++++++++++++++ src/storage/spac/spacDealGrouping.ts | 16 +++++---- .../spac/BackfillMergerProxiesTask.test.ts | 34 +++++++++++++++++-- src/task/spac/BackfillMergerProxiesTask.ts | 6 +++- src/task/spac/BackfillRedemptionsTask.ts | 6 +++- src/util/parseDate.test.ts | 15 ++++++++ src/util/parseDate.ts | 8 +++++ 7 files changed, 107 insertions(+), 10 deletions(-) diff --git a/src/storage/spac/spacDealGrouping.test.ts b/src/storage/spac/spacDealGrouping.test.ts index dcc4ba1c..07ff12ae 100644 --- a/src/storage/spac/spacDealGrouping.test.ts +++ b/src/storage/spac/spacDealGrouping.test.ts @@ -196,6 +196,38 @@ describe("deriveDeals", () => { expect(deals[0].pipe_amount).toBe(200_000_000); }); + it("ranks a same-date definitive proxy above a preliminary-revised one (PRER < DEFM)", () => { + // PRER (preliminary-revised) and DEFM (definitive) land on the SAME + // filing_date. Per the PREM { + const deals = deriveDeals( + 1, + [ev("definitive_agreement", "2021-03-01"), ev("completed", "2021-06-15")], + [ + ext("prem", "2021-05-10", { form: "PREM14A", target_name: "Preliminary Target" }), + ext("prer", "2021-05-10", { form: "PRER14A", target_name: "Preliminary Revised Target" }), + ], + [], + [] + ); + expect(deals[0].target_name).toBe("Preliminary Revised Target"); + }); + it("leaves an extraction with no matching open deal unattached", () => { // proxy filed before any DA event -> no deal yet const deals = deriveDeals(1, [], [ext("p1", "2021-05-01", { target_name: "Acme" })], [], []); diff --git a/src/storage/spac/spacDealGrouping.ts b/src/storage/spac/spacDealGrouping.ts index e0f6cd1e..23758663 100644 --- a/src/storage/spac/spacDealGrouping.ts +++ b/src/storage/spac/spacDealGrouping.ts @@ -11,15 +11,19 @@ import type { SpacRedemptionExtraction } from "./SpacRedemptionExtractionSchema" /** * Supersession precedence for merger-proxy forms when two land on the same - * filing_date: revised (DEFR / PRER) supersedes definitive (DEFM) supersedes - * preliminary (PREM). Used only as a same-date tiebreak; filing_date still - * dominates. + * filing_date, ordered by the filing lifecycle PREM -> PRER -> DEFM -> DEFR: + * definitive supersedes preliminary, and within each stage the revised form + * supersedes its base. A preliminary-revised proxy (PRER) therefore outranks + * the preliminary it revises (PREM) but is still superseded by the definitive + * (DEFM) — it does NOT outrank DEFM the way the definitive-revised (DEFR) does. + * Used only as a same-date tiebreak; filing_date still dominates. */ function mergerFormRank(form: string): number { const f = form.toUpperCase(); - if (f.startsWith("DEFR") || f.startsWith("PRER")) return 2; - if (f.startsWith("DEFM")) return 1; - return 0; + if (f.startsWith("DEFR")) return 3; // definitive, revised + if (f.startsWith("DEFM")) return 2; // definitive + if (f.startsWith("PRER")) return 1; // preliminary, revised + return 0; // PREM (preliminary) and anything else } /** Event types that shape a business-combination attempt. */ diff --git a/src/task/spac/BackfillMergerProxiesTask.test.ts b/src/task/spac/BackfillMergerProxiesTask.test.ts index f8682051..20a49d37 100644 --- a/src/task/spac/BackfillMergerProxiesTask.test.ts +++ b/src/task/spac/BackfillMergerProxiesTask.test.ts @@ -4,13 +4,16 @@ * SPDX-License-Identifier: Apache-2.0 */ import { beforeEach, describe, expect, it } from "bun:test"; -import { globalServiceRegistry } from "workglow"; +import { globalServiceRegistry, TaskAbortedError, type IExecuteContext } from "workglow"; import { resetDependencyInjectionsForTesting } from "../../config/TestingDI"; import { setupAllDatabases } from "../../config/setupAllDatabases"; import { FILING_REPOSITORY_TOKEN } from "../../storage/filing/FilingSchema"; import { SpacReportWriter } from "../../storage/spac/SpacReportWriter"; import { SpacMergerExtractionRepo } from "../../storage/spac/SpacMergerExtractionRepo"; -import { selectMergerProxyBackfillAccessions } from "./BackfillMergerProxiesTask"; +import { + BackfillMergerProxiesTask, + selectMergerProxyBackfillAccessions, +} from "./BackfillMergerProxiesTask"; async function seedSpac(cik: number): Promise { await new SpacReportWriter().recordRegistration({ @@ -92,3 +95,30 @@ describe("selectMergerProxyBackfillAccessions", () => { expect(accessions.sort()).toEqual(["acc-defm", "acc-prem"]); }); }); + +describe("BackfillMergerProxiesTask abort handling", () => { + beforeEach(async () => { + resetDependencyInjectionsForTesting(); + await setupAllDatabases(); + }); + + it("rethrows TaskAbortedError and processes nothing when the signal is aborted", async () => { + await seedSpac(7); + await seedFiling({ cik: 7, accession_number: "acc-defm-7", form: "DEFM14A" }); + // Sanity: there is a selectable filing, so the loop body is reached. + expect((await selectMergerProxyBackfillAccessions()).length).toBeGreaterThan(0); + + const controller = new AbortController(); + controller.abort(); + const ctx = { signal: controller.signal } as IExecuteContext; + + // A cooperative cancellation must surface as TaskAbortedError, not be + // swallowed by the per-accession catch like a fetch/parse failure. + await expect(new BackfillMergerProxiesTask().execute({}, ctx)).rejects.toBeInstanceOf( + TaskAbortedError + ); + + // Nothing was reprocessed -> still no extraction row for the filing. + expect(await new SpacMergerExtractionRepo().getByAccession("acc-defm-7")).toBeUndefined(); + }); +}); diff --git a/src/task/spac/BackfillMergerProxiesTask.ts b/src/task/spac/BackfillMergerProxiesTask.ts index 8d17cc75..46175537 100644 --- a/src/task/spac/BackfillMergerProxiesTask.ts +++ b/src/task/spac/BackfillMergerProxiesTask.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import { Static, Type } from "typebox"; -import { globalServiceRegistry, IExecuteContext, Task, Workflow } from "workglow"; +import { globalServiceRegistry, IExecuteContext, Task, TaskAbortedError, Workflow } from "workglow"; import { FILING_REPOSITORY_TOKEN } from "../../storage/filing/FilingSchema"; import { SpacRepo } from "../../storage/spac/SpacRepo"; import { SpacMergerExtractionRepo } from "../../storage/spac/SpacMergerExtractionRepo"; @@ -94,12 +94,16 @@ export class BackfillMergerProxiesTask extends Task< // must not abort the sweep over the remaining accessions. let processed = 0; for (const accessionNumber of accessions) { + if (context.signal?.aborted) throw new TaskAbortedError(); try { const wf = context.own(new Workflow()); wf.pipe(new ProcessAccessionDocFormTask()); await wf.run({ accessionNumber }); processed++; } catch (err) { + // A cooperative cancellation must stop the sweep, not be swallowed as a + // per-filing failure like a fetch/parse error. + if (err instanceof TaskAbortedError) throw err; console.error(`backfill-merger-proxies: failed to reprocess ${accessionNumber}:`, err); } } diff --git a/src/task/spac/BackfillRedemptionsTask.ts b/src/task/spac/BackfillRedemptionsTask.ts index 1ae3809f..842890d4 100644 --- a/src/task/spac/BackfillRedemptionsTask.ts +++ b/src/task/spac/BackfillRedemptionsTask.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import { Static, Type } from "typebox"; -import { globalServiceRegistry, IExecuteContext, Task, Workflow } from "workglow"; +import { globalServiceRegistry, IExecuteContext, Task, TaskAbortedError, Workflow } from "workglow"; import { FILING_REPOSITORY_TOKEN } from "../../storage/filing/FilingSchema"; import { SpacRepo } from "../../storage/spac/SpacRepo"; import { hasRedemptionTriggerItem } from "../../sec/forms/miscellaneous-filings/spac8kRedemptionTriggers"; @@ -81,12 +81,16 @@ export class BackfillRedemptionsTask extends Task< // must not abort the sweep over the remaining accessions. let processed = 0; for (const accessionNumber of accessions) { + if (context.signal?.aborted) throw new TaskAbortedError(); try { const wf = context.own(new Workflow()); wf.pipe(new ProcessAccessionDocFormTask()); await wf.run({ accessionNumber }); processed++; } catch (err) { + // A cooperative cancellation must stop the sweep, not be swallowed as a + // per-filing failure like a fetch/parse error. + if (err instanceof TaskAbortedError) throw err; console.error(`backfill-redemptions: failed to reprocess ${accessionNumber}:`, err); } } diff --git a/src/util/parseDate.test.ts b/src/util/parseDate.test.ts index 9ec707a7..ce6766a3 100644 --- a/src/util/parseDate.test.ts +++ b/src/util/parseDate.test.ts @@ -29,6 +29,21 @@ describe("parseDate", () => { it("throws on an unrecognised format", () => { expect(() => parseDate("not-a-date")).toThrow("Invalid date format"); }); + + it("rejects out-of-range months and days the digit-count regexes admit", () => { + // The regexes only constrain digit counts, so these would otherwise parse to + // bogus { month: '13' } / { day: '45' } instead of being rejected. + expect(() => parseDate("20251301")).toThrow("Invalid date format"); // month 13 + expect(() => parseDate("20250045")).toThrow("Invalid date format"); // day 45 (and month 00) + expect(() => parseDate("2025-00-10")).toThrow("Invalid date format"); // month 00 + expect(() => parseDate("2025-12-32")).toThrow("Invalid date format"); // day 32 + expect(() => parseDate("13/05/2025")).toThrow("Invalid date format"); // month 13 (MM/dd/yyyy) + }); + + it("accepts boundary months and days", () => { + expect(parseDate("20250101")).toEqual({ year: 2025, month: "01", day: "01" }); + expect(parseDate("20251231")).toEqual({ year: 2025, month: "12", day: "31" }); + }); }); describe("secDate", () => { diff --git a/src/util/parseDate.ts b/src/util/parseDate.ts index d7d0a3f0..dd6088f4 100644 --- a/src/util/parseDate.ts +++ b/src/util/parseDate.ts @@ -39,6 +39,14 @@ export function parseDate(dateStr: string): { year: number; month: string; day: day = parseInt(match[2], 10); } + // The shapes above only constrain digit counts, so "20251301" (month 13) + // or "2025-00-45" pass the regex but are not real dates. Reject + // out-of-range months/days rather than silently emitting "13"/"45", which + // would corrupt date ordering and as_of guards downstream. + if (month < 1 || month > 12 || day < 1 || day > 31) { + throw new Error("Invalid date format"); + } + return { year, month: month.toString().padStart(2, "0"), From 1c36177977dcbc6d3960ffad8a1d1a81d601c7f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Jun 2026 20:24:45 +0000 Subject: [PATCH 11/12] fix(storage): make the Form D offering as_of guard atomic per (cik, file_number) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The as_of staleness guard in Form_D.storage did an unsynchronized get -> compare -> save of the mutable current offering row. The form tasks map over a CIK's filings with concurrencyLimit 5/10, so a D and D/A for the same (cik, file_number) processed concurrently could both read the same prior row, both decide they were not stale, and let the older filing's write land last — regressing the current row (the same lost-update class the SPAC writer fixed with a per-CIK lock). Move the read-guard-write into InvestmentOfferingRepo.saveInvestmentOfferingAsOf and run it inside a per-(cik, file_number) lock. Introduce a reusable KeyedMutex primitive (generalizing the hand-rolled withCikLock / resolver per-key mutex) rather than copy the pattern a third time. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- .../forms/exempt-offerings/Form_D.storage.ts | 23 ++---- .../InvestmentOfferingRepo.test.ts | 40 +++++++++ .../InvestmentOfferingRepo.ts | Bin 8874 -> 10780 bytes src/util/KeyedMutex.test.ts | 77 ++++++++++++++++++ src/util/KeyedMutex.ts | 49 +++++++++++ 5 files changed, 172 insertions(+), 17 deletions(-) create mode 100644 src/util/KeyedMutex.test.ts create mode 100644 src/util/KeyedMutex.ts diff --git a/src/sec/forms/exempt-offerings/Form_D.storage.ts b/src/sec/forms/exempt-offerings/Form_D.storage.ts index 35826e0d..d17391ca 100644 --- a/src/sec/forms/exempt-offerings/Form_D.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_D.storage.ts @@ -131,25 +131,14 @@ async function processOffering( // History is per-(cik, file_number, accession) and always recorded — it is // the append-only time series. The mutable current row must reflect the - // latest filing BY FILING DATE, not by processing order: skip an out-of-order - // older D / D-A so a back-catalog replay can't regress industry_group / - // security-type flags / date_of_first_sale. An undated incoming filing ("") - // cannot be ordered against a dated row and is treated as stale; when the - // existing row is also undated or absent it still applies. (Each Form D fully + // latest filing BY FILING DATE, not by processing order: an out-of-order + // older D / D-A is skipped so a back-catalog replay can't regress + // industry_group / security-type flags / date_of_first_sale. The read-guard- + // write is atomic per (cik, file_number) so two filings for the same offering + // processed concurrently can't lost-update the row. (Each Form D fully // restates the offering, so no field-merge is needed — only the date guard.) await investmentOfferingRepo.saveInvestmentOfferingHistory(investmentOfferingHistory); - - const existing = await investmentOfferingRepo.getInvestmentOffering(cik, file_number); - const isStale = - existing?.as_of != null && - existing.as_of !== "" && - (filing_date === "" || filing_date < existing.as_of); - if (!isStale) { - await investmentOfferingRepo.saveInvestmentOffering({ - ...investmentOffering, - as_of: filing_date || existing?.as_of || null, - }); - } + await investmentOfferingRepo.saveInvestmentOfferingAsOf(investmentOffering, filing_date); if (offering.salesCompensationList.recipient) { let salesIndex = 100; diff --git a/src/storage/investment-offering/InvestmentOfferingRepo.test.ts b/src/storage/investment-offering/InvestmentOfferingRepo.test.ts index 38aaa485..ef0fece4 100644 --- a/src/storage/investment-offering/InvestmentOfferingRepo.test.ts +++ b/src/storage/investment-offering/InvestmentOfferingRepo.test.ts @@ -128,6 +128,46 @@ describe("InvestmentOfferingRepo", () => { }); }); + describe("saveInvestmentOfferingAsOf", () => { + const older = { ...mockOffering, industry_group: "Old", as_of: null }; + const newer = { ...mockOffering, industry_group: "New", as_of: null }; + + it("skips an out-of-order older filing (as_of guard)", async () => { + await investmentOfferingRepo.saveInvestmentOfferingAsOf(newer, "2024-01-01"); + await investmentOfferingRepo.saveInvestmentOfferingAsOf(older, "2023-01-01"); + + const final = await investmentOfferingRepo.getInvestmentOffering( + mockOffering.cik, + mockOffering.file_number + ); + expect(final?.as_of).toBe("2024-01-01"); + expect(final?.industry_group).toBe("New"); + }); + + it("lets the newer filing win regardless of concurrent submission order", async () => { + // Both orders: the per-(cik,file_number) lock serialises the + // read-guard-write so the older filing can never lost-update the row. + for (const ops of [ + [ + investmentOfferingRepo.saveInvestmentOfferingAsOf(older, "2023-01-01"), + investmentOfferingRepo.saveInvestmentOfferingAsOf(newer, "2024-01-01"), + ], + [ + investmentOfferingRepo.saveInvestmentOfferingAsOf(newer, "2024-01-01"), + investmentOfferingRepo.saveInvestmentOfferingAsOf(older, "2023-01-01"), + ], + ]) { + await Promise.all(ops); + const final = await investmentOfferingRepo.getInvestmentOffering( + mockOffering.cik, + mockOffering.file_number + ); + expect(final?.as_of).toBe("2024-01-01"); + expect(final?.industry_group).toBe("New"); + } + }); + }); + describe("getInvestmentOfferingsByCik", () => { it("should return offerings associated with a cik", async () => { const offering1 = { ...mockOffering }; diff --git a/src/storage/investment-offering/InvestmentOfferingRepo.ts b/src/storage/investment-offering/InvestmentOfferingRepo.ts index 61c8fc1e1c4ff48e37558cf4d463e8e4a0e680d5..8ffe7e3131e98b640393b971cbc332e12aa60871 100644 GIT binary patch delta 1778 zcmah}(TW^J6m^9d%!>p~9+bqJL6ezfrzbuL;${*TH)u$rvKz>YuwBz#lPRXAtEsA< z&afFepCF8nzW6G>NWLSV;EUh~_yGpbt(qRkBp@scQ+?~6bI&=qe!2bAgCBnF(OtQa z`Tfev$F!ueH8biY$pC+qS9FwQKY2N)`SBSl3itnkKCA?JWJ8AfsF$bD}gesypZh*Koa zfZQrq&Z}>V%+pjZ9OxF;JY_~}mCFPLgL9byr;QvBS+jd4o6-2@_t0D()uuX+KF=LG6g_ecHsKKzk*bg(#6zjT3=m>5`U*%(- zB>IKnrj07&hv;%Wk?0HOP;!zBFH=%3_A{Dc{RF@q5C;k2W6Pn7V^jIQ8TXCl!dQMt z13K&<^!C6vCxeT&M#f@ChHQ|xTKP%n38ipz-7=vAHaf!;pc^U0(!vl^qi(0uV=G{M zY&b$p863PAPGK_S*z->pRoZ`rIQWma zBKZB+rJgiL0GahQV#F@TIUJ1wJ?a8pM! zQIbJ=V^2X}-nn}XJ%ep)J=0BtY;jM`luzl?!6(;S!Ds|CFjgb-C<)UjtupJLyw0ql zmNAjh&WHf0m=+x>1Va1A(;q_{ZJh5{Ww3`ikAz*D9ioY;+D(Pvb37oQ;bBnL8m!=Y z)sIV&9m&#IAC|~20mu4c#vo@jXs~+m)`w>f+_&TI>iXp?=R@e%3WVUU{yKlLnH!bg zq*IoO1qD&8FIF=B=(LqQ!CSSYt>2ijeYTeAez+=Xp=ij2BF8n$o@ z_#Pu}Hf~6v&+6~XAFM8&eY;`gy^ZzB7H%)Xww#qa<_)J|Aj7k?J#Qu937{)@Wh$-d z>eW|DHWfqabhZn=??yL+S~MJn?@yj=Ye+XXNU_a^m>N@eO{<3#`N|e8=Q!j}=hDNd uJM&GFq}4kb#fO3V=l{V2{L7=^h(~L$EqnW`7w=qLJ$?7Wv*%}jfBzrylu=dy delta 24 gcmbOevdVSBnvD-^88 + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from "bun:test"; +import { KeyedMutex } from "./KeyedMutex"; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((r) => (resolve = r)); + return { promise, resolve }; +} + +describe("KeyedMutex", () => { + it("serialises critical sections sharing a key", async () => { + const km = new KeyedMutex(); + const order: string[] = []; + let inFlight = 0; + let maxInFlight = 0; + + const section = (label: string) => async () => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + order.push(`start:${label}`); + await Promise.resolve(); + await Promise.resolve(); + order.push(`end:${label}`); + inFlight--; + }; + + await Promise.all([km.lock("k", section("a")), km.lock("k", section("b"))]); + + // Same key never overlaps, and runs FIFO. + expect(maxInFlight).toBe(1); + expect(order).toEqual(["start:a", "end:a", "start:b", "end:b"]); + }); + + it("lets different keys run concurrently", async () => { + const km = new KeyedMutex(); + const a = deferred(); + let bRan = false; + + // 'a' blocks until released; 'b' (different key) must still complete. + const pa = km.lock("a", async () => { + await a.promise; + }); + const pb = km.lock("b", async () => { + bRan = true; + }); + + await pb; + expect(bRan).toBe(true); // not blocked behind 'a' + a.resolve(); + await pa; + }); + + it("evicts a key's mutex once no holders remain", async () => { + const km = new KeyedMutex(); + await km.lock("k", async () => {}); + expect(km.size).toBe(0); + }); + + it("propagates a critical section's rejection without poisoning later locks", async () => { + const km = new KeyedMutex(); + await expect( + km.lock("k", async () => { + throw new Error("boom"); + }) + ).rejects.toThrow("boom"); + // A subsequent lock on the same key still runs. + const ran = await km.lock("k", async () => 42); + expect(ran).toBe(42); + expect(km.size).toBe(0); + }); +}); diff --git a/src/util/KeyedMutex.ts b/src/util/KeyedMutex.ts new file mode 100644 index 00000000..2a402cd0 --- /dev/null +++ b/src/util/KeyedMutex.ts @@ -0,0 +1,49 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { AsyncMutex } from "./AsyncMutex"; + +/** + * A per-key {@link AsyncMutex}. `lock(key, fn)` serialises `fn` against other + * `lock` calls with the SAME key while letting different keys run concurrently — + * the building block for an atomic read-modify-write of a mutable "current" row + * keyed by `(cik, file_number)` / portal CIK / etc. + * + * Each key's mutex is created on first use, refcounted, and evicted at zero so + * the map stays bounded under a long-lived process. This generalises the + * hand-rolled `withCikLock` in {@link SpacReportWriter} and the resolvers' + * per-key mutex into one reusable primitive. + * + * Single-process only: like {@link AsyncMutex} it is invisible across processes, + * so multi-process callers still need a backend UNIQUE constraint / append-only + * PK for race-freedom. + */ +export class KeyedMutex { + private readonly entries = new Map(); + + lock(key: K, fn: () => Promise): Promise { + let entry = this.entries.get(key); + if (entry === undefined) { + entry = { mutex: new AsyncMutex(), refs: 0 }; + this.entries.set(key, entry); + } + entry.refs += 1; + const held = entry; + return held.mutex.lock(fn).finally(() => { + held.refs -= 1; + // Same-identity check guards against a caller recreating the entry between + // the decrement and the delete. + if (held.refs === 0 && this.entries.get(key) === held) { + this.entries.delete(key); + } + }); + } + + /** Number of live (refcounted) keys — for tests asserting eviction. */ + get size(): number { + return this.entries.size; + } +} From 55b12d8d5e0f0bb2e1345d5db2f2b3aea9d82db6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Jun 2026 20:46:13 +0000 Subject: [PATCH 12/12] fix(storage): make every mutable current-row as_of guard atomic per key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Form D was the only mutable "current" row whose out-of-order as_of guard ran inside a per-key lock. Its siblings — RegAOffering (1-A / 1-K / 1-Z), Portal (CFPORTAL) and Crowdfunding (Form C family) — did the same get -> compare -> build -> save unsynchronized. Because the form tasks map over a CIK's filings with concurrencyLimit 5/10, two filings for the same key processed concurrently could both read the same prior row, both decide they were not stale, and let the older filing's write land last — regressing the current row (and, for CFPORTAL, potentially resurrecting a withdrawn portal). Bring them all into alignment: - Extract the shared staleness predicate into util/asOfGuard.ts (isStaleByAsOf), the single source of the "" / null / undated-incoming semantics. - Add atomic, per-key guarded-write methods that read-merge-write inside a KeyedMutex, taking a builder that receives the row read inside the lock so the per-form field merges (1-K/1-Z carry no tier/SIC; C-AR carries no portal CIK) stay correct: - InvestmentOfferingRepo.saveInvestmentOfferingAsOf (refactored to the builder shape + shared predicate) - RegAOfferingRepo.saveOfferingAsOf - PortalRepo.savePortalAsOf - CrowdfundingTemporalRepo.saveCurrentByFilingDate (filing_date is its as-of marker; still always writes the history snapshot via skipMutableUpdate) - Rewire Form_D / Form_1_A / Form_1_K / Form_1_Z / Form_CFPORTAL / Form_C to the new methods; behavior is otherwise unchanged. Adds repo-level concurrency tests (newer filing wins regardless of submission order) and a unit test for the shared predicate. Also strips a stray NUL byte that a prior commit left in an InvestmentOfferingRepo comment. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018GW5p9FW5RPHXPjRwvwywR --- .../exempt-offerings/Form_1_A.storage.ts | 52 +++++------- .../exempt-offerings/Form_1_K.storage.ts | 41 ++++------ .../exempt-offerings/Form_1_Z.storage.ts | 43 ++++------ .../forms/exempt-offerings/Form_C.storage.ts | 77 ++++++++---------- .../forms/exempt-offerings/Form_D.storage.ts | 7 +- src/sec/forms/portal/Form_CFPORTAL.storage.ts | 44 ++++------ .../InvestmentOfferingRepo.test.ts | 27 +++--- .../InvestmentOfferingRepo.ts | Bin 10780 -> 10773 bytes .../portal/CrowdfundingTemporalRepo.test.ts | 61 ++++++++++++++ .../portal/CrowdfundingTemporalRepo.ts | 44 ++++++++++ src/storage/portal/PortalRepo.test.ts | 35 ++++++++ src/storage/portal/PortalRepo.ts | 32 ++++++++ src/storage/reg-a/RegAOfferingRepo.test.ts | 63 ++++++++++++++ src/storage/reg-a/RegAOfferingRepo.ts | 37 +++++++++ src/util/asOfGuard.test.ts | 34 ++++++++ src/util/asOfGuard.ts | 37 +++++++++ 16 files changed, 464 insertions(+), 170 deletions(-) create mode 100644 src/util/asOfGuard.test.ts create mode 100644 src/util/asOfGuard.ts diff --git a/src/sec/forms/exempt-offerings/Form_1_A.storage.ts b/src/sec/forms/exempt-offerings/Form_1_A.storage.ts index 2f3a8d5c..e2cd4054 100644 --- a/src/sec/forms/exempt-offerings/Form_1_A.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_1_A.storage.ts @@ -423,39 +423,25 @@ export async function processForm1A({ const employeesInfo = form1A.formData.employeesInfo[0]; // 1-A/A and 1-A POS restate the offering data but say nothing about the - // lifecycle: a post-qualification amendment processed after a 1-K / 1-Z - // must not regress a "reporting"/"exit" offering back to "pending". - const existing = await regARepo.getOffering(cik, file_number); - - // Mutable row = latest filing by filing date; skip stale out-of-order - // writes. An undated incoming filing ("") cannot be ordered against a - // dated existing row and is treated as stale, so a filer error with no - // SGML date in the header does not clobber a known-dated mutable row. - // (When the existing row is also undated or absent, an undated filing - // still applies — there's nothing to regress.) The history row below is - // per-accession and always recorded. - const isStale = - existing?.as_of != null && - existing.as_of !== "" && - (filing_date === "" || filing_date < existing.as_of); - - if (!isStale) { - const offering: RegAOffering = { - cik, - file_number, - issuer_name: employeesInfo.issuerName, - jurisdiction: employeesInfo.jurisdictionOrganization, - sic_code: employeesInfo.sicCode, - tier: summaryInfo.indicateTier1Tier2Offering, - financial_statement_audit_status: summaryInfo.financialStatementAuditStatus, - securities_offered_type: summaryInfo.securitiesOfferedTypes, - industry_group: form1A.formData.issuerInfo.industryGroup, - status: existing?.status ?? "pending", - as_of: filing_date || existing?.as_of || null, - }; - - await regARepo.saveOffering(offering); - } + // lifecycle: a post-qualification amendment processed after a 1-K / 1-Z must + // not regress a "reporting"/"exit" offering back to "pending" — so the status + // is carried forward from the existing row. The mutable row is latest-by- + // filing-date; the read-merge-write is atomic per (cik, file_number) and skips + // stale out-of-order writes (an undated "" filing is treated as stale). The + // history row below is per-accession and always recorded. + await regARepo.saveOfferingAsOf(cik, file_number, filing_date, (existing) => ({ + cik, + file_number, + issuer_name: employeesInfo.issuerName, + jurisdiction: employeesInfo.jurisdictionOrganization, + sic_code: employeesInfo.sicCode, + tier: summaryInfo.indicateTier1Tier2Offering, + financial_statement_audit_status: summaryInfo.financialStatementAuditStatus, + securities_offered_type: summaryInfo.securitiesOfferedTypes, + industry_group: form1A.formData.issuerInfo.industryGroup, + status: existing?.status ?? "pending", + as_of: filing_date || existing?.as_of || null, + })); const history: RegAOfferingHistory = { cik, diff --git a/src/sec/forms/exempt-offerings/Form_1_K.storage.ts b/src/sec/forms/exempt-offerings/Form_1_K.storage.ts index 4bfd9a1e..bbcf5a77 100644 --- a/src/sec/forms/exempt-offerings/Form_1_K.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_1_K.storage.ts @@ -243,31 +243,22 @@ export async function processForm1K({ // Upsert the offering. A 1-K carries no tier/SIC/audit/securities data, so // preserve whatever the 1-A wrote — a full-row put with nulls here clobbers // the tier and makes queries like `reg-a --tier Tier2 --status reporting` - // unsatisfiable. The mutable row is latest-by-filing-date: skip stale - // out-of-order writes (unknown "" dates apply as-is). - const existing = await regARepo.getOffering(cik, file_number); - const isStale = - existing?.as_of != null && - existing.as_of !== "" && - (filing_date === "" || filing_date < existing.as_of); - - if (!isStale) { - const offering: RegAOffering = { - cik, - file_number, - issuer_name: primaryIssuer?.issuerName ?? existing?.issuer_name ?? null, - jurisdiction: primaryIssuer?.jurisdictionOrganization ?? existing?.jurisdiction ?? null, - sic_code: existing?.sic_code ?? null, - tier: existing?.tier ?? null, - financial_statement_audit_status: existing?.financial_statement_audit_status ?? null, - securities_offered_type: existing?.securities_offered_type ?? null, - industry_group: existing?.industry_group ?? null, - status: "reporting", - as_of: filing_date || existing?.as_of || null, - }; - - await regARepo.saveOffering(offering); - } + // unsatisfiable. The mutable row is latest-by-filing-date; the read-merge-write + // is atomic per (cik, file_number) and skips stale out-of-order writes (unknown + // "" dates apply as-is). + await regARepo.saveOfferingAsOf(cik, file_number, filing_date, (existing) => ({ + cik, + file_number, + issuer_name: primaryIssuer?.issuerName ?? existing?.issuer_name ?? null, + jurisdiction: primaryIssuer?.jurisdictionOrganization ?? existing?.jurisdiction ?? null, + sic_code: existing?.sic_code ?? null, + tier: existing?.tier ?? null, + financial_statement_audit_status: existing?.financial_statement_audit_status ?? null, + securities_offered_type: existing?.securities_offered_type ?? null, + industry_group: existing?.industry_group ?? null, + status: "reporting", + as_of: filing_date || existing?.as_of || null, + })); await processIssuer(cik, form1K, ctx, 0); await processOfferingHistory(cik, file_number, accession_number, filing_date, form1K, ctx, 100); diff --git a/src/sec/forms/exempt-offerings/Form_1_Z.storage.ts b/src/sec/forms/exempt-offerings/Form_1_Z.storage.ts index 05530916..35b539a8 100644 --- a/src/sec/forms/exempt-offerings/Form_1_Z.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_1_Z.storage.ts @@ -330,32 +330,23 @@ export async function processForm1Z({ const item1 = form1Z.formData.item1; // A 1-Z exit report carries only the issuer name; preserve the descriptive - // fields the 1-A wrote instead of clobbering them with nulls. The mutable - // row is latest-by-filing-date: skip stale out-of-order writes (unknown "" - // dates apply as-is). - const existing = await regARepo.getOffering(cik, file_number); - const isStale = - existing?.as_of != null && - existing.as_of !== "" && - (filing_date === "" || filing_date < existing.as_of); - - if (!isStale) { - const offering: RegAOffering = { - cik, - file_number, - issuer_name: item1.issuerName ?? existing?.issuer_name ?? null, - jurisdiction: existing?.jurisdiction ?? null, - sic_code: existing?.sic_code ?? null, - tier: existing?.tier ?? null, - financial_statement_audit_status: existing?.financial_statement_audit_status ?? null, - securities_offered_type: existing?.securities_offered_type ?? null, - industry_group: existing?.industry_group ?? null, - status: "exit", - as_of: filing_date || existing?.as_of || null, - }; - - await regARepo.saveOffering(offering); - } + // fields the 1-A wrote instead of clobbering them with nulls. The mutable row + // is latest-by-filing-date; the read-merge-write is atomic per + // (cik, file_number) and skips stale out-of-order writes (unknown "" dates + // apply as-is). + await regARepo.saveOfferingAsOf(cik, file_number, filing_date, (existing) => ({ + cik, + file_number, + issuer_name: item1.issuerName ?? existing?.issuer_name ?? null, + jurisdiction: existing?.jurisdiction ?? null, + sic_code: existing?.sic_code ?? null, + tier: existing?.tier ?? null, + financial_statement_audit_status: existing?.financial_statement_audit_status ?? null, + securities_offered_type: existing?.securities_offered_type ?? null, + industry_group: existing?.industry_group ?? null, + status: "exit", + as_of: filing_date || existing?.as_of || null, + })); await processIssuer(cik, form1Z, ctx, 0); await processOfferingSummaries(cik, file_number, accession_number, filing_date, form1Z, ctx, 100); diff --git a/src/sec/forms/exempt-offerings/Form_C.storage.ts b/src/sec/forms/exempt-offerings/Form_C.storage.ts index f48f7c85..68874a26 100644 --- a/src/sec/forms/exempt-offerings/Form_C.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_C.storage.ts @@ -424,58 +424,45 @@ export async function processFormC({ const issuer = issuerInfo.issuerInfo; const submissionType = formC.headerData.submissionType; - // Post-offering filings (C-AR / C-TR) carry no and often no - // legal-status block; preserve what the original Form C established for the - // same (cik, file_number) instead of clobbering the portal link and issuer - // details with empties. - const crowdfundingRepo = new CrowdfundingRepo(); - const existing = await crowdfundingRepo.getCrowdfunding(cik, file_number); const parsedPortalCik = parseCikSafely(issuerInfo.commissionCik); // The mutable row reflects the latest filing by *filing date*, not by - // processing order: a back-catalog replay of an older filing must not - // regress it. An undated incoming filing ("") cannot be ordered against - // a dated existing row and is treated as stale, so a filer error with no - // SGML date in the header does not clobber a known-dated mutable row. - // (When the existing row is also undated or absent, an undated filing - // still applies — there's nothing to regress.) The per-filing tables + // processing order: a back-catalog replay of an older filing must not regress + // it. The read-merge-write is atomic per (cik, file_number) and treats an + // undated incoming filing ("") as stale, so a filer header with no SGML date + // does not clobber a known-dated mutable row. Post-offering filings (C-AR / + // C-TR) carry no and often no legal-status block, so the + // builder merges those fields forward from the existing row instead of + // clobbering them with empties. The history snapshot is always written (a + // stale replay still belongs in the time series); only the mutable-row write + // is suppressed via skipMutableUpdate. The filing_date is exempt from the + // merge fallback: a stale replay records its own filing_date so the series + // reflects when each filing was actually made. The per-filing tables // (offerings, disclosure reports, observations) below are keyed by // filing/accession and always record the older filing too. - const isStale = - existing?.filing_date !== undefined && - existing.filing_date !== "" && - (filing_date === "" || filing_date < existing.filing_date); - - // Always build and write the history snapshot — a stale replay still - // belongs in the time series, only the mutable row write is suppressed - // (via skipMutableUpdate) so out-of-order processing doesn't regress it. - // Falling back to existing fields keeps the snapshot meaningful when the - // older filing carried sparser data than the current state. The - // filing_date is exempt from that fallback: when stale, the snapshot must - // record this replay's own filing_date so the time series reflects when - // each filing was actually made. The schema requires a string, so unknown - // dates are stored as "" rather than inheriting the existing row's date. - const crowdfunding: Crowdfunding = { + await temporalRepo.saveCurrentByFilingDate( cik, file_number, - filing_date: isStale ? filing_date : filing_date || existing?.filing_date || "", - name: strScalar(issuer.nameOfIssuer) || existing?.name || "", - legal_status: issuer.legalStatus?.legalStatusForm ?? existing?.legal_status ?? "", - state_jurisdiction: - issuer.legalStatus?.jurisdictionOrganization ?? existing?.state_jurisdiction ?? "", - date_incorporation: - issuer.legalStatus?.dateIncorporation ?? existing?.date_incorporation ?? "", - url: issuer.issuerWebsite ?? existing?.url ?? "", - portal_cik: parsedPortalCik > 0 ? parsedPortalCik : (existing?.portal_cik ?? 0), - status: determineStatus(submissionType), - progress_update: issuerInfo.progressUpdate ?? existing?.progress_update ?? null, - nature_of_amendment: issuerInfo.natureOfAmendment ?? existing?.nature_of_amendment ?? null, - }; - - await temporalRepo.saveCrowdfundingWithHistory(crowdfunding, `Form ${submissionType}`, { - skipMutableUpdate: isStale, - accessionNumber: accession_number, - }); + filing_date, + `Form ${submissionType}`, + accession_number, + (existing, isStale) => ({ + cik, + file_number, + filing_date: isStale ? filing_date : filing_date || existing?.filing_date || "", + name: strScalar(issuer.nameOfIssuer) || existing?.name || "", + legal_status: issuer.legalStatus?.legalStatusForm ?? existing?.legal_status ?? "", + state_jurisdiction: + issuer.legalStatus?.jurisdictionOrganization ?? existing?.state_jurisdiction ?? "", + date_incorporation: + issuer.legalStatus?.dateIncorporation ?? existing?.date_incorporation ?? "", + url: issuer.issuerWebsite ?? existing?.url ?? "", + portal_cik: parsedPortalCik > 0 ? parsedPortalCik : (existing?.portal_cik ?? 0), + status: determineStatus(submissionType), + progress_update: issuerInfo.progressUpdate ?? existing?.progress_update ?? null, + nature_of_amendment: issuerInfo.natureOfAmendment ?? existing?.nature_of_amendment ?? null, + }) + ); // Issuers: index 0 (issuer), 1+ (co-issuers) await processIssuer(cik, formC, ctx, 0); diff --git a/src/sec/forms/exempt-offerings/Form_D.storage.ts b/src/sec/forms/exempt-offerings/Form_D.storage.ts index d17391ca..bb2a4349 100644 --- a/src/sec/forms/exempt-offerings/Form_D.storage.ts +++ b/src/sec/forms/exempt-offerings/Form_D.storage.ts @@ -138,7 +138,12 @@ async function processOffering( // processed concurrently can't lost-update the row. (Each Form D fully // restates the offering, so no field-merge is needed — only the date guard.) await investmentOfferingRepo.saveInvestmentOfferingHistory(investmentOfferingHistory); - await investmentOfferingRepo.saveInvestmentOfferingAsOf(investmentOffering, filing_date); + await investmentOfferingRepo.saveInvestmentOfferingAsOf( + cik, + file_number, + filing_date, + (existing) => ({ ...investmentOffering, as_of: filing_date || existing?.as_of || null }) + ); if (offering.salesCompensationList.recipient) { let salesIndex = 100; diff --git a/src/sec/forms/portal/Form_CFPORTAL.storage.ts b/src/sec/forms/portal/Form_CFPORTAL.storage.ts index dcfc5459..1759a637 100644 --- a/src/sec/forms/portal/Form_CFPORTAL.storage.ts +++ b/src/sec/forms/portal/Form_CFPORTAL.storage.ts @@ -115,36 +115,24 @@ export async function processFormCFPORTAL({ const portalRepo = new PortalRepo(); const isWithdrawal = submissionType === "CFPORTAL-W"; - const existing = await portalRepo.getPortal(cik); // The mutable row reflects the latest filing by *filing date*, not by - // processing order — a back-catalog replay of the original registration - // must not resurrect a withdrawn portal. An undated incoming filing ("") - // cannot be ordered against a dated existing row and is treated as - // stale, so a filer error with no SGML date in the header does not - // clobber a known-dated mutable row. (When the existing row is also - // undated or absent, an undated filing still applies — there's nothing - // to regress.) Observations below are keyed by accession and always - // record the older filing too. - const isStale = - existing?.as_of != null && - existing.as_of !== "" && - (filing_date === "" || filing_date < existing.as_of); - - if (!isStale) { - // Absent fields inherit from the existing portal row — a CFPORTAL/A may - // omit identifying info the registered portal still carries. A fresh - // CFPORTAL with an absent field stays null (no existing row). - const hasName = identifying?.nameOfPortal !== undefined; - await portalRepo.savePortal({ - cik, - name: hasName ? (identifying!.nameOfPortal ?? null) : (existing?.name ?? null), - brand: brand ?? existing?.brand ?? null, - url: url ?? existing?.url ?? null, - live: !isWithdrawal, - as_of: filing_date || existing?.as_of || null, - }); - } + // processing order — a back-catalog replay of the original registration must + // not resurrect a withdrawn portal. The read-merge-write is atomic per CIK and + // skips stale out-of-order writes (an undated "" filing is treated as stale). + // Absent fields inherit from the existing portal row — a CFPORTAL/A may omit + // identifying info the registered portal still carries; a fresh CFPORTAL with + // an absent field stays null. Observations below are keyed by accession and + // always record the older filing too. + const hasName = identifying?.nameOfPortal !== undefined; + await portalRepo.savePortalAsOf(cik, filing_date, (existing) => ({ + cik, + name: hasName ? (identifying!.nameOfPortal ?? null) : (existing?.name ?? null), + brand: brand ?? existing?.brand ?? null, + url: url ?? existing?.url ?? null, + live: !isWithdrawal, + as_of: filing_date || existing?.as_of || null, + })); // Observation tier. Index layout: 0 = the portal company itself, // 1 = contact employee, 100+ = Schedule A direct/indirect owners. diff --git a/src/storage/investment-offering/InvestmentOfferingRepo.test.ts b/src/storage/investment-offering/InvestmentOfferingRepo.test.ts index ef0fece4..f3e4407a 100644 --- a/src/storage/investment-offering/InvestmentOfferingRepo.test.ts +++ b/src/storage/investment-offering/InvestmentOfferingRepo.test.ts @@ -129,12 +129,21 @@ describe("InvestmentOfferingRepo", () => { }); describe("saveInvestmentOfferingAsOf", () => { - const older = { ...mockOffering, industry_group: "Old", as_of: null }; - const newer = { ...mockOffering, industry_group: "New", as_of: null }; + const save = (industry_group: string, filing_date: string) => + investmentOfferingRepo.saveInvestmentOfferingAsOf( + mockOffering.cik, + mockOffering.file_number, + filing_date, + (existing) => ({ + ...mockOffering, + industry_group, + as_of: filing_date || existing?.as_of || null, + }) + ); it("skips an out-of-order older filing (as_of guard)", async () => { - await investmentOfferingRepo.saveInvestmentOfferingAsOf(newer, "2024-01-01"); - await investmentOfferingRepo.saveInvestmentOfferingAsOf(older, "2023-01-01"); + await save("New", "2024-01-01"); + await save("Old", "2023-01-01"); const final = await investmentOfferingRepo.getInvestmentOffering( mockOffering.cik, @@ -148,14 +157,8 @@ describe("InvestmentOfferingRepo", () => { // Both orders: the per-(cik,file_number) lock serialises the // read-guard-write so the older filing can never lost-update the row. for (const ops of [ - [ - investmentOfferingRepo.saveInvestmentOfferingAsOf(older, "2023-01-01"), - investmentOfferingRepo.saveInvestmentOfferingAsOf(newer, "2024-01-01"), - ], - [ - investmentOfferingRepo.saveInvestmentOfferingAsOf(newer, "2024-01-01"), - investmentOfferingRepo.saveInvestmentOfferingAsOf(older, "2023-01-01"), - ], + [save("Old", "2023-01-01"), save("New", "2024-01-01")], + [save("New", "2024-01-01"), save("Old", "2023-01-01")], ]) { await Promise.all(ops); const final = await investmentOfferingRepo.getInvestmentOffering( diff --git a/src/storage/investment-offering/InvestmentOfferingRepo.ts b/src/storage/investment-offering/InvestmentOfferingRepo.ts index 8ffe7e3131e98b640393b971cbc332e12aa60871..0d0ba5a46535216ab6c123c9b032f2fe3af7b011 100644 GIT binary patch delta 573 zcmY+B&ubGw6vwFr38A$VX;XyCml7n~Zd2$*Y^k-S_~X=r6`_~xcK1y?WOpXKnKgzG z>fMVl_U1n!`9HjQ6x5@CgMyRYDk;ny=Do+~`~AN8baHj_;rj-pd1>Q}4hIM3SYQOf zI>%bOz2#3~WwsUj`m-YD^z+xlTQeB1%>KH)W-ka*B|4YtybfR;k&u~xbkf*T5 zMV1Y`hOs`_wI%-uuX&0nHBh=TUjc=rEJaQ{drralPyNe`Tdp<3aMH!pns#6;`<+#d zGUE;j@XGt`GIcoZDsb=bY%i8>{?7%$uIn|HU2kP+yua}6 F-d{&-!F&J! delta 593 zcmZXR!7c+q9L7}>hoT};7ZRTh%~on7E<$ZY;^5?hi;G>||LUYQBQrxuHC}-XHwW*~ zw{Y+b&K|+Z>~7mM9B2ODe82De&+EzC$o^tEL(YZH{4NMWMkFapX-q*&@~qUJ6_gHJR7*ihi$v5X*|fNpq)6UU)O(ry z%$fJq#o20N+6Sk#%*MjJ`I(+tstc|(nsg5z!AeKOF-aTJtFvYoD;@Lf&gHv7&E$$s z-m1(_@o;)nR5ed->9JBv0YGNQ)Nv9Je*YGrJEpYt-Sg6oVA%)1X|+%;|C0Te5s Om9!W<>-HvR3Jbsb*|?Yh diff --git a/src/storage/portal/CrowdfundingTemporalRepo.test.ts b/src/storage/portal/CrowdfundingTemporalRepo.test.ts index a8cfa97f..c08a4603 100644 --- a/src/storage/portal/CrowdfundingTemporalRepo.test.ts +++ b/src/storage/portal/CrowdfundingTemporalRepo.test.ts @@ -458,4 +458,65 @@ describe("CrowdfundingTemporalRepo", () => { const entityAtTime2 = await temporalRepo.getCrowdfundingAtTime(12345, "020-12345", time2); expect(entityAtTime2?.name).toBe("Updated Name"); }); + + describe("saveCurrentByFilingDate", () => { + const CIK = 44444; + const FN = "020-44444"; + + // Mirrors Form_C.storage: merge fields forward from the existing row; a + // stale replay records its own filing_date in the snapshot. + const save = ( + filing_date: string, + status: string, + portalCik: number, + source: string, + acc: string + ) => + temporalRepo.saveCurrentByFilingDate(CIK, FN, filing_date, source, acc, (existing, isStale) => ({ + cik: CIK, + file_number: FN, + filing_date: isStale ? filing_date : filing_date || existing?.filing_date || "", + name: existing?.name || "Issuer Co", + legal_status: existing?.legal_status ?? "Corporation", + state_jurisdiction: existing?.state_jurisdiction ?? "DE", + date_incorporation: existing?.date_incorporation ?? "2020-01-01", + url: existing?.url ?? "http://example.com", + portal_cik: portalCik > 0 ? portalCik : (existing?.portal_cik ?? 0), + status, + progress_update: null, + nature_of_amendment: null, + })); + + test("merges a later C-AR onto the C and a stale replay snapshots without regressing", async () => { + await save("2024-01-01", "active", 54321, "Form C", "acc-c"); + await save("2024-06-01", "annual-report", 0, "Form C-AR", "acc-car"); // no portal cik -> merged + + let mutable = await temporalRepo.getCrowdfunding(CIK, FN); + expect(mutable?.filing_date).toBe("2024-06-01"); + expect(mutable?.status).toBe("annual-report"); + expect(mutable?.portal_cik).toBe(54321); // merged forward, not clobbered to 0 + + // An out-of-order older replay must not regress the mutable row... + await save("2023-01-01", "active", 0, "Form C-AR", "acc-stale"); + mutable = await temporalRepo.getCrowdfunding(CIK, FN); + expect(mutable?.filing_date).toBe("2024-06-01"); + expect(mutable?.status).toBe("annual-report"); + + // ...but still lands in the history time series as a closed snapshot. + const history = await temporalRepo.getCrowdfundingHistory(CIK, FN); + expect(history.some((h) => h.accession_number === "acc-stale")).toBe(true); + }); + + test("lets the newer filing win regardless of concurrent submission order", async () => { + for (const ops of [ + [save("2024-01-01", "active", 54321, "Form C", "acc-c"), save("2024-06-01", "annual-report", 0, "Form C-AR", "acc-car")], + [save("2024-06-01", "annual-report", 0, "Form C-AR", "acc-car"), save("2024-01-01", "active", 54321, "Form C", "acc-c")], + ]) { + await Promise.all(ops); + const mutable = await temporalRepo.getCrowdfunding(CIK, FN); + expect(mutable?.filing_date).toBe("2024-06-01"); + expect(mutable?.status).toBe("annual-report"); + } + }); + }); }); diff --git a/src/storage/portal/CrowdfundingTemporalRepo.ts b/src/storage/portal/CrowdfundingTemporalRepo.ts index 6b0f624f..616bb2d4 100644 --- a/src/storage/portal/CrowdfundingTemporalRepo.ts +++ b/src/storage/portal/CrowdfundingTemporalRepo.ts @@ -5,6 +5,8 @@ */ import { globalServiceRegistry } from "workglow"; +import { KeyedMutex } from "../../util/KeyedMutex"; +import { isStaleByAsOf } from "../../util/asOfGuard"; import { CHANGE_LOG_REPOSITORY_TOKEN, ChangeLog, @@ -24,6 +26,14 @@ interface CrowdfundingTemporalRepoOptions { changeLogRepository?: ChangeLogRepositoryStorage; } +/** + * Serialises the read-guard-write of the mutable current crowdfunding row per + * `(cik, file_number)`. Module-scoped because every caller builds a fresh + * {@link CrowdfundingTemporalRepo}. The ` ` separator never occurs in an EDGAR + * file number, so distinct keys can't collide. + */ +const crowdfundingWriteLock = new KeyedMutex(); + /** * Temporal Crowdfunding repository - extends CrowdfundingRepo with history tracking */ @@ -162,6 +172,40 @@ export class CrowdfundingTemporalRepo { await this.crowdfundingRepo.saveCrowdfunding(crowdfunding); } + /** + * Atomically refresh the mutable current crowdfunding row from the latest + * filing BY FILING DATE. Under a per-`(cik, file_number)` lock: read the + * existing row, decide staleness against its `filing_date` (its as-of marker — + * see {@link isStaleByAsOf}), let `build` construct the row (it receives both + * the existing row, for field merges, and `isStale`, since a stale replay + * records its own filing_date in the snapshot), then delegate to + * {@link saveCrowdfundingWithHistory} with `skipMutableUpdate` so a stale + * filing still lands in the history time series but does not regress the + * mutable row. + * + * The lock spans this read AND saveCrowdfundingWithHistory's own + * read-modify-write, so two C / C-AR filings for the same offering processed + * concurrently cannot lost-update the mutable row. + */ + async saveCurrentByFilingDate( + cik: number, + file_number: string, + filing_date: string, + changeSource: string, + accessionNumber: string, + build: (existing: Crowdfunding | undefined, isStale: boolean) => Crowdfunding + ): Promise { + await crowdfundingWriteLock.lock(`${cik} ${file_number}`, async () => { + const existing = await this.crowdfundingRepo.getCrowdfunding(cik, file_number); + const isStale = isStaleByAsOf(existing?.filing_date, filing_date); + const crowdfunding = build(existing, isStale); + await this.saveCrowdfundingWithHistory(crowdfunding, changeSource, { + skipMutableUpdate: isStale, + accessionNumber, + }); + }); + } + /** * Get crowdfunding entity at a specific point in time */ diff --git a/src/storage/portal/PortalRepo.test.ts b/src/storage/portal/PortalRepo.test.ts index 3543ee66..198d9dce 100644 --- a/src/storage/portal/PortalRepo.test.ts +++ b/src/storage/portal/PortalRepo.test.ts @@ -78,6 +78,41 @@ describe("PortalRepo", () => { }); }); + describe("savePortalAsOf", () => { + const CIK = 1234567; + const save = (name: string, live: boolean, filing_date: string) => + portalRepo.savePortalAsOf(CIK, filing_date, (existing) => ({ + cik: CIK, + name, + brand: existing?.brand ?? "Brand", + url: existing?.url ?? null, + live, + as_of: filing_date || existing?.as_of || null, + })); + + it("skips an out-of-order older filing (as_of guard)", async () => { + await save("Withdrawn", false, "2024-06-01"); + await save("Registered", true, "2023-01-01"); + + const final = await portalRepo.getPortal(CIK); + expect(final?.as_of).toBe("2024-06-01"); + expect(final?.name).toBe("Withdrawn"); + expect(final?.live).toBe(false); // not resurrected by the older registration + }); + + it("lets the newer filing win regardless of concurrent submission order", async () => { + for (const ops of [ + [save("Registered", true, "2023-01-01"), save("Withdrawn", false, "2024-06-01")], + [save("Withdrawn", false, "2024-06-01"), save("Registered", true, "2023-01-01")], + ]) { + await Promise.all(ops); + const final = await portalRepo.getPortal(CIK); + expect(final?.as_of).toBe("2024-06-01"); + expect(final?.live).toBe(false); + } + }); + }); + describe("deletePortal", () => { it("should successfully delete a portal", async () => { await portalStorage.put(mockPortal1); diff --git a/src/storage/portal/PortalRepo.ts b/src/storage/portal/PortalRepo.ts index a9f36c11..9032d88b 100644 --- a/src/storage/portal/PortalRepo.ts +++ b/src/storage/portal/PortalRepo.ts @@ -5,6 +5,8 @@ */ import { globalServiceRegistry } from "workglow"; +import { KeyedMutex } from "../../util/KeyedMutex"; +import { isStaleByAsOf } from "../../util/asOfGuard"; import { Portal, PORTAL_REPOSITORY_TOKEN, PortalRepositoryStorage } from "./PortalSchema"; // Options for the PortalRepo @@ -12,6 +14,12 @@ interface PortalRepoOptions { portalRepository?: PortalRepositoryStorage; } +/** + * Serialises the read-guard-write of the mutable current portal row per CIK. + * Module-scoped because every caller builds a fresh {@link PortalRepo}. + */ +const portalWriteLock = new KeyedMutex(); + /** * Portal repository */ @@ -32,6 +40,30 @@ export class PortalRepo implements PortalRepoOptions { return portal; } + /** + * Persist the mutable current portal row under an `as_of` staleness guard, + * atomically. Reads the existing row, skips the write when the incoming + * `filing_date` is older than the stored `as_of` (an out-of-order replay — see + * {@link isStaleByAsOf}), and otherwise writes the row `build` returns. `build` + * receives the row read inside the lock so a CFPORTAL/A can inherit identifying + * fields the registered portal still carries. + * + * The whole read-merge-write runs inside a per-CIK lock so two portal filings + * processed concurrently cannot both read the same prior row and lost-update it + * (e.g. resurrect a withdrawn portal). + */ + async savePortalAsOf( + cik: number, + filing_date: string, + build: (existing: Portal | undefined) => Portal + ): Promise { + await portalWriteLock.lock(cik, async () => { + const existing = await this.getPortal(cik); + if (isStaleByAsOf(existing?.as_of, filing_date)) return; + await this.savePortal(build(existing)); + }); + } + async deletePortal(cik: number): Promise { await this.portalRepository.delete({ cik }); } diff --git a/src/storage/reg-a/RegAOfferingRepo.test.ts b/src/storage/reg-a/RegAOfferingRepo.test.ts index 7135fa5a..261b3708 100644 --- a/src/storage/reg-a/RegAOfferingRepo.test.ts +++ b/src/storage/reg-a/RegAOfferingRepo.test.ts @@ -138,6 +138,69 @@ describe("RegAOfferingRepo", () => { }); }); + describe("saveOfferingAsOf", () => { + const CIK = 55555; + const FN = "024-09999"; + + // 1-A: full row with tier; status carried forward from existing. + const save1A = (filing_date: string) => + repo.saveOfferingAsOf(CIK, FN, filing_date, (existing) => ({ + cik: CIK, + file_number: FN, + issuer_name: "RegA Issuer Inc", + jurisdiction: "DE", + sic_code: 7372, + tier: "Tier2", + financial_statement_audit_status: "Audited", + securities_offered_type: "Equity (common or preferred stock)", + industry_group: "Other", + status: existing?.status ?? "pending", + as_of: filing_date || existing?.as_of || null, + })); + + // 1-K: carries no tier/SIC — merges them forward from the existing row. + const save1K = (filing_date: string) => + repo.saveOfferingAsOf(CIK, FN, filing_date, (existing) => ({ + cik: CIK, + file_number: FN, + issuer_name: existing?.issuer_name ?? null, + jurisdiction: existing?.jurisdiction ?? null, + sic_code: existing?.sic_code ?? null, + tier: existing?.tier ?? null, + financial_statement_audit_status: existing?.financial_statement_audit_status ?? null, + securities_offered_type: existing?.securities_offered_type ?? null, + industry_group: existing?.industry_group ?? null, + status: "reporting", + as_of: filing_date || existing?.as_of || null, + })); + + it("merges a later 1-K onto the 1-A and skips a stale out-of-order replay", async () => { + await save1A("2024-01-01"); + await save1K("2024-06-01"); + await save1A("2023-01-01"); // out-of-order older replay -> skipped + + const final = await repo.getOffering(CIK, FN); + expect(final?.as_of).toBe("2024-06-01"); + expect(final?.status).toBe("reporting"); + expect(final?.tier).toBe("Tier2"); // merged forward, not clobbered to null + }); + + it("lets the newer filing win regardless of concurrent submission order", async () => { + // The guarded invariant: the newest filing's as_of and authoritative + // status win; the older 1-A can never lost-update them. (The merged tier + // is order-dependent by design — only asserted in the sequential case.) + for (const ops of [ + [save1A("2024-01-01"), save1K("2024-06-01")], + [save1K("2024-06-01"), save1A("2024-01-01")], + ]) { + await Promise.all(ops); + const final = await repo.getOffering(CIK, FN); + expect(final?.as_of).toBe("2024-06-01"); + expect(final?.status).toBe("reporting"); + } + }); + }); + describe("Offering History", () => { it("should save and retrieve offering history", async () => { const history: RegAOfferingHistory = { diff --git a/src/storage/reg-a/RegAOfferingRepo.ts b/src/storage/reg-a/RegAOfferingRepo.ts index 030b9966..725e001f 100644 --- a/src/storage/reg-a/RegAOfferingRepo.ts +++ b/src/storage/reg-a/RegAOfferingRepo.ts @@ -5,6 +5,8 @@ */ import { globalServiceRegistry } from "workglow"; +import { KeyedMutex } from "../../util/KeyedMutex"; +import { isStaleByAsOf } from "../../util/asOfGuard"; import { REGA_EQUITY_CLASS_REPOSITORY_TOKEN, RegAEquityClass, @@ -39,6 +41,14 @@ interface RegAOfferingRepoOptions { equityClassRepository?: RegAEquityClassRepositoryStorage; } +/** + * Serialises the read-guard-write of the mutable current Reg-A offering row per + * `(cik, file_number)`. Module-scoped because every caller builds a fresh + * {@link RegAOfferingRepo}. The ` ` separator never occurs in an EDGAR file + * number, so distinct keys can't collide. + */ +const regAOfferingWriteLock = new KeyedMutex(); + /** * Reg-A Offering repository - aggregates all Reg-A schemas */ @@ -78,6 +88,33 @@ export class RegAOfferingRepo { await this.offeringRepository.put(offering); } + /** + * Persist the mutable current Reg-A offering row under an `as_of` staleness + * guard, atomically. Reads the existing row, skips the write when the incoming + * `filing_date` is older than the stored `as_of` (an out-of-order amendment — + * see {@link isStaleByAsOf}), and otherwise writes the row `build` returns. + * `build` receives the row read inside the lock so a 1-K / 1-Z (which carry no + * tier / SIC / audit data) can merge those fields forward from the 1-A instead + * of clobbering them with nulls. + * + * The whole read-merge-write runs inside a per-`(cik, file_number)` lock so a + * 1-A and a 1-K / 1-Z for the same offering processed concurrently (forms map + * over a CIK's filings with `concurrencyLimit` 5/10) cannot both read the same + * prior row and lost-update it. + */ + async saveOfferingAsOf( + cik: number, + fileNumber: string, + filing_date: string, + build: (existing: RegAOffering | undefined) => RegAOffering + ): Promise { + await regAOfferingWriteLock.lock(`${cik} ${fileNumber}`, async () => { + const existing = await this.getOffering(cik, fileNumber); + if (isStaleByAsOf(existing?.as_of, filing_date)) return; + await this.saveOffering(build(existing)); + }); + } + async getOfferingsByCik(cik: number): Promise { return (await this.offeringRepository.query({ cik })) || []; } diff --git a/src/util/asOfGuard.test.ts b/src/util/asOfGuard.test.ts new file mode 100644 index 00000000..ca04c91a --- /dev/null +++ b/src/util/asOfGuard.test.ts @@ -0,0 +1,34 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from "bun:test"; +import { isStaleByAsOf } from "./asOfGuard"; + +describe("isStaleByAsOf", () => { + it("is not stale when there is no existing marker", () => { + expect(isStaleByAsOf(undefined, "2024-01-01")).toBe(false); + expect(isStaleByAsOf(null, "2024-01-01")).toBe(false); + expect(isStaleByAsOf("", "2024-01-01")).toBe(false); + }); + + it("is stale when the incoming filing is older than the marker", () => { + expect(isStaleByAsOf("2024-06-01", "2024-01-01")).toBe(true); + }); + + it("is not stale when the incoming filing is newer or equal", () => { + expect(isStaleByAsOf("2024-01-01", "2024-06-01")).toBe(false); + expect(isStaleByAsOf("2024-01-01", "2024-01-01")).toBe(false); + }); + + it("treats an undated incoming filing as stale against a dated row", () => { + // It cannot be ordered, so it must not clobber a known-dated row. + expect(isStaleByAsOf("2024-01-01", "")).toBe(true); + }); + + it("applies an undated incoming filing when the existing row is also undated", () => { + expect(isStaleByAsOf("", "")).toBe(false); + }); +}); diff --git a/src/util/asOfGuard.ts b/src/util/asOfGuard.ts new file mode 100644 index 00000000..a674ef19 --- /dev/null +++ b/src/util/asOfGuard.ts @@ -0,0 +1,37 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * The shared "would this incoming filing regress the mutable current row?" rule. + * + * A mutable "current" row (InvestmentOffering / RegAOffering / Portal / + * Crowdfunding) must reflect the latest filing BY FILING DATE, not by processing + * order. `existingMarker` is the as-of date the stored row already reflects (its + * `as_of`, or for Crowdfunding its `filing_date`); `filing_date` is the incoming + * filing's date. The incoming write is stale — and must be skipped — when: + * + * - the existing row carries a known marker (`!= null` and not `""`), AND + * - the incoming filing is older (`filing_date < marker`) OR undated (`""`). + * + * An undated incoming filing cannot be ordered against a dated row, so it is + * treated as stale (a filer header with no SGML date does not clobber a + * known-dated row). When the existing row is absent or itself undated, nothing + * can be regressed and the write applies. + * + * Callers must run this read-check-write inside a per-key lock (see + * {@link KeyedMutex}) so two filings for the same key processed concurrently + * cannot both read the same prior marker and let the older write land last. + */ +export function isStaleByAsOf( + existingMarker: string | null | undefined, + filing_date: string +): boolean { + return ( + existingMarker != null && + existingMarker !== "" && + (filing_date === "" || filing_date < existingMarker) + ); +}