From 708f3fca0c5d26a418dda86c2278ac4d26d7cb14 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 28 Aug 2026 16:07:25 +0000 Subject: [PATCH 1/3] Two words of coincidence is not evidence, and a construction is not an article MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The anchored gate shipped and the hourly cron wrote its first 162 keywords across 11 sites. The vendor and junk classes are gone. Reading the actual output found two things the fixtures could not have. **Partial matches leaked.** Eight keywords, all one shape: one generic word of a multi-word subject matched, and a word from the generic commercial vocabulary rescued it. "open standards" matched on "open", "software" did the rest, and logicsrc was queued to write about OBS Studio. "supply chain security" matched on "supply" and admitted "industrial automation supply". "online shopping" matched on "online" and admitted "quickbooks online pricing". So a partial match on a multi-word subject may now only be rescued by an anchor the site itself supplied — never by the defaults, which are generic by construction. A complete match is unaffected, including a single-word subject, where "iptv alternatives" is a real query. **And the cross-product floor was publishing constructions.** That one is mine and it was new. Used as per-subject filler it produced "saving money pricing", "deals platform", "coordination d0rz" and "ai content loop" — on-niche, gate-passing, and not topics anybody searches. bl0ggers' niche is "human-in-the-loop AI publishing", so its derived modifiers are literally "human" and "loop", and crossing a subject with those yields nonsense. The crosses' real value was always upstream: as DataForSEO seeds they are what turns "peptide" into "peptide merchant account", and they still seed every expansion. They are simply no longer published on the strength of being grammatically adjacent to the niche. They remain the floor, but site-level rather than per-subject — only a run that would otherwise insert nothing falls back to them, which keeps "a blog with every upstream down still publishes" without letting constructions pad a healthy run. The result reports usedCrossFloor so a site sitting on that for weeks is visible. All eight leaked keywords are now regression cases, with the on-niche keywords from the same run that must still pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013cbcXdpLFUgJZuVfCB4Gzv --- lib/lx/keywordsResearch.ts | 91 +++++++++--- lib/lx/topicPlan.ts | 45 ++++++ tests/lx/topic-plan-partial-match.test.ts | 166 ++++++++++++++++++++++ 3 files changed, 286 insertions(+), 16 deletions(-) create mode 100644 tests/lx/topic-plan-partial-match.test.ts diff --git a/lib/lx/keywordsResearch.ts b/lib/lx/keywordsResearch.ts index 7f5bbc8..c5c93e6 100644 --- a/lib/lx/keywordsResearch.ts +++ b/lib/lx/keywordsResearch.ts @@ -37,6 +37,7 @@ import { crossQueries, dropDuplicates, isOnNiche, + ownAnchorTokens, resolveMasters, resolveModifiers, signature, @@ -77,8 +78,14 @@ const MAX_BUYER_JOURNEY_VOLUME_LOOKUP = 160; */ const CROSS_PER_MASTER = 3; -/** A candidate with the subject it belongs to. */ -type Candidate = { row: DfsKeywordRow; master: string }; +/** + * A candidate with the subject it belongs to. + * + * `fromCross` marks the locally-built subject x modifier constructions. They + * are excellent *seeds* — that is their real job — and mediocre *articles*, so + * they are tracked separately and only published as a last resort. + */ +type Candidate = { row: DfsKeywordRow; master: string; fromCross?: boolean }; function parseStoredKeyword(row: string): DfsKeywordRow | null { const idx = row.indexOf(","); @@ -271,6 +278,13 @@ export type KeywordResearchResult = { apiCost: number; /** Rows allocated per subject — surfaced so a skewed queue is visible. */ perMaster?: Record; + /** + * True when nothing researched survived and the run fell back to + * constructed subject x modifier topics. A degraded result, not a failure — + * reported rather than swallowed, because a site sitting on this for weeks + * means its upstreams or its modifiers need attention. + */ + usedCrossFloor?: boolean; error?: string; }; @@ -303,6 +317,9 @@ export async function researchKeywords( // halves of the gate with one token. See anchorTokens. const modifiers = resolveModifiers(site, masters); const anchors = anchorTokens(site, masters); + // Site-supplied anchors only. A partial match on a multi-word subject may + // not be rescued by the generic vocabulary — see isOnNiche. + const ownAnchors = ownAnchorTokens(site, masters); if (masters.length === 0) { return { @@ -376,6 +393,7 @@ export async function researchKeywords( const crosses = crossQueries(masters, modifiers, CROSS_PER_MASTER); for (const { master, query } of crosses) { candidates.push({ + fromCross: true, row: { keyword: query, search_volume: null, @@ -487,7 +505,9 @@ export async function researchKeywords( // ------------------------------------------------------------------ // Gate, rank, allocate. // ------------------------------------------------------------------ - const onNiche = candidates.filter((c) => isOnNiche(c.row.keyword, c.master, anchors)); + const onNiche = candidates.filter((c) => + isOnNiche(c.row.keyword, c.master, anchors, ownAnchors), + ); // Volume filtering applies only to what came back from an API with a volume // attached. The locally-built crosses have no volume by construction and @@ -524,20 +544,53 @@ export async function researchKeywords( publishedSignatures, ); - // Take each subject's allocated share, then interleave so the published - // sequence alternates subjects rather than running one to exhaustion. - const byMaster = new Map(); - for (const master of masters) byMaster.set(master, []); - for (const candidate of deduped) { - const bucket = byMaster.get(candidate.master); - if (!bucket) continue; - if (bucket.length >= (allocation.get(candidate.master) ?? 0)) continue; - bucket.push({ row: candidate.row, master: candidate.master }); + /** + * Take each subject's allocated share, then interleave so the published + * sequence alternates subjects rather than running one to exhaustion. + */ + function select(pool: Candidate[]): Candidate[] { + const byMaster = new Map(); + for (const master of masters) byMaster.set(master, []); + for (const candidate of pool) { + const bucket = byMaster.get(candidate.master); + if (!bucket) continue; + if (bucket.length >= (allocation.get(candidate.master) ?? 0)) continue; + bucket.push(candidate); + } + // Subjects that could not fill their share hand it back, so a subject + // with no available candidates costs the run coverage rather than volume. + return interleave(byMaster).slice(0, TARGET_KEYWORDS); } - // Subjects that could not fill their share hand it back, so a subject with - // no available candidates costs the run coverage rather than volume. - const chosen = interleave(byMaster).slice(0, TARGET_KEYWORDS); + // The crosses are held back from the normal pass. + // + // They were being used as per-subject filler, and the first production run + // showed what that publishes: "saving money pricing", "deals platform", + // "coordination d0rz", "ai content loop". On-niche, gate-passing, and not + // article topics anybody would search for — bl0ggers' niche is + // "human-in-the-loop AI publishing", so its derived modifiers are literally + // "human" and "loop", and crossing a subject with those yields nonsense. + // + // Their real value is upstream: as DataForSEO seeds they are what turns + // "peptide" into "peptide merchant account". So they still seed every + // expansion — they just no longer get published on the strength of being + // grammatically adjacent to the niche. + let chosen = select(deduped.filter((c) => !c.fromCross)); + + // Last resort, and site-level rather than per-subject: only when the entire + // run would otherwise insert nothing does the constructed floor get used. + // That preserves "a blog with every upstream down still publishes" without + // letting constructions pad an otherwise healthy run. + let usedCrossFloor = false; + if (chosen.length === 0) { + chosen = select(deduped); + usedCrossFloor = chosen.length > 0; + if (usedCrossFloor) { + console.warn( + `[lx] ${site.domain}: no researched keywords survived; falling back to ${chosen.length} constructed subject x modifier topics`, + ); + } + } if (chosen.length === 0) { const details = sourceErrors.length > 0 @@ -593,5 +646,11 @@ export async function researchKeywords( perMaster[row.master_keyword] = (perMaster[row.master_keyword] ?? 0) + 1; } - return { ok: true, inserted: insertRows.length, apiCost: totalCost, perMaster }; + return { + ok: true, + inserted: insertRows.length, + apiCost: totalCost, + perMaster, + usedCrossFloor, + }; } diff --git a/lib/lx/topicPlan.ts b/lib/lx/topicPlan.ts index 7514129..dc7eb5c 100644 --- a/lib/lx/topicPlan.ts +++ b/lib/lx/topicPlan.ts @@ -223,6 +223,29 @@ export function resolveModifiers( * the very same word, satisfying a two-part test with one token. The anchor * has to be evidence the subject match did not already provide. */ +export function ownAnchorTokens( + site: SiteTopicFields, + masters: string[] = [], +): Set { + const masterTokens = new Set(masters.flatMap((m) => tokens(m).map(stem))); + const defaults = new Set(DEFAULT_MODIFIERS.flatMap((m) => tokens(m).map(stem))); + const out = new Set(); + + const add = (phrase: string) => { + for (const token of tokens(phrase)) { + const stemmed = stem(token); + if (!masterTokens.has(stemmed) && !defaults.has(stemmed)) out.add(stemmed); + } + }; + + const explicit = (site.modifiers ?? []) + .map((m) => (m ?? "").trim()) + .filter((m) => m.length > 0); + for (const modifier of explicit) add(modifier); + add(site.niche ?? ""); + return out; +} + export function anchorTokens( site: SiteTopicFields, masters: string[] = [], @@ -274,6 +297,7 @@ export function isOnNiche( keyword: string, master: string, anchors: Set, + ownAnchors?: Set, ): boolean { const candidate = new Set(tokens(keyword).map(stem)); if (candidate.size === 0) return false; @@ -284,6 +308,27 @@ export function isOnNiche( const hits = masterTokens.filter((t) => candidate.has(t)).length; if (hits === 0) return false; + // A PARTIAL match on a multi-word subject may only be rescued by an anchor + // the site itself supplied — never by the generic commercial vocabulary. + // + // The first production run leaked exactly this shape: "open standards" + // matched on "open" alone and the default anchor "software" then admitted + // "open broadcaster software"; "supply chain security" matched on "supply" + // and "automation" admitted "industrial automation supply"; "online + // shopping" matched on "online" and "pricing" admitted "quickbooks online + // pricing". One generic subject word plus one generic tail word is not + // evidence of anything, and the defaults are generic by construction. + // + // A complete subject match is unaffected — including a single-word subject, + // where "iptv" + "alternatives" is a real query. + if (ownAnchors && masterTokens.length > 1 && hits < masterTokens.length) { + if (ownAnchors.size === 0) return false; + for (const token of candidate) { + if (ownAnchors.has(token)) return true; + } + return false; + } + // A COMPLETE match on a multi-word subject is its own evidence, and needs no // anchor. // diff --git a/tests/lx/topic-plan-partial-match.test.ts b/tests/lx/topic-plan-partial-match.test.ts new file mode 100644 index 0000000..5f28b0b --- /dev/null +++ b/tests/lx/topic-plan-partial-match.test.ts @@ -0,0 +1,166 @@ +// What leaked on the first production run, pinned. +// +// The anchored gate shipped and killed the vendor/junk classes outright. The +// hourly cron then wrote 162 keywords across 11 sites, and eight of them were +// still wrong — all of one shape, none of it visible in the fixtures that +// existed before real output could be inspected. +// +// The shape: a *partial* match on one generic word of a multi-word subject, +// rescued by a word from the generic commercial vocabulary. "open standards" +// matched on "open", "software" did the rest, and the blog was queued to write +// about OBS Studio. One generic subject word plus one generic tail word is not +// evidence of anything. + +import { describe, expect, it } from "vitest"; +import { + anchorTokens, + isOnNiche, + ownAnchorTokens, + resolveMasters, +} from "@/lib/lx/topicPlan"; + +type Case = { + domain: string; + site: { master_keywords: string[]; modifiers: string[]; niche: string }; + leaked: Array<[string, string]>; + keeps: Array<[string, string]>; +}; + +const CASES: Case[] = [ + { + domain: "logicsrc.com", + site: { + master_keywords: [ + "ai agents", "agent orchestration", "developer tools", + "api integration", "open standards", "mcp", + ], + modifiers: [], + niche: "open AI agent standards", + }, + leaked: [ + // Both queued for real. OBS Studio, on a blog about agent standards. + ["open broadcaster software", "open standards"], + ["open source software", "open standards"], + ], + keeps: [ + // Complete subject matches, unaffected by the partial-match rule. + ["agent orchestration tools", "agent orchestration"], + ["ai agents platform", "ai agents"], + ["api integration comparison", "api integration"], + ], + }, + { + domain: "vu1nz.com", + site: { + master_keywords: [ + "ci/cd security", "supply chain security", "github actions security", + "devops security", "npm security", + ], + modifiers: [], + niche: "CI/CD and supply chain security", + }, + leaked: [["industrial automation supply", "supply chain security"]], + keeps: [ + ["supply chain security tools", "supply chain security"], + ["github actions security platform", "github actions security"], + ], + }, + { + domain: "c0upons.com", + site: { + master_keywords: [ + "coupon codes", "promo codes", "deals", "discount codes", + "online shopping", "saving money", + ], + modifiers: [], + niche: "coupon codes and savings", + }, + leaked: [ + ["quickbooks online pricing", "online shopping"], + ["quickbooks online simple start pricing", "online shopping"], + ], + keeps: [ + ["coupon codes for online shopping", "online shopping"], + ["how to save money online shopping", "online shopping"], + ], + }, + { + domain: "ugig.net", + site: { + master_keywords: [ + "freelancing", "gig economy", "AI tools", "remote work", "freelance jobs", + ], + modifiers: [], + niche: "AI-assisted freelancing and gig work", + }, + leaked: [ + ["software engineer remote", "remote work"], + ["work from home software engineer", "remote work"], + ], + keeps: [["ai assisted freelancing", "freelancing"]], + }, +]; + +describe.each(CASES)("$domain — partial-match leaks", ({ site, leaked, keeps }) => { + const masters = resolveMasters(site); + const anchors = anchorTokens(site, masters); + const own = ownAnchorTokens(site, masters); + + it.each(leaked)("rejects %j", (keyword, master) => { + expect(isOnNiche(keyword, master, anchors, own)).toBe(false); + }); + + it.each(keeps)("still keeps %j", (keyword, master) => { + expect(isOnNiche(keyword, master, anchors, own)).toBe(true); + }); +}); + +describe("ownAnchorTokens", () => { + it("excludes the generic vocabulary, which is the whole point", () => { + const site = { + master_keywords: ["open standards"], + modifiers: [], + niche: "open AI agent standards", + }; + const own = ownAnchorTokens(site, resolveMasters(site)); + // "software" and "pricing" are commercial-generic; they may complete a + // FULL subject match but must not rescue a partial one. + expect(own.has("software")).toBe(false); + expect(own.has("pricing")).toBe(false); + }); + + it("keeps a site's explicit modifiers", () => { + const site = { + master_keywords: ["peptide"], + modifiers: ["merchant account", "payment gateway"], + niche: "crypto payments for high-risk merchants", + }; + const own = ownAnchorTokens(site, resolveMasters(site)); + expect(own.has("merchant")).toBe(true); + expect(own.has("gateway")).toBe(true); + }); +}); + +describe("a complete subject match is still exempt", () => { + // Single-word subjects match completely by definition, so "iptv + // alternatives" is a real query and must survive the new rule. + const site = { + master_keywords: ["streaming", "torrents", "iptv", "media tech"], + modifiers: [], + niche: "streaming torrent media tech", + }; + const masters = resolveMasters(site); + const anchors = anchorTokens(site, masters); + const own = ownAnchorTokens(site, masters); + + it.each([ + ["iptv alternatives", "iptv"], + ["streaming media software", "streaming"], + ])("keeps %j", (keyword, master) => { + expect(isOnNiche(keyword, master, anchors, own)).toBe(true); + }); + + it("still rejects a partial match on the multi-word subject", () => { + expect(isOnNiche("micron technology", "media tech", anchors, own)).toBe(false); + }); +}); From eb8fc732b87f1f192de81cbc3d514811b7ffe696 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 28 Aug 2026 16:10:33 +0000 Subject: [PATCH 2/3] Tell a construction from a keyword when cleaning up after the fact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to purge-offniche-keywords.ts, for rows written before the gate was tightened. Two verdicts, both computed with the real planner rather than approximated in SQL. The second one needed care. A construction cannot be found by checking for a null search volume, which was the obvious approach: the buyer-journey model also returns keywords with no volume, and those are the best output in the run — "zeek vs suricata for network security monitoring" and "langchain alternatives for agents" both sit in the same null bucket as "deals platform". So a construction is identified by fingerprint against the cross set the planner would build for that site, which is exact rather than heuristic. Ran against the 162 rows from the first production cron: kept 53, removed 95 constructions and 14 partial-match leaks. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013cbcXdpLFUgJZuVfCB4Gzv --- scripts/purge-constructed-keywords.ts | 136 ++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 scripts/purge-constructed-keywords.ts diff --git a/scripts/purge-constructed-keywords.ts b/scripts/purge-constructed-keywords.ts new file mode 100644 index 0000000..308bd71 --- /dev/null +++ b/scripts/purge-constructed-keywords.ts @@ -0,0 +1,136 @@ +// Remove the two classes the first production run got wrong. +// +// Companion to purge-offniche-keywords.ts, for rows that pipeline wrote before +// PR #213 tightened it. Two verdicts, both computed with the real code rather +// than approximated in SQL: +// +// 1. **Partial-match leaks** — now rejected by `isOnNiche` once it is given +// the site's own anchors, so this just re-runs the gate. +// 2. **Constructions** — a keyword that IS a `subject x modifier` cross. +// These pass the gate (they are on-niche by construction) and cannot be +// found by a volume check, because the buyer-journey model also returns +// keywords with no volume and those are the best output in the run. +// Identified by fingerprint against the cross set the planner would build +// for that site, which is exact. +// +// Prints SQL rather than executing it. Only `queued` rows are ever considered. +// +// Usage: npx tsx scripts/purge-constructed-keywords.ts + +import { readFileSync } from "node:fs"; +import { + anchorTokens, + crossQueries, + isOnNiche, + ownAnchorTokens, + resolveMasters, + resolveModifiers, + signature, + stem, + tokens, +} from "../lib/lx/topicPlan"; + +type Row = { + id: string; + keyword: string; + master_keyword: string | null; + domain: string; + niche: string | null; + master_keywords: string[] | null; + modifiers: string[] | null; +}; + +function attribute(keyword: string, masters: string[]): string | null { + let best: string | null = null; + let bestLen = 0; + const candidate = new Set(tokens(keyword).map(stem)); + for (const master of masters) { + const masterTokens = tokens(master).map(stem); + if (masterTokens.length === 0) continue; + const hit = masterTokens.filter((t) => candidate.has(t)); + if (hit.length === 0) continue; + const len = hit.join("").length; + if (len > bestLen) { + bestLen = len; + best = master; + } + } + return best; +} + +function extractRows(raw: string): Row[] { + const slice = raw.slice(raw.indexOf("[{"), raw.lastIndexOf("}]") + 2); + const text = slice.includes('\\"') ? slice.replace(/\\"/g, '"') : slice; + const parsed = JSON.parse(text); + const first = parsed[0]; + return Array.isArray(first?.payload) ? first.payload : parsed; +} + +const rows: Row[] = extractRows(readFileSync(process.argv[2], "utf8")); + +// One cross-fingerprint set per site, built once. +const crossSigs = new Map>(); +function crossesFor(row: Row): Set { + const cached = crossSigs.get(row.domain); + if (cached) return cached; + const masters = resolveMasters(row); + // Depth well past the planner's 3, so a construction is caught regardless of + // how deep that run happened to go. + const built = crossQueries(masters, resolveModifiers(row, masters), 8); + const set = new Set(built.map((c) => signature(c.query))); + crossSigs.set(row.domain, set); + return set; +} + +type Bucket = { keep: string[]; leak: string[]; built: string[]; ids: string[] }; +const bySite = new Map(); + +for (const row of rows) { + const masters = resolveMasters(row); + const b = bySite.get(row.domain) ?? { keep: [], leak: [], built: [], ids: [] }; + + const master = row.master_keyword ?? attribute(row.keyword, masters); + const anchors = anchorTokens(row, masters); + const own = ownAnchorTokens(row, masters); + + const passes = + master !== null && isOnNiche(row.keyword, master, anchors, own); + const constructed = crossesFor(row).has(signature(row.keyword)); + + if (!passes) { + b.leak.push(row.keyword); + b.ids.push(row.id); + } else if (constructed) { + b.built.push(row.keyword); + b.ids.push(row.id); + } else { + b.keep.push(row.keyword); + } + bySite.set(row.domain, b); +} + +const allIds: string[] = []; +let k = 0; +let l = 0; +let c = 0; +for (const [domain, b] of Array.from(bySite).sort()) { + k += b.keep.length; + l += b.leak.length; + c += b.built.length; + allIds.push(...b.ids); + console.log(`\n=== ${domain}: keep ${b.keep.length}, leak ${b.leak.length}, constructed ${b.built.length}`); + if (b.leak.length) console.log(` leaked : ${b.leak.join(" | ")}`); + if (b.built.length) console.log(` constructed : ${b.built.slice(0, 10).join(" | ")}${b.built.length > 10 ? " | …" : ""}`); + if (b.keep.length) console.log(` keeping : ${b.keep.slice(0, 8).join(" | ")}${b.keep.length > 8 ? " | …" : ""}`); +} + +console.log(`\n--- keep ${k}, delete ${l + c} (${l} leaked, ${c} constructed) of ${rows.length}`); +console.log("\n-- SQL:"); +for (let i = 0; i < allIds.length; i += 200) { + console.log( + `delete from lx_keyword where status='queued' and id in (${allIds + .slice(i, i + 200) + .map((id) => `'${id}'`) + .join(",")});`, + ); +} From 1f65a14d187cccb891683c1027a0c9508fce3128 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 28 Aug 2026 16:16:41 +0000 Subject: [PATCH 3/3] Record why a keyword failed, instead of only logging it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishing stopped on 2026-08-19 and nobody could tell why. All 308 failed lx_keyword rows carry a null status_reason, because failKeyword() wrote the reason to console.warn and nothing else — so it lived in the Railway log buffer and expired there. From the database and the dashboard the outage was invisible: a number going up, with no cause attached. Every caller already computes a precise reason ("embedding failed: …", "quality gate failed after N attempts (slop=…)", "insert failed: …"). It was simply not being kept. Now it lands on the row, truncated, with a fallback write so a keyword that cannot record its reason is still marked failed rather than retried forever. The quota path gets the same treatment even though it requeues rather than fails, because it is the one outcome that leaves no trace anywhere: the keyword looks untouched, no article row is created, nothing reaches the model, and the dashboard shows a healthy queue while the site quietly stops publishing. That is the exact shape of this outage. Diagnosis so far, recorded here because the evidence is easy to lose: article generation is dying BEFORE the writer model. ai_usage shows zero lx_article calls since 2026-08-19 while buyer_journey_keyword_research ran today on the same OpenAI key; credits are not the blocker (3,583 on the account, 20 per article); and text-embedding-3-small answers normally when called directly. The next 09:00 run will now say which step it is. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013cbcXdpLFUgJZuVfCB4Gzv --- lib/lx/articleGen.ts | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/lib/lx/articleGen.ts b/lib/lx/articleGen.ts index ef47471..16f36be 100644 --- a/lib/lx/articleGen.ts +++ b/lib/lx/articleGen.ts @@ -1057,9 +1057,19 @@ export async function generateArticle( }; if (chargeSource === "none") { // Return the claim — the keyword can run later once credits exist. + // + // The reason is written even though the row goes back to `queued`, because + // this is the one failure that leaves no trace anywhere else: the keyword + // looks untouched, no article row is created, and nothing reaches the + // model, so a site silently stops publishing while its dashboard shows a + // healthy queue. That is exactly the shape of the outage that ran from + // 2026-08-19 undetected. await supabase .from("lx_keyword") - .update({ status: "queued" }) + .update({ + status: "queued", + status_reason: "out of article quota and credits", + }) .eq("id", keyword.id); return { ok: false, error: "out of article quota and credits" }; } @@ -1500,16 +1510,37 @@ export async function generateArticle( return { ok: true, articleId: inserted.id, slug: finalSlug }; } +/** + * Mark a keyword failed, and record WHY on the row. + * + * The reason used to go only to `console.warn`, which meant it lived in the + * Railway log buffer and nowhere else. That cost nine days: publishing stopped + * on 2026-08-19 and every one of the 308 failed rows carried a null + * `status_reason`, so the outage was invisible from the database and from the + * dashboard — the only symptom was a number going up. Every caller here + * already computes a precise reason ("embedding failed: …", "quality gate + * failed after N attempts (slop=…)"); it simply was not being kept. + * + * Best-effort and never throws: a keyword that cannot record its reason must + * still be marked failed, or the generator retries it forever. + */ async function failKeyword( supabase: SupabaseClient, keywordId: string, reason: string, ): Promise { console.warn(`[lx] keyword ${keywordId} failed:`, reason); - await supabase + const { error } = await supabase .from("lx_keyword") - .update({ status: "failed" }) + .update({ status: "failed", status_reason: reason.slice(0, 2000) }) .eq("id", keywordId); + if (error) { + console.warn(`[lx] keyword ${keywordId}: could not record reason:`, error.message); + await supabase + .from("lx_keyword") + .update({ status: "failed" }) + .eq("id", keywordId); + } } // Bump the user's credit balance back by SCAN_CREDITS — called when generation