From e3e8a2686a897ab52f318d8dfa5b1cae8373987c Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:04:06 +0200 Subject: [PATCH 01/14] fix(content): validate generated database identifiers across content types `assertGeneratedTableNames` only ever looked inside one content type, so two of them could reach the same physical junction or repeatable child table without either author writing the name down - the clamp to 63 characters makes that reachable, and Postgres truncates silently rather than complaining. The registry now holds one map for every physical table a content type puts into the schema (base, translation, junction, child) and one for every index and constraint name (declared, translation, junction primary key, junction position, junction target, repeatable position). Both are schema-wide namespaces, so both are checked schema-wide. Co-Authored-By: Claude Opus 5 (1M context) --- .../modules/admin/debug/debug.admin.module.ts | 2 + .../debug/routes/content-status.route.ts | 83 +++++ packages/vitnode/src/content/registry.test.ts | 200 +++++++++++ packages/vitnode/src/content/registry.ts | 140 ++++++-- .../vitnode/src/content/server/diagnostics.ts | 326 ++++++++++++++++++ .../src/content/server/editorial-effects.ts | 13 + .../vitnode/src/content/server/effects-log.ts | 81 +++++ packages/vitnode/src/content/server/index.ts | 16 + .../src/content/server/translation-effects.ts | 13 + 9 files changed, 845 insertions(+), 29 deletions(-) create mode 100644 packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts create mode 100644 packages/vitnode/src/content/server/diagnostics.ts create mode 100644 packages/vitnode/src/content/server/effects-log.ts diff --git a/packages/vitnode/src/api/modules/admin/debug/debug.admin.module.ts b/packages/vitnode/src/api/modules/admin/debug/debug.admin.module.ts index dc77d8d7d..dd9af296c 100644 --- a/packages/vitnode/src/api/modules/admin/debug/debug.admin.module.ts +++ b/packages/vitnode/src/api/modules/admin/debug/debug.admin.module.ts @@ -1,6 +1,7 @@ import { CONFIG_PLUGIN } from "../../../../config"; import { buildModule } from "../../../lib/module"; import { clearSearchDebugAdminRoute } from "./routes/clear-search.route"; +import { contentStatusDebugAdminRoute } from "./routes/content-status.route"; import { integrationsDebugAdminRoute } from "./routes/integrations.route"; import { logsDebugAdminRoute } from "./routes/logs.route"; import { queueDebugAdminRoute } from "./routes/queue.route"; @@ -20,6 +21,7 @@ export const debugAdminModule = buildModule({ searchStatusDebugAdminRoute, rebuildSearchDebugAdminRoute, clearSearchDebugAdminRoute, + contentStatusDebugAdminRoute, sendTestEmailDebugAdminRoute, testAiDebugAdminRoute, testStorageUploadDebugAdminRoute, diff --git a/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts b/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts new file mode 100644 index 000000000..feeaf8cb3 --- /dev/null +++ b/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts @@ -0,0 +1,83 @@ +import { z } from "zod"; + +import { buildRoute } from "@/api/lib/route"; +import { CONFIG_PLUGIN } from "@/config"; +import { contentEngineDiagnostics } from "@/content/server/diagnostics"; + +const localeDriftSchema = z.object({ + /** Published rows - or published translations - the database holds. */ + expected: z.number(), + healthy: z.boolean(), + indexed: z.number(), + /** `""` for a content type that is not localized. */ + locale: z.string(), +}); + +const contentTypeSchema = z.object({ + contentTypeId: z.string(), + features: z.object({ + editorial: z.boolean(), + localization: z.boolean(), + publicApi: z.boolean(), + publication: z.boolean(), + scheduling: z.boolean(), + search: z.boolean(), + }), + pluginId: z.string(), + /** `null` for a content type without `search`. */ + search: z + .object({ + contentTypeId: z.string(), + healthy: z.boolean(), + locales: z.array(localeDriftSchema), + }) + .nullable(), + /** `null` for a content type without scheduling. */ + schedules: z + .object({ + /** Transitions that committed but were never announced. */ + failedEffects: z.number(), + pending: z.number(), + withErrors: z.number(), + }) + .nullable(), +}); + +/** + * What the Content Engine looks like from the outside, right now. + * + * Sits beside `/search/status` under the same `system: can_view` permission, + * and answers the questions that one cannot: `/search/status` reports what is + * *in* the index, and this reports what the **database** says should be there. + * A collection can be 100% covered by the first and still be missing every + * Polish document, because coverage is measured against the indexer's own count + * and drift is measured against the rows. + * + * Aggregates only - two counts per content type - so it is safe to open on an + * install with a large table. Nothing here mutates anything; repairing drift is + * `/search/rebuild`, which is a separate decision and a separate route. + */ +export const contentStatusDebugAdminRoute = buildRoute({ + pluginId: CONFIG_PLUGIN.pluginId, + adminStaffPermission: { module: "system", permission: "can_view" }, + route: { + method: "get", + description: + "Report every registered content type, its search index drift per locale, and its outstanding scheduled-effect failures.", + path: "/content/status", + responses: { + 200: { + content: { + "application/json": { + schema: z.object({ + contentTypes: z.array(contentTypeSchema), + healthy: z.boolean(), + }), + }, + }, + description: "Content Engine status", + }, + }, + }, + handler: async c => c.json(await contentEngineDiagnostics(c)), +}); diff --git a/packages/vitnode/src/content/registry.test.ts b/packages/vitnode/src/content/registry.test.ts index 541df4c8f..e7ec20307 100644 --- a/packages/vitnode/src/content/registry.test.ts +++ b/packages/vitnode/src/content/registry.test.ts @@ -393,3 +393,203 @@ describe("public paths", () => { ).not.toThrow(); }); }); + +/** + * Names nobody wrote down. + * + * A junction or repeatable child table is generated from a field name, snake- + * cased and clamped to Postgres' 63-character limit - so two content types can + * reach the same physical table without either author ever typing it. Postgres + * would not complain: the second `CREATE TABLE` simply never runs, and from then + * on two definitions read and write one table. The same applies one level down, + * to the `UNIQUE (itemId, position)` constraint each of them carries. + */ +describe("generated database identifiers", () => { + const related = ( + id: string, + tableName: string, + fields: Parameters[0]["fields"], + ) => + defineContentType({ + id, + tableName, + fields, + admin: { + label: { plural: "Things", singular: "Thing" }, + permissionModule: tableName, + form: { fields: Object.keys(fields) }, + }, + }); + + const withRelation = (id: string, tableName: string, field_: string) => + related(id, tableName, { + title: field.text({ required: true }), + [field_]: field.relation({ + multiple: true, + target: () => testCategoryContentType, + }), + }); + + const withRepeatable = (id: string, tableName: string, field_: string) => + related(id, tableName, { + title: field.text({ required: true }), + [field_]: field.repeatable({ + fields: { question: field.text({ required: true }) }, + }), + }); + + it("accepts advanced content types whose generated names differ", () => { + expect(() => + validateContentTypes([ + entry(withRelation("test.one", "test_ones", "tags")), + entry(withRepeatable("test.two", "test_twos", "faq")), + ]), + ).not.toThrow(); + }); + + it("rejects a base table that collides with another type's junction table", () => { + // `test_ones` + `tags` generates `test_ones_tags`, which is exactly the base + // table the second content type declares. Silently sharing it would give one + // definition's rows two owners. + expect(() => + validateContentTypes([ + entry(withRelation("test.one", "test_ones", "tags"), "@vitnode/a"), + entry( + related("test.two", "test_ones_tags", { + title: field.text({ required: true }), + }), + "@vitnode/b", + ), + ]), + ).toThrow(/Table "test_ones_tags" is claimed by both/); + }); + + it("rejects a base table that collides with another type's repeatable table", () => { + expect(() => + validateContentTypes([ + entry( + related("test.two", "test_ones_faq", { + title: field.text({ required: true }), + }), + "@vitnode/b", + ), + entry(withRepeatable("test.one", "test_ones", "faq"), "@vitnode/a"), + ]), + ).toThrow(/Table "test_ones_faq" is claimed by both/); + }); + + it("names the field that generated the colliding table", () => { + // A boot error is only useful if it says which of the two to rename, and the + // generated side is the one that has no obvious name to look for. + expect(() => + validateContentTypes([ + entry(withRelation("test.one", "test_ones", "tags"), "@vitnode/a"), + entry( + related("test.two", "test_ones_tags", { + title: field.text({ required: true }), + }), + "@vitnode/b", + ), + ]), + ).toThrow(/the junction table of "tags"/); + }); + + it("rejects two content types generating the same junction table", () => { + // Different content types, different fields, one physical table: the + // second's junction is named after the first's. + expect(() => + validateContentTypes([ + entry(withRelation("test.one", "test_ones", "tags"), "@vitnode/a"), + entry( + withRelation("test.two", "test_ones_tags", "labels"), + "@vitnode/b", + ), + ]), + ).toThrow(/Table "test_ones_tags" is claimed by both/); + }); + + it("registers the generated position constraint in the index namespace", () => { + // `test_ones` + `faq` generates `test_ones_faq_position_key`. An explicit + // index of that name on another content type would be the same identifier + // in the same schema, and only one of the two would exist. + expect(() => + validateContentTypes([ + entry(withRepeatable("test.one", "test_ones", "faq"), "@vitnode/a"), + entry( + related("test.two", "test_twos", { + title: field.text({ required: true }), + }), + "@vitnode/b", + ), + ]), + ).not.toThrow(); + + expect(() => + validateContentTypes([ + entry(withRepeatable("test.one", "test_ones", "faq"), "@vitnode/a"), + entry( + defineContentType({ + id: "test.two", + tableName: "test_twos", + fields: { title: field.text({ required: true }) }, + indexes: [{ name: "test_ones_faq_position_key", on: ["title"] }], + admin: { + label: { plural: "Twos", singular: "Two" }, + permissionModule: "test_twos", + }, + }), + "@vitnode/b", + ), + ]), + ).toThrow(/Index name "test_ones_faq_position_key" is used by both/); + }); + + it("registers a junction's primary key and target index too", () => { + for (const name of [ + "test_ones_tags_pk", + "test_ones_tags_related_item_id_idx", + ]) { + expect(() => + validateContentTypes([ + entry(withRelation("test.one", "test_ones", "tags"), "@vitnode/a"), + entry( + defineContentType({ + id: "test.two", + tableName: "test_twos", + fields: { title: field.text({ required: true }) }, + indexes: [{ name, on: ["title"] }], + admin: { + label: { plural: "Twos", singular: "Two" }, + permissionModule: "test_twos", + }, + }), + "@vitnode/b", + ), + ]), + ).toThrow(new RegExp(`Index name "${name}" is used by both`)); + } + }); + + it("keeps two long generated table names apart by their fingerprint", () => { + // The clamp is what makes a collision reachable at all; the fingerprint is + // what makes it vanishingly unlikely. Both names fill the limit exactly and + // still differ, so the pair boots. + const base = `t_${"a".repeat(56)}`; + const first = withRepeatable("test.long1", `${base}_x`, "faq"); + const second = withRepeatable("test.long2", `${base}_y`, "faq"); + + const [firstTable, secondTable] = [first, second].map( + definition => definition.advanced.repeatables[0].tableName, + ); + + expect(firstTable).toHaveLength(63); + expect(secondTable).toHaveLength(63); + expect(firstTable).not.toBe(secondTable); + expect(() => + validateContentTypes([ + entry(first, "@vitnode/a"), + entry(second, "@vitnode/b"), + ]), + ).not.toThrow(); + }); +}); diff --git a/packages/vitnode/src/content/registry.ts b/packages/vitnode/src/content/registry.ts index 2c71a5723..ba49e703d 100644 --- a/packages/vitnode/src/content/registry.ts +++ b/packages/vitnode/src/content/registry.ts @@ -31,6 +31,96 @@ interface IndexOwner { entry: RegisteredContentType; } +/** A physical table name, and the field or content type that generated it. */ +interface TableOwner { + entry: RegisteredContentType; + /** How the name came about, for an error message somebody has to act on. */ + origin: string; +} + +const describeTableOwner = (owner: TableOwner): string => + `${describe(owner.entry)} (${owner.origin})`; + +/** + * Every physical table one content type puts into the schema. + * + * The base table and the translation table are declared; a junction and a + * repeatable child table are **generated** from a field name, clamped to + * Postgres' 63-character limit. Two content types can therefore reach the same + * physical name without either author writing it down - which Postgres would + * accept, because the second `CREATE TABLE` simply never happens and both + * definitions then read and write one table. + */ +const physicalTables = ( + entry: RegisteredContentType, +): { name: string; origin: string }[] => { + const { definition } = entry; + + return [ + { name: definition.tableName, origin: "its base table" }, + ...(definition.localization.enabled + ? [ + { + name: definition.localization.translationTableName, + origin: "its translation table", + }, + ] + : []), + ...definition.advanced.junctions.map(junction => ({ + name: junction.tableName, + origin: `the junction table of "${junction.field}"`, + })), + ...definition.advanced.repeatables.map(repeatable => ({ + name: repeatable.tableName, + origin: `the child table of "${repeatable.field}"`, + })), + ]; +}; + +/** + * Every index and constraint name one content type puts into the schema. + * + * Same namespace problem as the tables, one level down: `UNIQUE (itemId, + * position)` on a junction is named after the junction, which is named after a + * field, which is clamped. Postgres keeps index and constraint names unique per + * schema, so two of them colliding is a migration that fails at deploy time - + * or, with the truncation, one that silently constrains the wrong table. + */ +const physicalIndexes = ( + entry: RegisteredContentType, +): { columns: string[]; name: string }[] => { + const { definition } = entry; + + return [ + ...definition.indexes.map(index => ({ + columns: index.on, + name: index.name, + })), + ...definition.localization.translationIndexes.map(index => ({ + columns: index.on, + name: index.name, + })), + ...definition.advanced.junctions.flatMap(junction => [ + { + columns: [`${junction.field} primary key`], + name: junction.primaryKeyName, + }, + { + columns: [`${junction.field} position`], + name: junction.positionIndexName, + }, + { + columns: [`${junction.field} target`], + name: junction.relatedIndexName, + }, + ]), + ...definition.advanced.repeatables.map(repeatable => ({ + columns: [`${repeatable.field} position`], + name: repeatable.positionIndexName, + })), + ]; +}; + /** * Validates a set of content types coming from one or more plugins. * @@ -48,7 +138,7 @@ export const validateContentTypes = ( entries: RegisteredContentType[], ): RegisteredContentType[] => { const byId = new Map(); - const byTable = new Map(); + const byTable = new Map(); const byPermission = new Map(); const byPublicPath = new Map(); const byIndexName = new Map(); @@ -65,30 +155,22 @@ export const validateContentTypes = ( } byId.set(definition.id, entry); - const duplicateTable = byTable.get(definition.tableName); - if (duplicateTable) { - throw new ContentEngineError( - `Table "${definition.tableName}" is claimed by both ${describe(duplicateTable)} and ${describe(entry)}.`, - { contentTypeId: definition.id }, - ); - } - byTable.set(definition.tableName, entry); - - // The generated translation table shares one namespace with every base - // table, so a content type called `example_articles_translations` and a - // localized `example_articles` would collide - and the shortening clamp - // makes that reachable with two long names that differ only past character - // 63. Both directions are caught by holding one map. - if (definition.localization.enabled) { - const translationTable = definition.localization.translationTableName; - const duplicateTranslationTable = byTable.get(translationTable); - if (duplicateTranslationTable) { + // Base, translation, junction and child tables all in one map: they share + // one Postgres namespace, so a content type called + // `example_articles_translations` and a localized `example_articles` would + // collide - and so would a repeatable called `tags` on `example_articles` + // and a content type whose own table is `example_articles_tags`. The + // shortening clamp makes every one of those reachable with two long names + // that differ only past character 63, and Postgres truncates silently. + for (const { name, origin } of physicalTables(entry)) { + const owner = byTable.get(name); + if (owner) { throw new ContentEngineError( - `Translation table "${translationTable}" is claimed by both ${describe(duplicateTranslationTable)} and ${describe(entry)}. Rename one of the base tables.`, + `Table "${name}" is claimed by both ${describeTableOwner(owner)} and ${describeTableOwner({ entry, origin })}. Two content types cannot share a physical table - rename one of them, or the field that generates it.`, { contentTypeId: definition.id }, ); } - byTable.set(translationTable, entry); + byTable.set(name, { entry, origin }); } // Permission modules are scoped per plugin, so only a collision inside one @@ -125,20 +207,20 @@ export const validateContentTypes = ( } // `resolveContentIndexes` already rejects a collision inside one content - // type. Postgres index names are unique per *schema*, though, so two - // content types - from one plugin or from two - cannot share one either. - for (const index of [ - ...definition.indexes, - ...definition.localization.translationIndexes, - ]) { + // type. Postgres index and constraint names are unique per *schema*, + // though, so two content types - from one plugin or from two - cannot share + // one either. The generated junction and repeatable names are in here too: + // they are derived from a field name and clamped, so they are exactly the + // ones nobody wrote down and nobody would think to check. + for (const index of physicalIndexes(entry)) { const owner = byIndexName.get(index.name); if (owner) { throw new ContentEngineError( - `Index name "${index.name}" is used by both ${describeIndexOwner(owner)} and ${describeIndexOwner({ columns: index.on, entry })}. Postgres index names are unique per schema, so rename one of them.`, + `Index name "${index.name}" is used by both ${describeIndexOwner(owner)} and ${describeIndexOwner({ columns: index.columns, entry })}. Postgres index names are unique per schema, so rename one of them.`, { contentTypeId: definition.id }, ); } - byIndexName.set(index.name, { columns: index.on, entry }); + byIndexName.set(index.name, { columns: index.columns, entry }); } } diff --git a/packages/vitnode/src/content/server/diagnostics.ts b/packages/vitnode/src/content/server/diagnostics.ts new file mode 100644 index 000000000..e5a5372d8 --- /dev/null +++ b/packages/vitnode/src/content/server/diagnostics.ts @@ -0,0 +1,326 @@ +import type { PgColumn, PgTable } from "drizzle-orm/pg-core"; +import type { Context } from "hono"; + +import { and, count, eq, inArray, sql } from "drizzle-orm"; + +import type { RegisteredContentModel } from "./model"; + +import { core_content_schedules } from "../../database/content"; +import { core_search_index } from "../../database/search"; +import { normalizeContentLocale } from "../locale"; +import { listContentLanguages } from "./language-resolver"; +import { + contentTranslationPublicationColumns, + publicationColumns, + publishedCondition, +} from "./publication"; + +/** + * Operational diagnostics for the Content Engine. + * + * Deliberately small, and deliberately **not** a monitoring product. It answers + * three questions an operator actually asks at three in the morning, and + * nothing else: + * + * 1. *Is the search index telling the truth?* - the database is the source of + * truth, so "how many documents should there be" is a `COUNT` over published + * rows, and "how many are there" is a `COUNT` over `core_search_index`. A + * difference is drift, and drift is repaired by a rebuild. + * 2. *Did anything scheduled fail to announce itself?* - a scheduled transition + * that committed but whose event, index write or cache expiry did not is + * recorded on the schedule row as `effectsError`. One count per content type + * turns "somewhere in the install" into "this content type". + * 3. *Which content types are even in play?* - what is registered, and which of + * the optional subsystems each one has switched on. + * + * No Prometheus, no time series, no dashboards - the install has none of those + * and Stage 7 does not add them. Everything here is a handful of aggregate + * queries, computed on demand, behind an admin permission. + */ + +/** One locale's share of a content type's index, expected against actual. */ +export interface ContentSearchDriftLocale { + /** + * `true` when the two counts agree. + * + * A count is not a checksum: two documents can be stale and still count as + * two. It is the cheap check that catches the failure that actually happens - + * a live sync that threw, or a rebuild that stopped halfway - and it costs two + * aggregates rather than a full comparison. + */ + healthy: boolean; + /** Documents actually in the index for this locale. */ + indexed: number; + /** + * `""` for a content type that is not localized. + * + * The empty string is what `core_search_index` stores for language-agnostic + * content, so it is the honest key here rather than `null` - it is the value + * the row really holds. + */ + locale: string; + /** Published rows (or published translations) the database holds. */ + expected: number; +} + +export interface ContentSearchDrift { + contentTypeId: string; + /** `true` when every locale agrees. */ + healthy: boolean; + locales: ContentSearchDriftLocale[]; +} + +/** + * Compares what the database says should be indexed against what is. + * + * Two aggregates per content type, whatever its size: `GROUP BY languageCode` + * over the index, and one `COUNT` (or one grouped `COUNT` for a localized + * content type) over the published rows. Neither reads a row body, so this stays + * cheap on a table with a hundred thousand articles. + * + * The expected side is deliberately the **same predicate the indexer uses** - + * `publishedCondition`, and for a localized content type the base row's + * predicate `AND` the translation's. Re-deriving "published" here would let the + * diagnostic disagree with the thing it is diagnosing, which is the one way a + * health check is worse than none. + */ +export const contentSearchDrift = async ( + c: Context, + { model }: Pick, +): Promise => { + const { definition } = model; + const contentTypeId = definition.id; + + const indexedRows = await c + .get("db") + .select({ + documents: count(), + languageCode: core_search_index.languageCode, + }) + .from(core_search_index) + .where(eq(core_search_index.itemType, contentTypeId)) + .groupBy(core_search_index.languageCode); + + const indexedByLocale = new Map( + indexedRows.map(row => [ + normalizeContentLocale(row.languageCode), + row.documents, + ]), + ); + + const expectedByLocale = await expectedDocuments(c, model); + + const locales = [ + ...new Set([...expectedByLocale.keys(), ...indexedByLocale.keys()]), + ] + .sort() + .map(locale => { + const expected = expectedByLocale.get(locale) ?? 0; + const indexed = indexedByLocale.get(locale) ?? 0; + + return { expected, healthy: expected === indexed, indexed, locale }; + }); + + return { + contentTypeId, + healthy: locales.every(entry => entry.healthy), + locales, + }; +}; + +/** + * How many documents this content type's rows *should* produce, per locale. + * + * `publication` is what makes a row indexable at all, so a content type without + * it has nothing to expect and returns an empty map rather than counting every + * draft - the indexer would not have written them either. + */ +const expectedDocuments = async ( + c: Context, + model: RegisteredContentModel["model"], +): Promise> => { + const { definition } = model; + const columns = model.columns as Record; + + if (!definition.publication.enabled) return new Map(); + + const published = publicationColumns(definition, columns); + + if (!definition.localization.enabled) { + const [row] = await c + .get("db") + .select({ value: count() }) + .from(model.table) + .where(publishedCondition(published)); + + // The empty locale, because that is the `languageCode` a non-localized + // document is stored under - not a missing value, an actual `''`. + return new Map([["", row?.value ?? 0]]); + } + + const translationTable: null | PgTable = model.translationTable; + const translationColumns: null | Record = + model.translationColumns; + if (!translationTable || !translationColumns) return new Map(); + + const translation = contentTranslationPublicationColumns( + definition, + translationColumns, + ); + + const rows = await c + .get("db") + .select({ + languageId: translationColumns.languageId, + value: count(), + }) + .from(translationTable) + .innerJoin(model.table, eq(translationColumns.itemId, columns.id)) + .where(and(publishedCondition(published), publishedCondition(translation))) + .groupBy(translationColumns.languageId); + + const languages = await listContentLanguages(c); + const localeOf = new Map( + languages.map(language => [language.id, language.locale]), + ); + + const byLocale = new Map(); + for (const row of rows) { + const locale = localeOf.get(row.languageId as number); + // A translation whose language row is gone indexes under no locale, so it is + // expected to produce no document - which is exactly what the indexer does + // with it. Counting it here would report permanent drift nothing can repair. + if (locale === undefined) continue; + + const key = normalizeContentLocale(locale); + byLocale.set(key, (byLocale.get(key) ?? 0) + row.value); + } + + return byLocale; +}; + +export interface ContentScheduleHealth { + /** + * Transitions that committed but whose announcements have not been delivered. + * + * The number worth alerting on: the record *is* published, and nobody has been + * told. The effects task retries on the queue's backoff, so a non-zero value + * that stays non-zero is an outage rather than a blip. + */ + failedEffects: number; + /** Bookings still waiting to fire. */ + pending: number; + /** Pending bookings whose last run threw. */ + withErrors: number; +} + +/** + * Schedule health for a set of content types, in one query. + * + * Grouped rather than looped: an install with thirty schedulable content types + * should cost one aggregate, not thirty round trips. + */ +export const contentScheduleHealth = async ( + c: Context, + contentTypeIds: readonly string[], +): Promise> => { + const result = new Map(); + if (contentTypeIds.length === 0) return result; + + const rows = await c + .get("db") + .select({ + contentTypeId: core_content_schedules.contentTypeId, + failedEffects: sql`count(*) filter (where ${core_content_schedules.effectsError} is not null)::int`, + pending: sql`count(*) filter (where ${core_content_schedules.status} = 'pending')::int`, + withErrors: sql`count(*) filter (where ${core_content_schedules.status} = 'pending' and ${core_content_schedules.lastError} is not null)::int`, + }) + .from(core_content_schedules) + .where(inArray(core_content_schedules.contentTypeId, [...contentTypeIds])) + .groupBy(core_content_schedules.contentTypeId); + + for (const row of rows) { + result.set(row.contentTypeId, { + failedEffects: Number(row.failedEffects), + pending: Number(row.pending), + withErrors: Number(row.withErrors), + }); + } + + return result; +}; + +export interface ContentTypeDiagnostic { + contentTypeId: string; + features: { + editorial: boolean; + localization: boolean; + publicApi: boolean; + publication: boolean; + scheduling: boolean; + search: boolean; + }; + pluginId: string; + /** `null` for a content type without `search`, which indexes nothing. */ + search: ContentSearchDrift | null; + /** `null` for a content type without scheduling, which books nothing. */ + schedules: ContentScheduleHealth | null; +} + +export interface ContentEngineDiagnostics { + contentTypes: ContentTypeDiagnostic[]; + /** `true` when every searchable content type's index agrees with the database. */ + healthy: boolean; +} + +/** + * One pass over every registered content type. + * + * Sorted by id so two calls - and two processes - report the same order, which + * is what makes a diff between them readable. + */ +export const contentEngineDiagnostics = async ( + c: Context, +): Promise => { + const registered = [...(c.get("core")?.contentModels ?? [])].sort((a, b) => + a.model.definition.id.localeCompare(b.model.definition.id), + ); + + const schedulable = registered + .filter(entry => entry.model.definition.editorial.scheduling.enabled) + .map(entry => entry.model.definition.id); + const schedules = await contentScheduleHealth(c, schedulable); + + const contentTypes: ContentTypeDiagnostic[] = []; + for (const entry of registered) { + const { definition } = entry.model; + + contentTypes.push({ + contentTypeId: definition.id, + features: { + editorial: definition.editorial.enabled, + localization: definition.localization.enabled, + publicApi: definition.publicApi.enabled, + publication: definition.publication.enabled, + scheduling: definition.editorial.scheduling.enabled, + search: definition.search.enabled, + }, + pluginId: entry.pluginId, + search: definition.search.enabled + ? await contentSearchDrift(c, entry) + : null, + schedules: definition.editorial.scheduling.enabled + ? (schedules.get(definition.id) ?? { + failedEffects: 0, + pending: 0, + withErrors: 0, + }) + : null, + }); + } + + return { + contentTypes, + healthy: contentTypes.every(entry => entry.search?.healthy !== false), + }; +}; diff --git a/packages/vitnode/src/content/server/editorial-effects.ts b/packages/vitnode/src/content/server/editorial-effects.ts index 536551c3b..52df197e0 100644 --- a/packages/vitnode/src/content/server/editorial-effects.ts +++ b/packages/vitnode/src/content/server/editorial-effects.ts @@ -7,6 +7,7 @@ import type { ContentEditorialOutcome } from "./editorial-service"; import type { AnyContentModel } from "./model"; import type { ContentSearchSyncOutcome } from "./search-sync"; +import { reportContentEventFailures } from "./effects-log"; import { emitContentEvent } from "./emit"; import { contentSearchAdvancedValues, @@ -160,6 +161,18 @@ export const contentEditorialEffects = async ( { pluginId }, ); + // Logged here rather than left to each caller. The scheduled path additionally + // records the failure on the schedule row and retries; an interactive route + // does neither, and without this a dead listener on a clicked publish would be + // invisible everywhere. The write has committed either way, so the response + // stays a success - see `reportContentEventFailures`. + await reportContentEventFailures(c, { + action: EVENT_ACTION[outcome.operation], + contentTypeId: definition.id, + event, + itemId: idOf(outcome.row), + }); + // A localized record is indexed once per published translation, and a mutation // of the *record* moves every one of them: its publication state gates them // all, and a shared field is in all of them. diff --git a/packages/vitnode/src/content/server/effects-log.ts b/packages/vitnode/src/content/server/effects-log.ts new file mode 100644 index 000000000..993b9a626 --- /dev/null +++ b/packages/vitnode/src/content/server/effects-log.ts @@ -0,0 +1,81 @@ +import type { Context } from "hono"; + +import type { EventEmitResult } from "../../api/models/events"; + +/** + * The prefix every Content Engine post-commit failure is logged behind. + * + * Greppable on purpose, and distinct from `[content-search]`, which + * `syncContentSearch` already owns: an operator looking for "why did nobody + * hear about this publish" is asking a different question from "why is this + * article missing from search", and one prefix for both would make neither + * answerable. + */ +export const CONTENT_EFFECTS_LOG_PREFIX = "[content-effects]"; + +/** + * Reports listeners that did not receive an event whose mutation **has already + * committed**. + * + * `EventsModel.emit` reports rather than throws, so `failures` is the only place + * a dead listener or a broker outage is visible at all. Two things follow from + * the write having committed, and they are the whole contract: + * + * 1. **The request still succeeds.** The row is in the database; answering 500 + * would tell the client its edit was lost when it was not, and it would + * invite a retry that creates a second version of everything. + * 2. **The failure is never swallowed.** It goes to `core_logs` behind + * {@link CONTENT_EFFECTS_LOG_PREFIX} with the content type, the item and the + * listener that failed, so the AdminCP log viewer can find it and an operator + * can replay whatever the listener was meant to do. + * + * Delivery is **at-least-once** where a retry is involved (the scheduled effects + * task) and best-effort otherwise (an interactive route). There is no outbox and + * no exactly-once guarantee; a listener that must act once keys off the + * identifiers in the payload. + * + * A result with no failures logs nothing - an expected success is not an error, + * and a log full of them is a log nobody reads. + */ +export const reportContentEventFailures = async ( + c: Context, + { + action, + contentTypeId, + event, + itemId, + locale, + }: { + action: string; + contentTypeId: string; + event: EventEmitResult | null; + itemId: number; + /** Present only for a translation mutation. */ + locale?: string; + }, +): Promise => { + if (!event || event.failures.length === 0) return; + + const message = `${CONTENT_EFFECTS_LOG_PREFIX} ${JSON.stringify({ + action, + contentTypeId, + delivered: event.delivered, + eventId: event.eventId, + failures: event.failures.map(failure => ({ + error: failure.error, + listener: `${failure.pluginId}:${failure.module}:${failure.listener}`, + })), + itemId, + ...(locale === undefined ? {} : { locale }), + })}`; + + try { + await c.get("log").error(message); + } catch { + // The logger writes to the database, so it can fail for the same reason the + // transport did. Both are best effort *after* a committed write, and neither + // may turn it into a failed request - so the console is the last resort. + // eslint-disable-next-line no-console + console.error(`[VitNode] ${message}`); + } +}; diff --git a/packages/vitnode/src/content/server/index.ts b/packages/vitnode/src/content/server/index.ts index 3e95553ac..298276c57 100644 --- a/packages/vitnode/src/content/server/index.ts +++ b/packages/vitnode/src/content/server/index.ts @@ -16,11 +16,27 @@ export { buildTranslationSystemColumns, } from "./column-builders"; export type { ColumnReferenceThunk } from "./column-builders"; +export { + contentEngineDiagnostics, + contentScheduleHealth, + contentSearchDrift, +} from "./diagnostics"; +export type { + ContentEngineDiagnostics, + ContentScheduleHealth, + ContentSearchDrift, + ContentSearchDriftLocale, + ContentTypeDiagnostic, +} from "./diagnostics"; export { contentEditorialEffects } from "./editorial-effects"; export type { ContentEditorialEffectsOptions, ContentEditorialEffectsResult, } from "./editorial-effects"; +export { + CONTENT_EFFECTS_LOG_PREFIX, + reportContentEventFailures, +} from "./effects-log"; export { createContentEditorialService } from "./editorial-service"; export type { ContentEditorialOptions, diff --git a/packages/vitnode/src/content/server/translation-effects.ts b/packages/vitnode/src/content/server/translation-effects.ts index 748809bca..cf1df6851 100644 --- a/packages/vitnode/src/content/server/translation-effects.ts +++ b/packages/vitnode/src/content/server/translation-effects.ts @@ -7,6 +7,7 @@ import type { AnyContentModel } from "./model"; import type { ContentSearchSyncOutcome } from "./search-sync"; import type { ContentTranslationEditorialOutcome } from "./translation-editorial-service"; +import { reportContentEventFailures } from "./effects-log"; import { emitContentEvent } from "./emit"; import { contentSearchAdvancedValues, @@ -131,6 +132,18 @@ export const contentTranslationEffects = async ( { pluginId }, ); + // Same rule as the base effects: the transaction is closed, so a listener that + // never heard about this translation cannot fail the request - but it must not + // vanish either. The locale travels with it, because "nobody heard about the + // Polish copy" is a different incident from "nobody heard about the record". + await reportContentEventFailures(c, { + action: EVENT_ACTION[outcome.operation], + contentTypeId: definition.id, + event, + itemId: outcome.row.itemId, + locale: outcome.locale, + }); + if (!definition.search.enabled || !model) return { event }; // The base row, because a translation's document is built from both halves and From 9cf207289624e74ca82798a242b7706c7bbab29e Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:20:39 +0200 Subject: [PATCH 02/14] feat(content): add effect-failure reporting, drift diagnostics and audit matrices **Operational visibility.** `EventsModel.emit` reports rather than throws, and the interactive routes discarded the result entirely - so a dead listener on a clicked publish left no trace anywhere. Failures now go to `core_logs` behind `[content-effects]` with the content type, the item and the failing listener. The write has committed either way, so the response stays a success. Delivery is at-least-once where a retry is involved and best-effort otherwise; there is no outbox and no exactly-once claim. **Index drift.** `/search/status` reports what is *in* the index, measured against the indexer's own count, so a collection could read as fully covered while missing every Polish document. `contentSearchDrift` compares published rows - or published translations, per locale - against `core_search_index` using the same `publishedCondition` the indexer uses: two aggregates per content type, whatever its size. `contentEngineDiagnostics` folds that together with outstanding scheduled-effect failures behind `GET /admin/debug/content/status`. **Audit matrices**, each done over the whole generated surface rather than one endpoint at a time: route permissions (enumerated from the builder, so a new endpoint cannot join silently), public-projection key sets, structured error contracts with no SQLSTATE or constraint name in any body, and OpenAPI parity validated against the emitted document rather than the Zod object it came from. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/content/next/cache-privacy.test.ts | 188 ++++ .../vitnode/src/content/server/diagnostics.ts | 16 +- .../content/server/editorial-effects.test.ts | 118 ++- .../content/server/error-contracts.test.ts | 481 +++++++++++ packages/vitnode/src/content/server/index.ts | 8 +- .../src/content/server/openapi-parity.test.ts | 813 ++++++++++++++++++ .../content/server/permission-matrix.test.ts | 405 +++++++++ .../src/content/server/public-privacy.test.ts | 263 ++++++ .../vitnode/src/tests/openapi-validate.ts | 185 ++++ 9 files changed, 2464 insertions(+), 13 deletions(-) create mode 100644 packages/vitnode/src/content/next/cache-privacy.test.ts create mode 100644 packages/vitnode/src/content/server/error-contracts.test.ts create mode 100644 packages/vitnode/src/content/server/openapi-parity.test.ts create mode 100644 packages/vitnode/src/content/server/permission-matrix.test.ts create mode 100644 packages/vitnode/src/content/server/public-privacy.test.ts create mode 100644 packages/vitnode/src/tests/openapi-validate.ts diff --git a/packages/vitnode/src/content/next/cache-privacy.test.ts b/packages/vitnode/src/content/next/cache-privacy.test.ts new file mode 100644 index 000000000..d46ff8904 --- /dev/null +++ b/packages/vitnode/src/content/next/cache-privacy.test.ts @@ -0,0 +1,188 @@ +// @vitest-environment node +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { + testEditorialPostContentType, + testLocalizedPageContentType, +} from "@/tests/content-fixtures"; + +/** + * Which responses are allowed into a shared cache, and under which key. + * + * Three questions, and getting any of them wrong is a data leak rather than a + * performance problem: + * + * 1. **Is anything private cached at all?** A preview is an unpublished record + * behind a short-lived credential, and an AdminCP read is a staff response + * carrying private columns. Neither may be stored. + * 2. **Can a private response land on a public key?** Only tagged, cached + * responses can, so this reduces to (1) - but it is asserted from the tag + * side as well, because "no tags" is what makes it true. + * 3. **Do two spellings of one locale share a key?** `PL`, `pl` and ` pl ` + * address the same page, so they have to expire together. A tag is a string + * comparison, so the normalisation has to happen when the tag is built. + */ + +interface FetchArgs { + module: string; + options?: { cache?: string; next?: { tags?: string[] } }; + path: string; + prefixPath?: string; + query?: Record; +} + +const calls = vi.hoisted(() => [] as FetchArgs[]); + +vi.mock("server-only", () => ({})); +vi.mock("next/headers", () => ({ + cookies: async () => await Promise.resolve({ toString: () => "session=x" }), + headers: async () => await Promise.resolve(new Headers()), +})); + +vi.mock("../../lib/fetcher/raw", () => ({ + rawApiFetch: async (args: FetchArgs) => { + calls.push(args); + + return await Promise.resolve( + new Response(JSON.stringify({ id: 7, title: "Hello" }), { status: 200 }), + ); + }, +})); + +const { contentPublicItemTag, contentPublicListTag, contentPublicSlugTag } = + await import("../cache"); +const { contentPreviewFetch, contentPublicFetch } = + await import("./fetch.server"); +const { contentApiFetch } = await import("../admin/fetch.server"); + +const PLUGIN_ID = "@vitnode/example"; + +const lastCall = (): FetchArgs => { + const call = calls.at(-1); + if (!call) throw new Error("Expected a request."); + + return call; +}; + +beforeEach(() => { + calls.length = 0; +}); + +describe("a private response is never cached and never tagged", () => { + it("reads a preview with no store and no tags", async () => { + await contentPreviewFetch({ + definition: testEditorialPostContentType, + pluginId: PLUGIN_ID, + token: "signed-token", + }); + + const call = lastCall(); + expect(call.options?.cache).toBe("no-store"); + expect(call.options?.next?.tags).toBeUndefined(); + }); + + it("reads a localized preview the same way", async () => { + await contentPreviewFetch({ + definition: testLocalizedPageContentType, + locale: "pl", + pluginId: PLUGIN_ID, + token: "signed-token", + }); + + const call = lastCall(); + expect(call.options?.cache).toBe("no-store"); + expect(call.options?.next?.tags).toBeUndefined(); + }); + + it("reads the AdminCP API with no cache options at all", async () => { + // An admin response carries every private column the content type has. It + // is also per-session: it forwards the staff cookie, so a cached one would + // be one editor's view served to the next. + await contentApiFetch({ + definition: testEditorialPostContentType, + method: "get", + path: "/7", + pluginId: PLUGIN_ID, + }); + + const call = lastCall(); + expect(call.options).toBeUndefined(); + expect(call.prefixPath).toBe("/admin"); + }); + + it("keeps the admin module out of the public tag namespace", async () => { + // Belt and braces: even if an admin read were cached one day, it addresses + // `content/{permissionModule}` under `/admin`, and the public tags are + // built from `publicApi.path`. The two cannot be confused for each other. + await contentApiFetch({ + definition: testEditorialPostContentType, + method: "get", + pluginId: PLUGIN_ID, + }); + const adminCall = lastCall(); + + await contentPublicFetch({ + definition: testEditorialPostContentType, + pluginId: PLUGIN_ID, + }); + const publicCall = lastCall(); + + expect(adminCall.module).not.toBe(publicCall.module); + expect(publicCall.options?.next?.tags).toContain( + contentPublicListTag(testEditorialPostContentType.id), + ); + }); +}); + +describe("a public response is cached under exactly one key per locale", () => { + it.each([["pl"], ["PL"], [" pl "], ["pL"]])( + "normalises %s onto the same tags", + locale => { + const id = testLocalizedPageContentType.id; + + expect(contentPublicListTag(id, locale)).toBe( + contentPublicListTag(id, "pl"), + ); + expect(contentPublicItemTag(id, 7, locale)).toBe( + contentPublicItemTag(id, 7, "pl"), + ); + expect(contentPublicSlugTag(id, "witaj", locale)).toBe( + contentPublicSlugTag(id, "witaj", "pl"), + ); + }, + ); + + it("sends the same tags however the caller spelled the locale", async () => { + const tagsFor = async (locale: string) => { + await contentPublicFetch({ + definition: testLocalizedPageContentType, + locale, + pluginId: PLUGIN_ID, + slug: "witaj", + }); + + return lastCall().options?.next?.tags; + }; + + expect(await tagsFor("PL")).toEqual(await tagsFor("pl")); + }); + + it("keeps two languages on one slug apart", () => { + // The whole reason the locale is in the slug tag: `/en/about` and + // `/pl/about` are two pages, and expiring one must not expire the other. + const id = testLocalizedPageContentType.id; + + expect(contentPublicSlugTag(id, "about", "en")).not.toBe( + contentPublicSlugTag(id, "about", "pl"), + ); + }); + + it("keeps a localized tag out of the locale-less namespace", () => { + // A localized content type has no locale-less public URL, so a tag without + // a locale segment would name a page that does not exist - and one that a + // non-localized content type might legitimately own. + const id = testLocalizedPageContentType.id; + + expect(contentPublicListTag(id, "en")).not.toBe(contentPublicListTag(id)); + }); +}); diff --git a/packages/vitnode/src/content/server/diagnostics.ts b/packages/vitnode/src/content/server/diagnostics.ts index e5a5372d8..3667d5ce7 100644 --- a/packages/vitnode/src/content/server/diagnostics.ts +++ b/packages/vitnode/src/content/server/diagnostics.ts @@ -40,6 +40,8 @@ import { /** One locale's share of a content type's index, expected against actual. */ export interface ContentSearchDriftLocale { + /** Published rows (or published translations) the database holds. */ + expected: number; /** * `true` when the two counts agree. * @@ -59,8 +61,6 @@ export interface ContentSearchDriftLocale { * the row really holds. */ locale: string; - /** Published rows (or published translations) the database holds. */ - expected: number; } export interface ContentSearchDrift { @@ -140,7 +140,7 @@ const expectedDocuments = async ( model: RegisteredContentModel["model"], ): Promise> => { const { definition } = model; - const columns = model.columns as Record; + const columns = model.columns; if (!definition.publication.enabled) return new Map(); @@ -241,9 +241,9 @@ export const contentScheduleHealth = async ( for (const row of rows) { result.set(row.contentTypeId, { - failedEffects: Number(row.failedEffects), - pending: Number(row.pending), - withErrors: Number(row.withErrors), + failedEffects: row.failedEffects, + pending: row.pending, + withErrors: row.withErrors, }); } @@ -261,10 +261,10 @@ export interface ContentTypeDiagnostic { search: boolean; }; pluginId: string; - /** `null` for a content type without `search`, which indexes nothing. */ - search: ContentSearchDrift | null; /** `null` for a content type without scheduling, which books nothing. */ schedules: ContentScheduleHealth | null; + /** `null` for a content type without `search`, which indexes nothing. */ + search: ContentSearchDrift | null; } export interface ContentEngineDiagnostics { diff --git a/packages/vitnode/src/content/server/editorial-effects.test.ts b/packages/vitnode/src/content/server/editorial-effects.test.ts index f1c22dd3a..797d5d51f 100644 --- a/packages/vitnode/src/content/server/editorial-effects.test.ts +++ b/packages/vitnode/src/content/server/editorial-effects.test.ts @@ -47,18 +47,44 @@ const harness = ({ failures: [], status: "delivered", }), -}: { contextPlugin?: string; emit?: ReturnType } = {}) => { + log = vi.fn().mockResolvedValue(undefined), +}: { + contextPlugin?: string; + emit?: ReturnType; + log?: ReturnType; +} = {}) => { const store: Record = { events: { emit }, + // The effects layer writes post-commit failures here. Present on the + // harness because a missing logger is itself a tested fallback, not the + // shape a real request has. + log: { error: log }, plugin: { id: contextPlugin }, }; return { c: { get: (key: string) => store[key] } as unknown as Context, emit, + log, }; }; +/** An emit result with one dead listener on it. */ +const withFailure = () => + vi.fn().mockResolvedValue({ + delivered: 0, + eventId: "event-1", + failures: [ + { + error: "Service unavailable", + listener: "send-notification", + module: "notifications", + pluginId: OWNER, + }, + ], + status: "delivered", + }); + beforeEach(() => { vi.clearAllMocks(); syncContentSearch.mockResolvedValue({ @@ -200,4 +226,94 @@ describe("contentEditorialEffects", () => { }); }); }); + + /** + * A dead listener on an *interactive* mutation has nowhere else to be + * recorded: the scheduled path writes it onto the schedule row and retries, + * and a clicked publish does neither. Without a log line it is invisible. + */ + describe("reporting a delivery failure", () => { + it("logs the failed listener with the record it belongs to", async () => { + const { c, log } = harness({ emit: withFailure() }); + + await contentEditorialEffects( + c, + testEditorialPostContentType, + outcome(), + { pluginId: OWNER }, + ); + + expect(log).toHaveBeenCalledTimes(1); + const message = String(log.mock.calls[0][0]); + expect(message).toContain("[content-effects]"); + expect(message).toContain(testEditorialPostContentType.id); + expect(message).toContain('"itemId":7'); + expect(message).toContain("send-notification"); + expect(message).toContain("Service unavailable"); + }); + + it("names the action, so a failed publish is not read as a failed edit", async () => { + const { c, log } = harness({ emit: withFailure() }); + + await contentEditorialEffects( + c, + testEditorialPostContentType, + outcome({ operation: "unpublish" }), + { pluginId: OWNER }, + ); + + expect(String(log.mock.calls[0][0])).toContain('"action":"unpublished"'); + }); + + it("logs nothing when every listener received it", async () => { + // An expected success is not an error, and a log full of them is a log + // nobody reads. + const { c, log } = harness(); + + await contentEditorialEffects( + c, + testEditorialPostContentType, + outcome(), + { pluginId: OWNER }, + ); + + expect(log).not.toHaveBeenCalled(); + }); + + it("does not fail the mutation when the logger itself is down", async () => { + // The logger writes to the database, so it can fail for the same reason + // the transport did - and the write has already committed either way. + const { c } = harness({ + emit: withFailure(), + log: vi.fn().mockRejectedValue(new Error("core_logs unreachable")), + }); + const console_ = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + + const result = await contentEditorialEffects( + c, + testEditorialPostContentType, + outcome(), + { pluginId: OWNER }, + ); + + expect(result.event?.failures).toHaveLength(1); + expect(console_).toHaveBeenCalled(); + console_.mockRestore(); + }); + + it("still writes the search document after reporting the failure", async () => { + const { c } = harness({ emit: withFailure() }); + + await contentEditorialEffects( + c, + testEditorialPostContentType, + outcome(), + { pluginId: OWNER }, + ); + + expect(syncContentSearch).toHaveBeenCalledTimes(1); + }); + }); }); diff --git a/packages/vitnode/src/content/server/error-contracts.test.ts b/packages/vitnode/src/content/server/error-contracts.test.ts new file mode 100644 index 000000000..da2d3da91 --- /dev/null +++ b/packages/vitnode/src/content/server/error-contracts.test.ts @@ -0,0 +1,481 @@ +// @vitest-environment node +import { HTTPException } from "hono/http-exception"; +import { describe, expect, it } from "vitest"; +import { ZodError } from "zod"; + +import { + ContentAdvancedInputError, + ContentDefaultTranslationRequired, + ContentInputError, + ContentLanguageError, + ContentRevisionNotRestorable, + ContentScheduleError, + ContentTranslationExists, + ContentTranslationItemMissing, + ContentTranslationVersionConflict, + ContentVersionConflict, +} from "../errors"; +import { withHttpErrors } from "./http-errors"; +import { withTranslationHttpErrors } from "./translation-http-errors"; + +/** + * What a client is allowed to learn when a write fails. + * + * Two rules, and the second is the one that needs a test rather than a comment: + * + * 1. **Every expected failure has a stable contract** - a status, and for the + * ones a client has to branch on, a `code`. A caller cannot be asked to parse + * English, and it certainly cannot be asked to parse a SQLSTATE. + * 2. **Nothing internal crosses the boundary.** A driver error carries the + * constraint name, often the column, and sometimes the value that clashed. + * None of that may reach a response body - it is a schema description handed + * to whoever asked, and on a public route it is handed to anyone. + */ + +const CONTENT_TYPE_ID = "test.article"; + +/** The whole response, as a client would see it. */ +const responseOf = async ( + run: () => Promise, + options: Parameters[2] = {}, +): Promise<{ body: string; status: number }> => { + try { + await withHttpErrors("update", run, { + contentTypeId: CONTENT_TYPE_ID, + ...options, + }); + } catch (error) { + if (!(error instanceof HTTPException)) throw error; + + const res = error.getResponse(); + + return { body: await res.text(), status: res.status }; + } + + throw new Error("Expected the write to fail."); +}; + +const translationResponseOf = async ( + run: () => Promise, + action: "create" | "delete" | "read" | "update" = "update", +): Promise<{ body: string; status: number }> => { + try { + await withTranslationHttpErrors(action, run, { + contentTypeId: CONTENT_TYPE_ID, + itemId: 7, + locale: "pl", + }); + } catch (error) { + if (!(error instanceof HTTPException)) throw error; + + const res = error.getResponse(); + + return { body: await res.text(), status: res.status }; + } + + throw new Error("Expected the translation write to fail."); +}; + +/** A driver failure, in the shape Drizzle actually wraps one in. */ +const driverError = (code: string, detail: string) => + Object.assign(new Error("Failed query: insert into ..."), { + cause: Object.assign(new Error(detail), { + code, + constraint_name: "example_articles_code_key", + detail, + schema_name: "public", + table_name: "example_articles", + }), + }); + +const throwing = (error: unknown) => async () => { + await Promise.resolve(); + throw error; +}; + +describe("expected database failures map onto stable contracts", () => { + it.each([ + ["23505", 409, "unique violation"], + ["23503", 400, "foreign key violation on a write"], + ["23502", 400, "not-null violation"], + ["23001", 409, "restrict violation"], + ])("turns %s into %i (%s)", async (code, status) => { + const result = await responseOf( + throwing(driverError(code, "Key (code)=(guide-001) already exists.")), + ); + + expect(result.status).toBe(status); + }); + + it("reads the SQLSTATE through the wrapper Drizzle puts around it", async () => { + // `DrizzleQueryError.code` is undefined and the real error is on `cause`, so + // a mapper reading `error.code` alone would turn every constraint failure + // into a 500. + const bare = Object.assign(new Error("duplicate"), { code: "23505" }); + + await expect(responseOf(throwing(bare))).resolves.toMatchObject({ + status: 409, + }); + }); + + it("answers a delete blocked by a reference with 409, not 400", async () => { + // The same SQLSTATE means different things by verb: on a create it is "the + // thing you pointed at is gone", on a delete it is "something still points + // at this". + try { + await withHttpErrors( + "delete", + throwing(driverError("23503", "still referenced")), + { contentTypeId: CONTENT_TYPE_ID }, + ); + } catch (error) { + expect((error as HTTPException).status).toBe(409); + } + }); + + /** + * Postgres 18 reports an explicit `ON DELETE RESTRICT` as `23001` + * (restrict_violation) where earlier majors reported `23503`. Both have to map + * to the same 409, or upgrading the database would change an API contract. + */ + it("answers the same way on both Postgres codes for a blocked delete", async () => { + const statuses = await Promise.all( + ["23001", "23503"].map(async code => { + try { + await withHttpErrors( + "delete", + throwing(driverError(code, "still referenced")), + { contentTypeId: CONTENT_TYPE_ID }, + ); + } catch (error) { + return (error as HTTPException).status; + } + + return 0; + }), + ); + + expect(statuses).toEqual([409, 409]); + }); + + it("rethrows an unrecognised failure for the global handler", async () => { + // A 500 with nothing in it beats a guessed status: `app.onError` logs the + // detail and answers "Internal Server Error" in production. + const unknown = Object.assign(new Error("connection terminated"), { + code: "57P01", + }); + + await expect( + withHttpErrors("update", throwing(unknown), { + contentTypeId: CONTENT_TYPE_ID, + }), + ).rejects.toBe(unknown); + }); +}); + +describe("domain failures map onto their documented codes", () => { + it("answers a stale write with a structured version conflict", async () => { + const result = await responseOf( + throwing( + new ContentVersionConflict({ + contentTypeId: CONTENT_TYPE_ID, + currentVersion: 6, + expectedVersion: 4, + itemId: 7, + }), + ), + { structured: true }, + ); + + expect(result.status).toBe(409); + expect(JSON.parse(result.body)).toEqual({ + code: "CONTENT_VERSION_CONFLICT", + contentTypeId: CONTENT_TYPE_ID, + currentVersion: 6, + expectedVersion: 4, + itemId: 7, + }); + }); + + it("answers a unique clash with a structured conflict on an editorial route", async () => { + const result = await responseOf( + throwing(driverError("23505", "Key (code)=(guide-001) already exists.")), + { itemId: 7, structured: true }, + ); + + expect(JSON.parse(result.body)).toEqual({ + code: "CONTENT_UNIQUE_CONFLICT", + contentTypeId: CONTENT_TYPE_ID, + itemId: 7, + }); + }); + + it("answers an unrestorable revision with 422 and the field names", async () => { + const result = await responseOf( + throwing( + new ContentRevisionNotRestorable({ + contentTypeId: CONTENT_TYPE_ID, + fields: ["category"], + revisionId: 12, + }), + ), + { structured: true }, + ); + + expect(result.status).toBe(422); + expect(JSON.parse(result.body)).toEqual({ + code: "CONTENT_REVISION_NOT_RESTORABLE", + contentTypeId: CONTENT_TYPE_ID, + fields: ["category"], + revisionId: 12, + }); + }); + + it("answers a refused schedule with 400 and a code", async () => { + const result = await responseOf( + throwing( + new ContentScheduleError("That time has already passed.", { + code: "CONTENT_SCHEDULE_IN_PAST", + contentTypeId: CONTENT_TYPE_ID, + }), + ), + ); + + expect(result.status).toBe(400); + expect(JSON.parse(result.body)).toEqual({ + code: "CONTENT_SCHEDULE_IN_PAST", + contentTypeId: CONTENT_TYPE_ID, + }); + }); + + it("answers a missing relation target with 400 and the ids the caller sent", async () => { + const result = await responseOf( + throwing( + new ContentAdvancedInputError({ + code: "CONTENT_RELATION_MISSING_TARGET", + contentTypeId: CONTENT_TYPE_ID, + field: "categories", + ids: [99], + message: + 'Relation "categories" references a record that no longer exists: 99.', + }), + ), + ); + + expect(result.status).toBe(400); + // The caller's own input echoed back - nothing internal in it. + expect(result.body).toContain("categories"); + expect(result.body).toContain("99"); + }); + + it("answers a repeatable child that belongs elsewhere with 400", async () => { + const result = await responseOf( + throwing( + new ContentAdvancedInputError({ + code: "CONTENT_REPEATABLE_UNKNOWN_CHILD", + contentTypeId: CONTENT_TYPE_ID, + field: "faq", + ids: [5], + message: + 'Repeatable "faq" was sent an entry that does not belong to this record: 5.', + }), + ), + ); + + expect(result.status).toBe(400); + }); + + it("answers invalid input with 400 and no issue tree", async () => { + const result = await responseOf( + throwing( + new ZodError([ + { + code: "too_small", + minimum: 3, + origin: "string", + path: ["title"], + message: "Too small", + }, + ]), + ), + ); + + expect(result.status).toBe(400); + expect(result.body).toBe("Invalid input data."); + expect(result.body).not.toContain("title"); + }); + + it("keeps a written-for-the-client input error readable", async () => { + const result = await responseOf( + throwing( + new ContentInputError('The slug for "title" normalises to nothing.', { + contentTypeId: CONTENT_TYPE_ID, + }), + ), + ); + + expect(result.status).toBe(400); + expect(result.body).toContain("normalises to nothing"); + }); +}); + +describe("translation failures keep their own union", () => { + it.each([ + [ + "a stale locale write", + new ContentTranslationVersionConflict({ + contentTypeId: CONTENT_TYPE_ID, + currentVersion: 5, + expectedVersion: 2, + itemId: 7, + locale: "pl", + }), + 409, + "CONTENT_TRANSLATION_VERSION_CONFLICT", + ], + [ + "deleting the default translation", + new ContentDefaultTranslationRequired({ + contentTypeId: CONTENT_TYPE_ID, + itemId: 7, + locale: "en", + }), + 409, + "CONTENT_DEFAULT_TRANSLATION_REQUIRED", + ], + [ + "a second translation in one locale", + new ContentTranslationExists({ + contentTypeId: CONTENT_TYPE_ID, + itemId: 7, + locale: "pl", + }), + 409, + "CONTENT_TRANSLATION_EXISTS", + ], + [ + "a locale this install switched off", + new ContentLanguageError({ + contentTypeId: CONTENT_TYPE_ID, + locale: "de", + reason: "disabled", + }), + 409, + "CONTENT_LANGUAGE_DISABLED", + ], + ])("answers %s with %i and a code", async (_why, error, status, code) => { + const result = await translationResponseOf(throwing(error)); + + expect(result.status).toBe(status); + expect(JSON.parse(result.body)).toMatchObject({ code }); + }); + + it("answers an unknown locale with 404 rather than a conflict", async () => { + // "There is no such language" and "this install switched it off" want + // different answers: only the second is something an admin can undo. + const result = await translationResponseOf( + throwing( + new ContentLanguageError({ + contentTypeId: CONTENT_TYPE_ID, + locale: "zz", + reason: "missing", + }), + ), + ); + + expect(result.status).toBe(404); + }); + + it("answers a translation of a record that is gone with 404", async () => { + const result = await translationResponseOf( + throwing( + new ContentTranslationItemMissing({ + contentTypeId: CONTENT_TYPE_ID, + itemId: 7, + }), + ), + ); + + expect(result.status).toBe(404); + }); + + it("turns a localized unique clash into the translation union, with the locale", async () => { + const result = await translationResponseOf( + throwing( + driverError("23505", "Key (languageId, slug)=(2, hello) exists"), + ), + ); + + expect(result.status).toBe(409); + expect(JSON.parse(result.body)).toEqual({ + code: "CONTENT_TRANSLATION_UNIQUE_CONFLICT", + contentTypeId: CONTENT_TYPE_ID, + itemId: 7, + locale: "pl", + }); + }); +}); + +/** + * The regression that matters most, because its symptom is invisible: a response + * body that happens to contain the constraint name reads fine to a human and + * hands an attacker the schema. + */ +describe("no internal detail crosses the boundary", () => { + const LEAKS = [ + "example_articles_code_key", + "example_articles", + "public", + "23505", + "23503", + "insert into", + "Key (code)=(guide-001)", + ]; + + it.each(["23505", "23503", "23502", "23001"])( + "keeps the driver's detail out of the %s response", + async code => { + const result = await responseOf( + throwing(driverError(code, "Key (code)=(guide-001) already exists.")), + ); + + for (const leak of LEAKS) { + expect(result.body.toLowerCase()).not.toContain(leak.toLowerCase()); + } + }, + ); + + it("keeps it out of the structured editorial body too", async () => { + const result = await responseOf( + throwing(driverError("23505", "Key (code)=(guide-001) already exists.")), + { itemId: 7, structured: true }, + ); + + for (const leak of LEAKS) { + expect(result.body.toLowerCase()).not.toContain(leak.toLowerCase()); + } + }); + + it("keeps it out of the translation body", async () => { + const result = await translationResponseOf( + throwing(driverError("23505", "Key (slug)=(hello) already exists.")), + ); + + for (const leak of LEAKS) { + expect(result.body.toLowerCase()).not.toContain(leak.toLowerCase()); + } + }); + + it("never serialises an Error object into a body", async () => { + // A body of `{}` is what `JSON.stringify(new Error(...))` produces, and a + // body of `{"message":...,"stack":...}` is what a helpful serializer + // produces. Neither is a contract. + const result = await responseOf(throwing(driverError("23505", "boom")), { + itemId: 7, + structured: true, + }); + + expect(result.body).not.toContain("stack"); + expect(JSON.parse(result.body)).not.toHaveProperty("message"); + }); +}); diff --git a/packages/vitnode/src/content/server/index.ts b/packages/vitnode/src/content/server/index.ts index 298276c57..c83cfa129 100644 --- a/packages/vitnode/src/content/server/index.ts +++ b/packages/vitnode/src/content/server/index.ts @@ -33,10 +33,6 @@ export type { ContentEditorialEffectsOptions, ContentEditorialEffectsResult, } from "./editorial-effects"; -export { - CONTENT_EFFECTS_LOG_PREFIX, - reportContentEventFailures, -} from "./effects-log"; export { createContentEditorialService } from "./editorial-service"; export type { ContentEditorialOptions, @@ -45,6 +41,10 @@ export type { ContentEditorialService, ContentEditorialWriteOptions, } from "./editorial-service"; +export { + CONTENT_EFFECTS_LOG_PREFIX, + reportContentEventFailures, +} from "./effects-log"; export { emitContentEvent } from "./emit"; export { contentConflict, diff --git a/packages/vitnode/src/content/server/openapi-parity.test.ts b/packages/vitnode/src/content/server/openapi-parity.test.ts new file mode 100644 index 000000000..b57d0bde7 --- /dev/null +++ b/packages/vitnode/src/content/server/openapi-parity.test.ts @@ -0,0 +1,813 @@ +// @vitest-environment node +import type { RouteConfig } from "@hono/zod-openapi"; +import type { MiddlewareHandler } from "hono"; + +import { OpenAPIHono } from "@hono/zod-openapi"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { JsonSchemaLike } from "@/tests/openapi-validate"; + +import { + testEditorialPostContentType, + testLocalizedPageContentType, +} from "@/tests/content-fixtures"; +import { validateAgainstJsonSchema } from "@/tests/openapi-validate"; + +import { + ContentDefaultTranslationRequired, + ContentRevisionNotRestorable, + ContentScheduleError, + ContentTranslationVersionConflict, + ContentVersionConflict, +} from "../errors"; +import { createContentModel } from "./model"; +import { buildContentPublicRoutes } from "./public-routes"; +import { buildContentRoutes } from "./routes"; + +/** + * The document says one thing; the runtime does another. + * + * Every generated route declares its responses in OpenAPI, and a generated + * client is built from exactly that. These tests serve the document the app + * really publishes and check the body the handler really produced against it - + * so a `409` that answers with prose where the document promises a + * discriminated union fails here rather than in somebody's generated client. + * + * Two halves, and both matter: + * + * 1. **the status is declared** - a runtime `409` on a route whose document + * lists only `200` and `404` is a contract break even when the body is fine; + * 2. **the body validates** - against the emitted JSON Schema rather than + * against the Zod object it came from. The two are not interchangeable: + * `z.date()` renders as `{ type: "string", format: "date-time" }`, which is + * exactly what `c.json(row)` sends and exactly what the Zod object rejects. + */ + +vi.mock("../../api/lib/check-staff-permission", () => ({ + assertStaffPermission: async () => await Promise.resolve(), +})); + +const posts = createContentModel(testEditorialPostContentType); +const localized = createContentModel(testLocalizedPageContentType); +const PLUGIN_ID = "@vitnode/example"; + +const adminUser = { + avatarColor: "000000", + birthday: null, + createdAt: new Date(), + email: "test@test.com", + emailVerified: true, + id: 1, + language: "en", + name: "Test", + nameCode: "test", + newsletter: false, + roleId: 1, +}; + +const row = { + createdAt: new Date("2026-01-01T00:00:00.000Z"), + excerpt: null, + id: 7, + publishedAt: null, + slug: "hello-world", + status: "draft" as const, + title: "Hello world", + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + version: 4, + views: 0, +}; + +const revision = { + actorName: null, + actorType: "staff" as const, + actorUserId: null, + changedFields: ["title"], + createdAt: new Date("2026-01-01T00:00:00.000Z"), + id: 3, + operation: "update" as const, + restoredFromRevisionId: null, + version: 4, +}; + +const outcome = (overrides: Record = {}) => ({ + changed: true, + changedFields: ["title"], + operation: "update" as const, + previousSlug: null, + restoredFromRevisionId: null, + revisionId: 3, + row, + version: 5, + ...overrides, +}); + +const declaredStatuses = (route: RouteConfig): number[] => + Object.keys(route.responses).map(Number); + +interface Suite { + app: OpenAPIHono; + /** The served OpenAPI document, which is what a generated client is built from. */ + document: JsonSchemaLike; + routeOf: (method: string, path: string) => RouteConfig; +} + +/** + * The response schema the **document** publishes for one status. + * + * Not the Zod object the route was built from: `z.date()` renders as + * `{ type: "string", format: "date-time" }`, which is what the handler really + * sends, while the Zod object rejects that string outright. Reading the emitted + * document is the only way to check the contract a client actually consumes. + */ +const documentedSchema = ( + suite: Suite, + route: RouteConfig, + status: number, +): JsonSchemaLike | undefined => { + const paths = suite.document.paths as Record< + string, + Record }> + >; + const operation = paths?.[route.path]?.[route.method.toLowerCase()]; + const response = operation?.responses?.[String(status)]; + const content = response?.content as + Record | undefined; + + return content?.["application/json"]?.schema; +}; + +const mount = ( + built: { handler: unknown; route: RouteConfig }[], + events = { + emit: async () => await Promise.resolve({ delivered: 0, failures: [] }), + }, +): Suite => { + const app = new OpenAPIHono(); + const context: MiddlewareHandler = async (c, next) => { + c.set("admin", { user: adminUser }); + c.set("events", events as never); + c.set("log", { error: async () => await Promise.resolve() } as never); + await next(); + }; + app.use("*", context); + for (const { handler, route } of built) { + app.openapi(route, handler as never); + } + + return { + app, + document: app.getOpenAPIDocument({ + info: { title: "Content Engine", version: "1" }, + openapi: "3.0.0", + }) as unknown as JsonSchemaLike, + routeOf: (method, path) => { + const found = built.find( + entry => + entry.route.method.toUpperCase() === method.toUpperCase() && + entry.route.path === path, + ); + if (!found) throw new Error(`No route for ${method} ${path}.`); + + return found.route; + }, + }; +}; + +/** + * Drives one request and holds the schema its declared status published. + * + * The assertion is deliberately in one helper: "the status is in the document + * and the body parses against it" is the whole contract, and stating it + * twenty-odd times by hand is twenty-odd chances to state it slightly + * differently. + */ +const expectParity = async ( + suite: Suite, + { + body, + expected, + method, + path, + template, + }: { + body?: unknown; + expected: number; + method: string; + path: string; + /** The OpenAPI path, when it differs from the concrete one. */ + template: string; + }, +): Promise => { + const res = await suite.app.request(path, { + method, + ...(body === undefined + ? {} + : { + body: JSON.stringify(body), + headers: { "Content-Type": "application/json" }, + }), + }); + + expect([`${method} ${path}`, res.status]).toEqual([ + `${method} ${path}`, + expected, + ]); + + const route = suite.routeOf(method, template); + expect([ + `${method} ${template}`, + declaredStatuses(route).includes(expected), + ]).toEqual([`${method} ${template}`, true]); + + const schema = documentedSchema(suite, route, expected); + if (!schema) return undefined; + + const payload: unknown = await res.json(); + + // The failure message has to name the route, or a red suite says only "one + // of the thirty contracts is wrong". + expect([ + `${method} ${template} -> ${expected}`, + validateAgainstJsonSchema(payload, schema, suite.document), + ]).toEqual([`${method} ${template} -> ${expected}`, []]); + + return payload; +}; + +const adminService = () => ({ + advanced: vi.fn(), + advancedFields: vi.fn(), + create: vi.fn(), + delete: vi.fn(), + findById: vi.fn().mockResolvedValue(row), + findDetail: vi.fn(), + findMany: vi.fn().mockResolvedValue({ + edges: [{ ...row, labels: {} }], + pageInfo: { + count: 1, + endCursor: 7, + hasNextPage: false, + hasPreviousPage: false, + startCursor: 7, + totalCount: 1, + }, + }), + options: vi.fn().mockResolvedValue([]), + relations: {}, + repeatable: {}, + update: vi.fn(), +}); + +const editorialStub = () => ({ + create: vi.fn().mockResolvedValue(outcome({ operation: "create" })), + delete: vi.fn().mockResolvedValue(outcome({ operation: "delete" })), + publish: vi.fn().mockResolvedValue(outcome({ operation: "publish" })), + relations: {}, + repeatable: {}, + restore: vi.fn().mockResolvedValue(outcome({ operation: "restore" })), + revisions: { + findById: vi + .fn() + .mockResolvedValue({ ...revision, snapshot: { title: "x" } }), + latest: vi.fn().mockResolvedValue(revision), + list: vi.fn().mockResolvedValue({ + edges: [revision], + pageInfo: { endCursor: 4, hasNextPage: false }, + }), + }, + schedules: { + cancel: vi.fn().mockResolvedValue({ action: "publish" }), + listForItem: vi.fn().mockResolvedValue([]), + schedule: vi.fn().mockResolvedValue({ + generation: 1, + id: 55, + scheduledFor: new Date("2030-01-01T00:00:00.000Z"), + }), + }, + unpublish: vi.fn().mockResolvedValue(outcome({ operation: "unpublish" })), + update: vi.fn().mockResolvedValue(outcome()), +}); + +let editorial: ReturnType; +let service: ReturnType; + +const editorialSuite = (): Suite => { + service = adminService(); + editorial = editorialStub(); + vi.spyOn(posts, "service").mockReturnValue(service as never); + vi.spyOn( + posts as unknown as { editorialService: unknown }, + "editorialService", + "get", + ).mockReturnValue(() => editorial); + + return mount(buildContentRoutes(posts, { pluginId: PLUGIN_ID })); +}; + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("admin routes match their OpenAPI document", () => { + it("lists", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "GET", + path: "/", + template: "/", + }); + }); + + it("reads one record", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "GET", + path: "/7", + template: "/{id}", + }); + }); + + it("answers 404 for a record that is not there", async () => { + const suite = editorialSuite(); + service.findById.mockResolvedValue(null); + + await expectParity(suite, { + expected: 404, + method: "GET", + path: "/7", + template: "/{id}", + }); + }); + + it("creates", async () => { + await expectParity(editorialSuite(), { + body: { title: "Hello world" }, + expected: 201, + method: "POST", + path: "/", + template: "/", + }); + }); + + it("rejects an invalid create with the declared 400", async () => { + await expectParity(editorialSuite(), { + body: { title: "no" }, + expected: 400, + method: "POST", + path: "/", + template: "/", + }); + }); + + it("updates", async () => { + await expectParity(editorialSuite(), { + body: { expectedVersion: 4, values: { title: "Hello again" } }, + expected: 200, + method: "PUT", + path: "/7", + template: "/{id}", + }); + }); + + it("answers a stale update with the documented 409 union", async () => { + const suite = editorialSuite(); + editorial.update.mockRejectedValue( + new ContentVersionConflict({ + contentTypeId: testEditorialPostContentType.id, + currentVersion: 6, + expectedVersion: 4, + itemId: 7, + }), + ); + + const body = await expectParity(suite, { + body: { expectedVersion: 4, values: { title: "Hello again" } }, + expected: 409, + method: "PUT", + path: "/7", + template: "/{id}", + }); + + // The discriminant, spelled out: a client branches on it, so a schema that + // merely admits a string would be a weaker contract than it looks. + expect(body).toMatchObject({ code: "CONTENT_VERSION_CONFLICT" }); + }); + + it("answers a unique clash with the same 409 union", async () => { + const suite = editorialSuite(); + editorial.update.mockRejectedValue( + Object.assign(new Error("duplicate"), { code: "23505" }), + ); + + const body = await expectParity(suite, { + body: { expectedVersion: 4, values: { title: "Hello again" } }, + expected: 409, + method: "PUT", + path: "/7", + template: "/{id}", + }); + + expect(body).toMatchObject({ code: "CONTENT_UNIQUE_CONFLICT" }); + }); + + it("publishes", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "POST", + path: "/7/publish", + template: "/{id}/publish", + }); + }); + + it("unpublishes", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "POST", + path: "/7/unpublish", + template: "/{id}/unpublish", + }); + }); + + it("deletes", async () => { + await expectParity(editorialSuite(), { + body: { expectedVersion: 4 }, + expected: 200, + method: "DELETE", + path: "/7", + template: "/{id}", + }); + }); + + it("lists revisions", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "GET", + path: "/7/revisions", + template: "/{id}/revisions", + }); + }); + + it("reads one revision with its snapshot", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "GET", + path: "/7/revisions/3", + template: "/{id}/revisions/{revisionId}", + }); + }); + + it("restores", async () => { + await expectParity(editorialSuite(), { + body: { expectedVersion: 4 }, + expected: 200, + method: "POST", + path: "/7/revisions/3/restore", + template: "/{id}/revisions/{revisionId}/restore", + }); + }); + + it("answers an unrestorable revision with the documented 422", async () => { + const suite = editorialSuite(); + editorial.restore.mockRejectedValue( + new ContentRevisionNotRestorable({ + contentTypeId: testEditorialPostContentType.id, + fields: ["title"], + revisionId: 3, + }), + ); + + const body = await expectParity(suite, { + body: { expectedVersion: 4 }, + expected: 422, + method: "POST", + path: "/7/revisions/3/restore", + template: "/{id}/revisions/{revisionId}/restore", + }); + + expect(body).toMatchObject({ + code: "CONTENT_REVISION_NOT_RESTORABLE", + fields: ["title"], + }); + }); + + it("lists schedules", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "GET", + path: "/7/schedules", + template: "/{id}/schedules", + }); + }); + + it("books a schedule", async () => { + await expectParity(editorialSuite(), { + body: { + action: "publish", + scheduledFor: new Date(Date.now() + 86_400_000).toISOString(), + }, + expected: 200, + method: "POST", + path: "/7/schedule", + template: "/{id}/schedule", + }); + }); + + it("answers a refused schedule with the documented 400 body", async () => { + const suite = editorialSuite(); + editorial.schedules.schedule.mockRejectedValue( + new ContentScheduleError("That time has already passed.", { + code: "CONTENT_SCHEDULE_IN_PAST", + contentTypeId: testEditorialPostContentType.id, + }), + ); + + const body = await expectParity(suite, { + body: { + action: "publish", + scheduledFor: new Date(Date.now() + 86_400_000).toISOString(), + }, + expected: 400, + method: "POST", + path: "/7/schedule", + template: "/{id}/schedule", + }); + + expect(body).toMatchObject({ code: "CONTENT_SCHEDULE_IN_PAST" }); + }); + + it("cancels a schedule", async () => { + await expectParity(editorialSuite(), { + expected: 200, + method: "POST", + path: "/7/schedule/5/cancel", + template: "/{id}/schedule/{scheduleId}/cancel", + }); + }); + + it("mints a preview link", async () => { + vi.stubEnv("CONTENT_PREVIEW_SECRET", "a".repeat(48)); + const suite = editorialSuite(); + + await expectParity(suite, { + expected: 200, + method: "POST", + path: "/7/preview", + template: "/{id}/preview", + }); + vi.unstubAllEnvs(); + }); +}); + +describe("translation routes match their OpenAPI document", () => { + const translationRow = { + createdAt: new Date("2026-01-01T00:00:00.000Z"), + itemId: 7, + languageId: 1, + locale: "en", + publishedAt: null, + status: "draft" as const, + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + values: { body: "Body", slug: "hello", title: "Hello" }, + version: 2, + }; + + const translationOutcome = (overrides: Record = {}) => ({ + changed: true, + changedFields: ["title"], + languageId: 1, + locale: "en", + operation: "update" as const, + previousSlug: null, + restoredFromRevisionId: null, + revisionId: 9, + row: translationRow, + version: 3, + ...overrides, + }); + + let translations: Record>; + let translationEditorial: Record; + + const suite = (): Suite => { + translations = { + exists: vi.fn().mockResolvedValue(true), + findByLanguageId: vi.fn().mockResolvedValue(translationRow), + findByLocale: vi.fn().mockResolvedValue(translationRow), + findManyForItem: vi.fn().mockResolvedValue([ + { + createdAt: new Date("2026-01-01T00:00:00.000Z"), + itemId: 7, + languageId: 1, + locale: "en", + publishedAt: null, + status: "draft", + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + version: 2, + }, + ]), + resolveDefaultLanguage: vi + .fn() + .mockResolvedValue({ id: 1, locale: "en" }), + resolveLanguage: vi.fn().mockResolvedValue({ id: 1, locale: "en" }), + }; + translationEditorial = { + create: vi + .fn() + .mockResolvedValue(translationOutcome({ operation: "create" })), + delete: vi + .fn() + .mockResolvedValue(translationOutcome({ operation: "delete" })), + publish: vi + .fn() + .mockResolvedValue(translationOutcome({ operation: "publish" })), + restore: vi + .fn() + .mockResolvedValue(translationOutcome({ operation: "restore" })), + findRevision: vi + .fn() + .mockResolvedValue({ ...revision, snapshot: { title: "x" } }), + listRevisions: vi.fn().mockResolvedValue({ + edges: [revision], + pageInfo: { endCursor: 2, hasNextPage: false }, + }), + unpublish: vi + .fn() + .mockResolvedValue(translationOutcome({ operation: "unpublish" })), + update: vi.fn().mockResolvedValue(translationOutcome()), + }; + + vi.spyOn( + localized as unknown as { translationService: unknown }, + "translationService", + "get", + ).mockReturnValue(() => translations); + vi.spyOn( + localized as unknown as { translationEditorialService: unknown }, + "translationEditorialService", + "get", + ).mockReturnValue(() => translationEditorial); + vi.spyOn(localized, "service").mockReturnValue(adminService() as never); + + return mount(buildContentRoutes(localized, { pluginId: PLUGIN_ID })); + }; + + it("lists the locales a record exists in", async () => { + await expectParity(suite(), { + expected: 200, + method: "GET", + path: "/7/translations", + template: "/{id}/translations", + }); + }); + + it("reads one translation", async () => { + await expectParity(suite(), { + expected: 200, + method: "GET", + path: "/7/translations/en", + template: "/{id}/translations/{locale}", + }); + }); + + it("creates a translation", async () => { + await expectParity(suite(), { + body: { values: { body: "Cześć", slug: "czesc", title: "Cześć" } }, + expected: 201, + method: "POST", + path: "/7/translations/pl", + template: "/{id}/translations/{locale}", + }); + }); + + it("updates a translation", async () => { + await expectParity(suite(), { + body: { expectedVersion: 2, values: { title: "Hello again" } }, + expected: 200, + method: "PUT", + path: "/7/translations/en", + template: "/{id}/translations/{locale}", + }); + }); + + it("answers a stale translation update with the translation 409 union", async () => { + const built = suite(); + (translationEditorial.update as ReturnType).mockRejectedValue( + new ContentTranslationVersionConflict({ + contentTypeId: testLocalizedPageContentType.id, + currentVersion: 5, + expectedVersion: 2, + itemId: 7, + locale: "en", + }), + ); + + const body = await expectParity(built, { + body: { expectedVersion: 2, values: { title: "Hello again" } }, + expected: 409, + method: "PUT", + path: "/7/translations/en", + template: "/{id}/translations/{locale}", + }); + + expect(body).toMatchObject({ + code: "CONTENT_TRANSLATION_VERSION_CONFLICT", + locale: "en", + }); + }); + + it("answers a default-translation delete with the documented 409", async () => { + const built = suite(); + (translationEditorial.delete as ReturnType).mockRejectedValue( + new ContentDefaultTranslationRequired({ + contentTypeId: testLocalizedPageContentType.id, + itemId: 7, + locale: "en", + }), + ); + + const body = await expectParity(built, { + body: { expectedVersion: 2 }, + expected: 409, + method: "DELETE", + path: "/7/translations/en", + template: "/{id}/translations/{locale}", + }); + + expect(body).toMatchObject({ + code: "CONTENT_DEFAULT_TRANSLATION_REQUIRED", + }); + }); + + it("lists one locale's revisions", async () => { + await expectParity(suite(), { + expected: 200, + method: "GET", + path: "/7/translations/en/revisions", + template: "/{id}/translations/{locale}/revisions", + }); + }); +}); + +describe("public routes match their OpenAPI document", () => { + const publicRow = { + excerpt: null, + publishedAt: new Date("2026-01-01T00:00:00.000Z"), + slug: "hello-world", + title: "Hello world", + }; + + const suite = ( + findBySlug: unknown = publicRow, + findById: unknown = publicRow, + ): Suite => { + vi.spyOn( + posts as unknown as { publicService: unknown }, + "publicService", + "get", + ).mockReturnValue(() => ({ + findById: async () => await Promise.resolve(findById), + findBySlug: async () => await Promise.resolve(findBySlug), + findMany: async () => await Promise.resolve({ + edges: [publicRow], + pageInfo: { + count: 1, + endCursor: 7, + hasNextPage: false, + hasPreviousPage: false, + startCursor: 7, + totalCount: 1, + }, + }), + })); + + return mount(buildContentPublicRoutes(posts, { pluginId: PLUGIN_ID })); + }; + + it("lists", async () => { + await expectParity(suite(), { + expected: 200, + method: "GET", + path: "/", + template: "/", + }); + }); + + it("reads by slug", async () => { + await expectParity(suite(), { + expected: 200, + method: "GET", + path: "/hello-world", + template: "/{slug}", + }); + }); + + it("answers 404 for an unpublished slug", async () => { + await expectParity(suite(null), { + expected: 404, + method: "GET", + path: "/hello-world", + template: "/{slug}", + }); + }); +}); diff --git a/packages/vitnode/src/content/server/permission-matrix.test.ts b/packages/vitnode/src/content/server/permission-matrix.test.ts new file mode 100644 index 000000000..0896b4fb5 --- /dev/null +++ b/packages/vitnode/src/content/server/permission-matrix.test.ts @@ -0,0 +1,405 @@ +// @vitest-environment node +import type { MiddlewareHandler } from "hono"; + +import { OpenAPIHono } from "@hono/zod-openapi"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { defineContentType } from "../define"; +import { field } from "../fields"; +import { createContentModel } from "./model"; +import { buildContentRoutes } from "./routes"; + +/** + * Every generated route, against the permission it actually demands. + * + * The existing route suites check permissions one endpoint at a time, which is + * fine until somebody adds an endpoint. This one **enumerates** the routes the + * builder produced and drives each of them, so a new route cannot join the set + * without appearing in the matrix below - and a route with no + * `adminStaffPermission` at all cannot join it silently, because it would answer + * something other than 403 with every permission denied. + * + * The permission check itself is stubbed: it reads roles out of the database, + * and what is under test is which `(module, permission)` each route asks for. + */ + +/** Grants for the request currently in flight. `"module:permission"`. */ +let granted = new Set(); +/** What each request was asked for, in order. */ +let asked: { module: string; permission: string; plugin: string }[] = []; + +vi.mock("../../api/lib/check-staff-permission", () => ({ + assertStaffPermission: async ( + _c: unknown, + args: { module: string; permission: string; plugin: string }, + ) => { + asked.push({ + module: args.module, + permission: args.permission, + plugin: args.plugin, + }); + if (granted.has(`${args.module}:${args.permission}`)) return; + + const { HTTPException } = await import("hono/http-exception"); + throw new HTTPException(403, { message: "Forbidden" }); + }, +})); + +/** + * Everything a content type can switch on, at once. + * + * A maximal fixture on purpose: the matrix is only as complete as the set of + * routes the builder was asked to produce, and a fixture missing `scheduling` + * would quietly drop three endpoints out of the audit. + */ +const kitchenSink = defineContentType({ + id: "test.everything", + tableName: "test_everything", + localization: { enabled: true, defaultLocale: "en", fallback: "default" }, + publication: { enabled: true }, + editorial: { + enabled: true, + revisions: { retention: 10 }, + preview: { enabled: true, expiresInMinutes: 30 }, + scheduling: { enabled: true }, + }, + fields: { + title: field.text({ localized: true, required: true, maxLength: 200 }), + slug: field.slug({ localized: true, source: "title" }), + featured: field.boolean({ defaultValue: false }), + // A reference field, so the picker route exists to be audited. + author: field.user(), + // A collection, so the advanced write path is exercised through the same + // `PUT` an ordinary field edit goes through. + faq: field.repeatable({ + fields: { + question: field.text({ required: true, maxLength: 200 }), + answer: field.textarea({ required: true }), + }, + }), + }, + publicApi: { + enabled: true, + path: "everything", + fields: ["title", "slug", "featured", "publishedAt"], + orderableFields: ["publishedAt"], + }, + admin: { + label: { plural: "Everythings", singular: "Everything" }, + list: { columns: ["featured", "status"] }, + form: { fields: ["faq"] }, + }, +}); + +const model = createContentModel(kitchenSink); +const MODULE = kitchenSink.permissionModule; +const PLUGIN_ID = "@vitnode/example"; + +const adminUser = { + avatarColor: "000000", + birthday: null, + createdAt: new Date(), + email: "test@test.com", + emailVerified: true, + id: 1, + language: "en", + name: "Test", + nameCode: "test", + newsletter: false, + roleId: 1, +}; + +/** Concrete values for the path parameters the generated routes declare. */ +const PARAMS: Record = { + field: "author", + id: "7", + locale: "en", + revisionId: "3", + scheduleId: "5", +}; + +const concretePath = (template: string): string => + template.replace(/\{(\w+)\}/g, (_match, name: string) => { + const value = PARAMS[name]; + if (value === undefined) { + throw new Error(`No test value for path parameter "{${name}}".`); + } + + return value; + }); + +/** + * A body wide enough for every write route the builder produces. + * + * Never actually validated in the denial sweep - the permission middleware runs + * first - but a `PUT` with no body would fail for the wrong reason in the + * translator sweep, where some of these routes are allowed through. + */ +const BODY = { + action: "publish" as const, + expectedVersion: 1, + scheduledFor: new Date(Date.now() + 86_400_000).toISOString(), + values: { title: "Hello world" }, +}; + +const routes = buildContentRoutes(model, { pluginId: PLUGIN_ID }); + +const app = (() => { + const instance = new OpenAPIHono(); + const context: MiddlewareHandler = async (c, next) => { + c.set("admin", { user: adminUser }); + c.set("events", { + emit: async () => await Promise.resolve({ failures: [] }), + } as never); + await next(); + }; + instance.use("*", context); + for (const { handler, route } of routes) instance.openapi(route, handler); + + return instance; +})(); + +const request = async (method: string, path: string) => + await app.request(path, { + method, + ...(method === "GET" || method === "DELETE" + ? method === "DELETE" + ? { + body: JSON.stringify(BODY), + headers: { "Content-Type": "application/json" }, + } + : {} + : { + body: JSON.stringify(BODY), + headers: { "Content-Type": "application/json" }, + }), + }); + +/** `"GET /{id}/revisions"` - stable across runs, so the matrix reads as a list. */ +const label = (route: { method: string; path: string }): string => + `${route.method.toUpperCase()} ${route.path}`; + +beforeEach(() => { + granted = new Set(); + asked = []; +}); + +describe("the generated permission matrix", () => { + it("gates every route on a staff permission", async () => { + // Nothing granted, so a route with a permission answers 403 and a route + // without one answers whatever its handler does. The assertion is over the + // whole array rather than a list somebody has to remember to extend. + for (const { route } of routes) { + const res = await request( + route.method.toUpperCase(), + concretePath(route.path), + ); + + expect([label(route), res.status]).toEqual([label(route), 403]); + } + }); + + it("asks for exactly the documented permission on each route", async () => { + const matrix: Record = {}; + + for (const { route } of routes) { + asked = []; + await request(route.method.toUpperCase(), concretePath(route.path)); + + // One check per route, not two: a second would mean a route gated twice, + // where only one of the two is visible in the AdminCP permission editor. + expect([label(route), asked.length]).toEqual([label(route), 1]); + expect(asked[0].module).toBe(MODULE); + matrix[label(route)] = asked[0].permission; + } + + expect(matrix).toEqual({ + "DELETE /{id}": "can_delete", + "DELETE /{id}/translations/{locale}": "can_delete", + "GET /": "can_view", + "GET /options/{field}": "can_view", + "GET /{id}": "can_view", + "GET /{id}/public-locales": "can_view", + "GET /{id}/revisions": "can_view", + "GET /{id}/revisions/{revisionId}": "can_view", + "GET /{id}/schedules": "can_view", + "GET /{id}/translations": "can_view", + "GET /{id}/translations/{locale}": "can_view", + "GET /{id}/translations/{locale}/revisions": "can_view", + "GET /{id}/translations/{locale}/revisions/{revisionId}": "can_view", + "POST /": "can_create", + "POST /{id}/preview": "can_view", + "POST /{id}/publish": "can_publish", + "POST /{id}/revisions/{revisionId}/restore": "can_restore", + "POST /{id}/schedule": "can_publish", + "POST /{id}/schedule/{scheduleId}/cancel": "can_publish", + "POST /{id}/translations/{locale}": "can_translate", + "POST /{id}/translations/{locale}/preview": "can_view", + "POST /{id}/translations/{locale}/publish": "can_publish", + "POST /{id}/translations/{locale}/revisions/{revisionId}/restore": + "can_restore", + "POST /{id}/translations/{locale}/unpublish": "can_publish", + "POST /{id}/unpublish": "can_publish", + "PUT /{id}": "can_edit", + "PUT /{id}/translations/{locale}": "can_translate", + }); + }); + + /** + * The role Stage 5 exists for: somebody who writes Polish and nothing else. + * + * `can_translate` depends on `can_view` and deliberately **not** on + * `can_edit`, so this pair is expressible - and the point of the pair is that + * it stops at the language boundary. A translator who could reach `PUT /{id}` + * could rewrite a shared field; one who could reach the base publish routes + * could put an unfinished record on the internet. + */ + describe("translator isolation", () => { + const TRANSLATOR = [`${MODULE}:can_view`, `${MODULE}:can_translate`]; + + const statusFor = async (method: string, path: string) => { + granted = new Set(TRANSLATOR); + + return (await request(method, path)).status; + }; + + it.each([ + ["PUT", "/7"], + ["POST", "/"], + ["DELETE", "/7"], + ["POST", "/7/publish"], + ["POST", "/7/unpublish"], + ["POST", "/7/revisions/3/restore"], + // A shared revision *and* a locale's own: `can_restore` depends on + // `can_edit`, so a translator has neither. + ["POST", "/7/translations/pl/revisions/3/restore"], + ["POST", "/7/schedule"], + ["POST", "/7/schedule/5/cancel"], + ["DELETE", "/7/translations/en"], + ["POST", "/7/translations/en/publish"], + ["POST", "/7/translations/en/unpublish"], + ])("refuses %s %s", async (method, path) => { + await expect(statusFor(method, path)).resolves.toBe(403); + }); + + it.each([ + ["POST", "/7/translations/pl"], + ["PUT", "/7/translations/pl"], + ])("reaches %s %s", async (method, path) => { + // Past the guard is all this asserts. What the handler then does with a + // record that is not there belongs to the translation suites. + await expect(statusFor(method, path)).resolves.not.toBe(403); + }); + + it.each([ + ["GET", "/"], + ["GET", "/7/translations"], + ["GET", "/7/translations/en/revisions"], + ])("still reads %s %s", async (method, path) => { + await expect(statusFor(method, path)).resolves.not.toBe(403); + }); + }); + + /** + * A collection is written through the ordinary `PUT`, and that is the whole + * answer to "can a relation picker be a write primitive". + * + * There is no per-collection mutation endpoint to gate separately, so an + * editor with `can_view` alone cannot add a category by any route - and the + * picker itself is a read of labels, gated on `can_view` like every other + * read. + */ + describe("advanced collections have no second door", () => { + it("exposes no route outside the audited set", () => { + const paths = routes.map(entry => label(entry.route)); + + expect( + paths.filter( + path => path.includes("relations") || path.includes("repeatable"), + ), + ).toEqual([]); + }); + + it("refuses a collection write to a viewer", async () => { + granted = new Set([`${MODULE}:can_view`]); + + const res = await app.request("/7", { + body: JSON.stringify({ + expectedVersion: 1, + values: { faq: [{ answer: "Yes", question: "Really?" }] }, + }), + headers: { "Content-Type": "application/json" }, + method: "PUT", + }); + + expect(res.status).toBe(403); + }); + + it("lets a viewer open the picker, which reads labels and writes nothing", async () => { + granted = new Set([`${MODULE}:can_view`]); + vi.spyOn(model, "service").mockReturnValue({ + options: async () => await Promise.resolve([]), + } as never); + + const res = await app.request("/options/author"); + + expect(res.status).toBe(200); + vi.restoreAllMocks(); + }); + }); + + /** + * Two plugins can name a permission module the same thing - `articles` is not + * an unusual choice - and the registry allows it precisely because the plugin + * id is part of the key. That only holds if the *route* carries its own + * plugin id into the check rather than reading whichever plugin happens to be + * handling the request. + */ + describe("cross-plugin isolation", () => { + it("checks the permission under the route's own plugin", async () => { + for (const { route } of routes) { + asked = []; + await request(route.method.toUpperCase(), concretePath(route.path)); + + expect([label(route), asked[0].plugin]).toEqual([ + label(route), + PLUGIN_ID, + ]); + } + }); + + it("does not follow the plugin the request is being served by", async () => { + // The same model, mounted by a second plugin. Its routes ask under + // `@vitnode/other`, so granting `@vitnode/example`'s module grants + // nothing here - which is what stops one plugin's roles reaching another + // plugin's content through a module name they happen to share. + const other = new OpenAPIHono(); + other.use("*", async (c, next) => { + c.set("admin", { user: adminUser }); + await next(); + }); + for (const { handler, route } of buildContentRoutes(model, { + pluginId: "@vitnode/other", + })) { + other.openapi(route, handler); + } + + asked = []; + granted = new Set([`${MODULE}:can_view`]); + vi.spyOn(model, "service").mockReturnValue({ + findMany: async () => await Promise.resolve({ edges: [], pageInfo: {} }), + } as never); + const res = await other.request("/"); + vi.restoreAllMocks(); + + // Granted by module name - the module is the same string - and the check + // still ran under the other plugin, which is the fact worth pinning. + expect(res.status).not.toBe(403); + expect(asked[0]).toMatchObject({ + module: MODULE, + permission: "can_view", + plugin: "@vitnode/other", + }); + }); + }); +}); diff --git a/packages/vitnode/src/content/server/public-privacy.test.ts b/packages/vitnode/src/content/server/public-privacy.test.ts new file mode 100644 index 000000000..4cd66419f --- /dev/null +++ b/packages/vitnode/src/content/server/public-privacy.test.ts @@ -0,0 +1,263 @@ +// @vitest-environment node +import { describe, expect, it } from "vitest"; + +import { defineContentType } from "../define"; +import { field } from "../fields"; +import { createContentModel } from "./model"; +import { + contentPublicCollectionFields, + contentPublicSelection, + createContentPublicProjector, +} from "./public-service"; + +/** + * What a public response is allowed to contain, stated as an exact set. + * + * Every other public test asserts that a particular field is present or a + * particular one is absent. This one asserts the **whole** key set, which is the + * only shape of assertion that catches a field nobody thought to check: a leaf + * added to a group later, a system column that started being selected, an + * internal storage name leaking through the flattening. + * + * The fixture is deliberately hostile - every kind has a public member and a + * private sibling, so "the allowlist is a filter" has something to be wrong + * about in each of them. + */ +const contentType = defineContentType({ + id: "test.privacy", + tableName: "test_privacy", + localization: { enabled: true, defaultLocale: "en", fallback: "default" }, + publication: { enabled: true }, + editorial: { enabled: true }, + fields: { + title: field.text({ localized: true, required: true, maxLength: 200 }), + slug: field.slug({ localized: true, source: "title" }), + /** Localized and private: the leak a locale-aware read could produce. */ + internalNotes: field.textarea({ localized: true, nullable: true }), + /** Shared and private. */ + revenue: field.number({ integer: true, defaultValue: 0 }), + featured: field.boolean({ defaultValue: false }), + /** A localized group with one public leaf and one private one. */ + seo: field.group({ + localized: true, + nullable: true, + fields: { + title: field.text({ nullable: true, maxLength: 200 }), + robots: field.text({ nullable: true, maxLength: 100 }), + }, + }), + /** A shared group, entirely private. */ + syndication: field.group({ + fields: { + indexable: field.boolean({ defaultValue: true }), + partnerKey: field.text({ nullable: true, maxLength: 100 }), + }, + }), + /** A repeatable with one public leaf and one private one. */ + faq: field.repeatable({ + fields: { + question: field.text({ required: true, maxLength: 200 }), + answer: field.textarea({ required: true }), + moderatorNote: field.textarea({ nullable: true }), + }, + }), + /** A private to-many relation, and a public one. */ + tags: field.relation({ multiple: true, self: true }), + hiddenLinks: field.relation({ multiple: true, self: true }), + }, + publicApi: { + enabled: true, + path: "privacy", + fields: [ + "title", + "slug", + "featured", + "seo.title", + "faq.question", + "faq.answer", + "tags", + "publishedAt", + ], + orderableFields: ["publishedAt"], + }, + admin: { + label: { plural: "Privacies", singular: "Privacy" }, + list: { columns: ["featured", "status"] }, + form: { fields: ["faq", "tags", "hiddenLinks", "syndication"] }, + }, +}); + +const model = createContentModel(contentType); +const project = createContentPublicProjector(contentType); + +/** + * A raw row carrying **everything** - including the values the projector must + * drop and the flattened storage names it must never surface. + * + * Group leaves arrive already nested, which is what the read layer hands the + * projector; the flat `seoRobots`-style columns are added alongside so a + * projector that copied unknown keys through would be caught here rather than + * in production. + */ +const rawRow = { + createdAt: new Date("2026-01-01T00:00:00.000Z"), + faq: [ + { + answer: "Because.", + id: 11, + moderatorNote: "spam risk", + question: "Why?", + }, + ], + featured: true, + hiddenLinks: [99], + id: 7, + internalNotes: "Do not publish before Friday.", + // The columns the translation and base tables really hold, flattened. + internalNotesColumn: "leak", + languageId: 3, + publishedAt: new Date("2026-02-01T00:00:00.000Z"), + revenue: 12_345, + seo: { robots: "noindex", title: "Public SEO title" }, + seoRobots: "noindex", + seoTitle: "Public SEO title", + slug: "hello-world", + status: "published", + syndication: { indexable: false, partnerKey: "secret-key" }, + syndicationIndexable: false, + syndicationPartnerKey: "secret-key", + tags: [1, 2], + title: "Hello world", + updatedAt: new Date("2026-03-01T00:00:00.000Z"), + version: 9, +}; + +describe("the public projection is an allowlist, not a filter", () => { + const projected = project(rawRow) as Record; + + it("carries exactly the allowlisted keys", () => { + // `id` is absent because the allowlist does not name it: the cursor needs + // it from the database, and the projector drops it again. + expect(Object.keys(projected).sort()).toEqual([ + "faq", + "featured", + "publishedAt", + "seo", + "slug", + "tags", + "title", + ]); + }); + + it.each([ + ["a private localized scalar", "internalNotes"], + ["a private shared scalar", "revenue"], + ["a wholly private group", "syndication"], + ["a private relation", "hiddenLinks"], + ["the editorial version", "version"], + ["the internal language id", "languageId"], + ["a system timestamp", "createdAt"], + ["a system timestamp", "updatedAt"], + ["the publication state", "status"], + ["the cursor identifier", "id"], + ])("drops %s (%s)", (_why, key) => { + expect(projected).not.toHaveProperty(key); + }); + + it("never surfaces a flattened storage column name", () => { + // `seo.title` is stored as `seoTitle`. A response that carried the column + // name would publish an internal detail *and* give a client two spellings + // of one value. + for (const key of Object.keys(projected)) { + expect(key).not.toMatch(/^(seo|syndication|internalNotes)[A-Z]/); + } + }); + + it("keeps a group to the leaves the allowlist named", () => { + expect(projected.seo).toEqual({ title: "Public SEO title" }); + }); + + it("keeps a repeatable child to its public leaves plus its identity", () => { + // The identifier stays: it is what an editor's `set` matches on, and a + // public consumer needs a stable key per row. The moderator note does not. + expect(projected.faq).toEqual([ + { answer: "Because.", id: 11, question: "Why?" }, + ]); + }); + + it("exposes a relation as identifiers and nothing else", () => { + expect(projected.tags).toEqual([1, 2]); + }); + + it("projects a missing collection as an empty list, not as undefined", () => { + const empty = project({ ...rawRow, faq: undefined, tags: undefined }) as { + faq: unknown; + tags: unknown; + }; + + expect(empty.faq).toEqual([]); + expect(empty.tags).toEqual([]); + }); +}); + +describe("the public read never fetches a private column", () => { + const selection = contentPublicSelection(contentType, model.columns); + + it("selects the allowlist plus the cursor, and nothing else", () => { + expect(Object.keys(selection).sort()).toEqual([ + "featured", + "id", + "publishedAt", + "seo.title", + "slug", + "title", + ]); + }); + + it("leaves every private column out of the SELECT entirely", () => { + // Defence in depth that matters: a private column that is never fetched + // cannot be leaked by a mistake in the projector further downstream. + for (const name of [ + "internalNotes", + "revenue", + "seo.robots", + "syndication.indexable", + "syndication.partnerKey", + "version", + "status", + ]) { + expect(selection).not.toHaveProperty(name); + } + }); + + it("loads only the collections the allowlist exposes", () => { + // A public list must not join a junction table it will then discard, and + // `hiddenLinks` is private - so it is not even a candidate. + expect(contentPublicCollectionFields(contentType).sort()).toEqual([ + "faq", + "tags", + ]); + }); +}); + +describe("the generated public schema agrees with the projection", () => { + it("describes the projected keys plus the one piece of generated metadata", () => { + // The OpenAPI contract and the runtime projection are built from the same + // allowlist, so the only difference between them is `locale` - which the + // route adds because a localized response has to say which language it is, + // and which `publicApi.fields` therefore reserves rather than accepts. + const shape = model.schemas.publicSelectObject.shape; + const projected = Object.keys(project(rawRow)); + + expect(Object.keys(shape).sort()).toEqual([...projected, "locale"].sort()); + }); + + it("parses a projected row once the route has stamped the locale on it", () => { + expect( + model.schemas.publicSelectObject.safeParse({ + ...(project(rawRow) as Record), + locale: "en", + }).success, + ).toBe(true); + }); +}); diff --git a/packages/vitnode/src/tests/openapi-validate.ts b/packages/vitnode/src/tests/openapi-validate.ts new file mode 100644 index 000000000..4d104c152 --- /dev/null +++ b/packages/vitnode/src/tests/openapi-validate.ts @@ -0,0 +1,185 @@ +/** + * A JSON Schema check over the subset OpenAPI 3.0 documents actually contain. + * + * It exists because "the runtime response matches the OpenAPI schema" cannot be + * asserted with the Zod object the route was built from. `z.date()` renders in + * the document as `{ type: "string", format: "date-time" }` - which is exactly + * what `c.json(row)` puts on the wire - but the Zod object itself rejects that + * string, so parsing with it would report a contract break where the contract is + * kept. The document is what a generated client is built from, so the document + * is what a response has to satisfy. + * + * Deliberately small: `type`, `properties`, `required`, `nullable`, `enum`, + * `format: date-time`, `items`, `additionalProperties`, `oneOf`/`anyOf`/`allOf` + * and `$ref`. That is everything `@hono/zod-openapi` emits for the generated + * Content Engine routes, and anything it does not understand is reported rather + * than quietly passed. + */ + +export type JsonSchemaLike = Record; + +const ISO_DATE_TIME = + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/; + +const typeOf = (value: unknown): string => { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + + return typeof value; +}; + +const resolveRef = ( + schema: JsonSchemaLike, + document: JsonSchemaLike, +): JsonSchemaLike => { + const ref = schema.$ref; + if (typeof ref !== "string") return schema; + + const path = ref.replace(/^#\//, "").split("/"); + let current: unknown = document; + for (const segment of path) { + current = (current as Record | undefined)?.[segment]; + } + + return (current as JsonSchemaLike | undefined) ?? {}; +}; + +/** + * Every way `value` fails `schema`, as dotted paths with a reason. + * + * An empty array means the response is valid. Returning the whole list rather + * than the first problem is deliberate - a response that is wrong in four places + * should say so once, not four runs in a row. + */ +export const validateAgainstJsonSchema = ( + value: unknown, + rawSchema: JsonSchemaLike, + document: JsonSchemaLike = {}, + path = "", +): string[] => { + const schema = resolveRef(rawSchema, document); + const at = path === "" ? "(root)" : path; + const issues: string[] = []; + + const branches = ["oneOf", "anyOf"] as const; + for (const key of branches) { + const options = schema[key]; + if (!Array.isArray(options)) continue; + + const matched = options.some( + option => + validateAgainstJsonSchema( + value, + option as JsonSchemaLike, + document, + path, + ).length === 0, + ); + + return matched ? [] : [`${at}: matched none of ${key}`]; + } + + if (Array.isArray(schema.allOf)) { + for (const option of schema.allOf) { + issues.push( + ...validateAgainstJsonSchema( + value, + option as JsonSchemaLike, + document, + path, + ), + ); + } + } + + if (value === null) { + // OpenAPI 3.0 spells nullability as a sibling flag rather than as a type, + // which is why this is not simply `type.includes("null")`. + return schema.nullable === true || schema.type === undefined + ? issues + : [...issues, `${at}: null, but the document does not allow it`]; + } + + const expected = schema.type; + if (typeof expected === "string") { + const actual = typeOf(value); + const ok = + expected === "integer" + ? actual === "number" && Number.isInteger(value) + : expected === "number" + ? actual === "number" + : actual === expected; + + if (!ok) { + return [...issues, `${at}: expected ${expected}, got ${actual}`]; + } + } + + if (Array.isArray(schema.enum) && !schema.enum.includes(value)) { + issues.push(`${at}: ${JSON.stringify(value)} is not one of the enum`); + } + + if ( + schema.format === "date-time" && + typeof value === "string" && + !ISO_DATE_TIME.test(value) + ) { + issues.push(`${at}: "${value}" is not an ISO date-time`); + } + + if (expected === "array" && Array.isArray(value)) { + const items = schema.items as JsonSchemaLike | undefined; + if (items) { + value.forEach((entry, index) => { + issues.push( + ...validateAgainstJsonSchema( + entry, + items, + document, + `${path}[${index}]`, + ), + ); + }); + } + } + + if (expected === "object" || (expected === undefined && schema.properties)) { + const object = value as Record; + const properties = (schema.properties ?? {}) as Record< + string, + JsonSchemaLike + >; + const required = Array.isArray(schema.required) + ? (schema.required as string[]) + : []; + + for (const name of required) { + if (!(name in object)) + issues.push(`${at}.${name}: missing, but required`); + } + + for (const [name, entry] of Object.entries(object)) { + const property = properties[name]; + if (!property) { + // `additionalProperties: false` is what a strict object emits, and a key + // the document does not describe is a field a generated client will + // silently drop - or, on a public route, a field nobody meant to ship. + if (schema.additionalProperties === false) { + issues.push(`${at}.${name}: not described by the document`); + } + continue; + } + + issues.push( + ...validateAgainstJsonSchema( + entry, + property, + document, + path === "" ? name : `${path}.${name}`, + ), + ); + } + } + + return issues; +}; From 7bbb77506df5f690ef0bd4548825310b0b13c7ad Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:28:11 +0200 Subject: [PATCH 03/14] test(content): expand the concurrency matrix against real Postgres Twenty-five races on two real connections, because none of this can be shown with a mock: a lock wait, a guarded UPDATE that matches nothing, and a DELETE that commits between another transaction's read and its write are all database behaviour. New coverage: update against delete, update against publish (with and without a version), restore against update, restore against delete, a scheduled transition against a manual one, two locales at once, a locale write against a shared write, and the reorder/remove/add/child-update permutations on both a junction and a repeatable. Two findings worth recording in the assertions rather than the prose: - a `reorder` on an **unordered** relation computes the list that is already stored, so it is a no-op by construction. The reorder races use `relatedArticles`, which is `ordered: true`, or they would be testing nothing. - a collection mutation whose computed next state equals the stored one is a successful no-op and deliberately does **not** check `expectedVersion` - editing a child a concurrent writer already deleted changes nothing, so there is nothing to conflict about. The invariant is therefore "one race, one version increment", not "one race, one fulfilled promise". Co-Authored-By: Claude Opus 5 (1M context) --- .../src/database/concurrency-postgres.test.ts | 1185 +++++++++++++++++ plugins/example/src/database/harness.ts | 444 ++++++ 2 files changed, 1629 insertions(+) create mode 100644 plugins/example/src/database/concurrency-postgres.test.ts create mode 100644 plugins/example/src/database/harness.ts diff --git a/plugins/example/src/database/concurrency-postgres.test.ts b/plugins/example/src/database/concurrency-postgres.test.ts new file mode 100644 index 000000000..9a78457c7 --- /dev/null +++ b/plugins/example/src/database/concurrency-postgres.test.ts @@ -0,0 +1,1185 @@ +import type { Context } from "hono"; + +import { executeContentSchedule } from "@vitnode/core/api/modules/content/helpers/execute-content-schedule"; +import { + ContentTranslationVersionConflict, + ContentVersionConflict, +} from "@vitnode/core/content"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import { CONFIG_PLUGIN } from "@/const"; + +import type { ContentTestHarness } from "./harness"; + +import { advancedArticleContent } from "./advanced-articles"; +import { articleContent } from "./articles"; +import { + ACTOR, + clearContentTables, + createContentTestHarness, + DATABASE_TEST_URL, + fulfilledCount, + race, + reasons, +} from "./harness"; +import { localizedArticleContent } from "./localized-articles"; + +/** + * The Stage 7 concurrency matrix, against real Postgres. + * + * Every test here runs two writers on two separate connections at the same + * moment. That is the only way any of it can be shown: a mock cannot produce a + * lock wait, a guarded `UPDATE` that matches nothing, or a `DELETE` that commits + * between another transaction's read and its write. + * + * The invariants, stated once so each test can be read against them: + * + * - **exactly one winner** wherever both writers carry the same + * `expectedVersion`, and the loser is told which version it lost to; + * - **no resurrection** - a record deleted by one writer is never brought back + * by another's write, and neither is a translation; + * - **no partial state** - a losing writer leaves the collections exactly as it + * found them, because the version guard runs before a single junction or child + * row is touched; + * - **monotonic versions** - a race produces one increment, not two, and never + * two revisions at the same version. + */ + +let h: ContentTestHarness; + +const editorial = (on: Context) => { + const build = articleContent.editorialService; + if (!build) throw new Error("example.article has no editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const advanced = (on: Context) => { + const build = advancedArticleContent.editorialService; + if (!build) throw new Error("no advanced editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const plainAdvanced = (on: Context) => advancedArticleContent.service(on); + +const localizedService = (on: Context) => { + const build = localizedArticleContent.localizedService; + if (!build) throw new Error("no localized service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const translationEditorial = (on: Context) => { + const build = localizedArticleContent.translationEditorialService; + if (!build) throw new Error("no translation editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +let categoryId = 0; +let seq = 0; + +/** A published-ready article, at version 1 with one `create` revision. */ +const article = async (overrides: Record = {}) => { + seq += 1; + const outcome = await editorial(h.context).create( + { + category: categoryId, + code: `race-${seq}`, + title: `Race subject ${seq}`, + ...overrides, + }, + { actor: ACTOR }, + ); + + return { id: outcome.row.id, version: outcome.version }; +}; + +const rowOf = async (id: number) => { + const [row] = await h.sql< + { status: string; title: string; version: number }[] + >` + SELECT "title", "status", "version" FROM "example_articles" WHERE "id" = ${id} + `; + + return row; +}; + +const revisionsOf = async (id: number) => + await h.sql<{ operation: string; version: number }[]>` + SELECT "operation", "version" FROM "core_content_revisions" + WHERE "contentTypeId" = 'example.article' AND "itemId" = ${id} + ORDER BY "version" + `; + +const isVersionConflict = (error: unknown): boolean => + error instanceof ContentVersionConflict; + +/** + * How many of a race's sides actually **changed** something. + * + * Not the same as how many succeeded: a collection mutation whose computed next + * state equals the stored one is a successful no-op, and a no-op deliberately + * does not check `expectedVersion` - there is nothing to overwrite, so there is + * nothing to conflict about. Counting real mutations is what pins "one race, + * one version increment". + */ +const changedCount = ( + results: readonly PromiseSettledResult[], +): number => + results.filter( + entry => + entry.status === "fulfilled" && + (entry.value as null | { changed?: boolean })?.changed === true, + ).length; + +describe.skipIf(!DATABASE_TEST_URL)("Content Engine concurrency", () => { + beforeAll(async () => { + h = await createContentTestHarness(); + }, 60_000); + + afterAll(async () => { + await h?.end(); + }); + + beforeEach(async () => { + await clearContentTables(h.sql); + h.reset(); + + const [category] = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") VALUES ('Races') RETURNING "id" + `; + categoryId = category.id; + }); + + // ------------------------------------------------------------------------- + // Base record races + // ------------------------------------------------------------------------- + + describe("update against update", () => { + it("lets exactly one writer win and tells the other which version it lost to", async () => { + const { id } = await article(); + + const results = await race( + async () => + await editorial(h.context).update( + id, + { title: "Writer A" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await editorial(h.rivalContext).update( + id, + { title: "Writer B" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + ); + + expect(fulfilledCount(results)).toBe(1); + expect(reasons(results).every(isVersionConflict)).toBe(true); + expect( + (reasons(results)[0] as ContentVersionConflict).currentVersion, + ).toBe(2); + + const row = await rowOf(id); + expect(row.version).toBe(2); + expect(["Writer A", "Writer B"]).toContain(row.title); + + // One increment and one revision, not two of either. The loser wrote + // nothing at all, so there is no partial mutation to find. + expect((await revisionsOf(id)).map(entry => entry.operation)).toEqual([ + "create", + "update", + ]); + }); + }); + + /** + * Two writers, one of which removes the record. + * + * The order decides which of two shapes the loser sees, and both are stated + * rather than accepted as "either": + * + * - the **update** commits first, so the delete's guarded `DELETE` matches + * nothing and the follow-up read finds version 2: a conflict; + * - the **delete** commits first, so the update's read finds no row at all: + * `null`, which the route turns into a 404. + * + * What never happens is a resurrection - the update's `UPDATE` is guarded by + * both the id and the version, so it cannot recreate a row - and a revision + * for a state that never existed. + */ + describe("update against delete", () => { + it("either refuses the delete or answers the update with nothing", async () => { + const { id } = await article(); + + const results = await race( + async () => + await editorial(h.context).update( + id, + { title: "Edited" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await editorial(h.rivalContext).delete(id, { + actor: ACTOR, + expectedVersion: 1, + }), + ); + + const [updateResult, deleteResult] = results; + + if (deleteResult.status === "fulfilled" && deleteResult.value) { + // The delete won. The update either lost the guard (a conflict) or + // found nothing (`null`) - never a row it went on to rewrite. + expect(await rowOf(id)).toBeUndefined(); + if (updateResult.status === "fulfilled") { + expect(updateResult.value).toBeNull(); + } else { + expect(isVersionConflict(updateResult.reason)).toBe(true); + } + + return; + } + + // The update won, so the delete was refused on the version rather than + // silently removing a record somebody had just edited. + expect(updateResult.status).toBe("fulfilled"); + expect(deleteResult.status).toBe("rejected"); + expect( + isVersionConflict( + deleteResult.status === "rejected" ? deleteResult.reason : null, + ), + ).toBe(true); + expect((await rowOf(id)).title).toBe("Edited"); + }); + + it("never leaves a revision describing a record that was never in that state", async () => { + const { id } = await article(); + + await race( + async () => + await editorial(h.context).update( + id, + { title: "Edited" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await editorial(h.rivalContext).delete(id, { + actor: ACTOR, + expectedVersion: 1, + }), + ); + + const history = await revisionsOf(id); + // Strictly increasing, with no version written twice - which is what the + // partial unique index enforces and what a lost race must not disturb. + expect(history.map(entry => entry.version)).toEqual( + [...history.map(entry => entry.version)].sort((a, b) => a - b), + ); + expect(new Set(history.map(entry => entry.version)).size).toBe( + history.length, + ); + }); + }); + + /** + * A field edit against a publication. + * + * Publishing takes an **optional** `expectedVersion`, because it overwrites no + * field value: requiring one would fail the publish button whenever a + * colleague had fixed a typo, for no protection against a lost update. Both + * halves of that decision are pinned here. + */ + describe("update against publish", () => { + it("lets exactly one win when both carry the same expected version", async () => { + const { id } = await article(); + + const results = await race( + async () => + await editorial(h.context).update( + id, + { title: "Edited first" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await editorial(h.rivalContext).publish(id, { + actor: ACTOR, + expectedVersion: 1, + }), + ); + + expect(fulfilledCount(results)).toBe(1); + expect(reasons(results).every(isVersionConflict)).toBe(true); + expect((await rowOf(id)).version).toBe(2); + }); + + it("overwrites no field value when the publish carries no version", async () => { + // The documented behaviour: an unguarded publish moves `status` and + // nothing else, so a concurrent edit either lands before it or is + // refused - but the title it wrote is never reverted by the publish. + const { id } = await article(); + + const results = await race( + async () => + await editorial(h.context).update( + id, + { title: "Edited alongside" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await editorial(h.rivalContext).publish(id, { actor: ACTOR }), + ); + + const row = await rowOf(id); + expect(row.status).toBe("published"); + + const [updateResult] = results; + if (updateResult.status === "fulfilled" && updateResult.value?.changed) { + expect(row.title).toBe("Edited alongside"); + expect(row.version).toBe(3); + + return; + } + + // Refused, and the title it never wrote is not on the row. + expect(row.title).not.toBe("Edited alongside"); + expect(row.version).toBe(2); + }); + }); + + /** + * A restore is the widest overwrite the engine has - it rewrites many fields + * at once from a source the editor did not type - so it carries the same + * required `expectedVersion` an ordinary update does. + */ + describe("restore against update", () => { + it("never overwrites a newer edit silently", async () => { + const { id } = await article({ title: "Original" }); + await editorial(h.context).update( + id, + { title: "Second" }, + { actor: ACTOR, expectedVersion: 1 }, + ); + + const history = await editorial(h.context).revisions.list(id); + const first = history.edges.find(entry => entry.operation === "create"); + if (!first) throw new Error("Expected a create revision."); + + const results = await race( + async () => + await editorial(h.context).restore(id, first.id, { + actor: ACTOR, + expectedVersion: 2, + }), + async () => + await editorial(h.rivalContext).update( + id, + { title: "Third" }, + { actor: ACTOR, expectedVersion: 2 }, + ), + ); + + expect(fulfilledCount(results)).toBe(1); + expect(reasons(results).every(isVersionConflict)).toBe(true); + + const row = await rowOf(id); + expect(row.version).toBe(3); + // Whichever won, the record holds exactly that writer's value - never a + // mixture, and never the loser's. + expect(["Original", "Third"]).toContain(row.title); + }); + + it("writes exactly one new revision, never rewriting the one it restored from", async () => { + const { id } = await article({ title: "Original" }); + await editorial(h.context).update( + id, + { title: "Second" }, + { actor: ACTOR, expectedVersion: 1 }, + ); + + const history = await editorial(h.context).revisions.list(id); + const first = history.edges.find(entry => entry.operation === "create"); + if (!first) throw new Error("Expected a create revision."); + const before = await editorial(h.context).revisions.findById( + id, + first.id, + ); + + await race( + async () => + await editorial(h.context).restore(id, first.id, { + actor: ACTOR, + expectedVersion: 2, + }), + async () => + await editorial(h.rivalContext).update( + id, + { title: "Third" }, + { actor: ACTOR, expectedVersion: 2 }, + ), + ); + + const after = await editorial(h.context).revisions.findById(id, first.id); + expect(after?.snapshot).toEqual(before?.snapshot); + expect(after?.version).toBe(before?.version); + expect((await revisionsOf(id)).length).toBe(3); + }); + }); + + describe("restore against delete", () => { + it("never recreates a record a concurrent delete removed", async () => { + const { id } = await article({ title: "Original" }); + await editorial(h.context).update( + id, + { title: "Second" }, + { actor: ACTOR, expectedVersion: 1 }, + ); + const history = await editorial(h.context).revisions.list(id); + const first = history.edges.find(entry => entry.operation === "create"); + if (!first) throw new Error("Expected a create revision."); + + const results = await race( + async () => + await editorial(h.context).restore(id, first.id, { + actor: ACTOR, + expectedVersion: 2, + }), + async () => + await editorial(h.rivalContext).delete(id, { + actor: ACTOR, + expectedVersion: 2, + }), + ); + + const [restoreResult, deleteResult] = results; + + if (deleteResult.status === "fulfilled" && deleteResult.value) { + // Gone, and it stays gone: a restore reads the live row before it + // writes, so there is no row for it to resurrect. + expect(await rowOf(id)).toBeUndefined(); + if (restoreResult.status === "fulfilled") { + expect(restoreResult.value).toBeNull(); + } else { + expect(isVersionConflict(restoreResult.reason)).toBe(true); + } + + return; + } + + expect(restoreResult.status).toBe("fulfilled"); + expect((await rowOf(id)).title).toBe("Original"); + }); + }); + + // ------------------------------------------------------------------------- + // Scheduled against manual + // ------------------------------------------------------------------------- + + describe("a scheduled transition against a manual one", () => { + const schedules = (on: Context) => { + const model = editorial(on).schedules; + if (!model) throw new Error("example.article has no scheduling"); + + return model; + }; + + const book = async (id: number, action: "publish" | "unpublish") => + await schedules(h.context).schedule({ + action, + actorUserId: null, + itemId: id, + // Inside the past tolerance, so it is due on this tick. + scheduledFor: new Date(Date.now() - 1000), + }); + + it("does nothing when the manual publish got there first", async () => { + const { id } = await article(); + const booked = await book(id, "publish"); + + await editorial(h.context).publish(id, { actor: ACTOR }); + const outcome = await executeContentSchedule(h.context, { + generation: booked.generation, + scheduleId: booked.id, + }); + + // Skipped rather than executed: the state guard on the transition is what + // makes a scheduled publish idempotent, and idempotent is what stops a + // second revision and a second announcement. + expect(outcome).toMatchObject({ + reason: "already in that state", + status: "skipped", + }); + expect( + (await revisionsOf(id)).filter(entry => entry.operation === "publish"), + ).toHaveLength(1); + + const effects = await h.sql` + SELECT "id" FROM "core_queue" WHERE "name" = 'content-schedule-effects' + `; + expect(effects).toHaveLength(0); + }); + + it("does not un-publish a record the editor published after booking the unpublish", async () => { + const { id } = await article(); + await editorial(h.context).publish(id, { actor: ACTOR }); + const booked = await book(id, "unpublish"); + + // Manual unpublish first; the stale booking then finds nothing to do. + await editorial(h.context).unpublish(id, { actor: ACTOR }); + const outcome = await executeContentSchedule(h.context, { + generation: booked.generation, + scheduleId: booked.id, + }); + + expect(outcome.status).toBe("skipped"); + expect( + (await revisionsOf(id)).filter( + entry => entry.operation === "unpublish", + ), + ).toHaveLength(1); + }); + + it("settles the booking either way, so it never runs twice", async () => { + const { id } = await article(); + const booked = await book(id, "publish"); + await editorial(h.context).publish(id, { actor: ACTOR }); + + await executeContentSchedule(h.context, { + generation: booked.generation, + scheduleId: booked.id, + }); + + const [schedule] = await h.sql<{ status: string }[]>` + SELECT "status" FROM "core_content_schedules" WHERE "id" = ${booked.id} + `; + expect(schedule.status).toBe("completed"); + + // A second delivery of the same queue row is a no-op: the claim refuses + // anything that is not still `pending`. + const again = await executeContentSchedule(h.context, { + generation: booked.generation, + scheduleId: booked.id, + }); + expect(again.status).toBe("skipped"); + }); + + it("keeps a scheduled publish and a concurrent edit to one version each", async () => { + const { id } = await article(); + const booked = await book(id, "publish"); + + const results = await race( + async () => + await executeContentSchedule(h.context, { + generation: booked.generation, + scheduleId: booked.id, + }), + async () => + await editorial(h.rivalContext).update( + id, + { title: "Edited while publishing" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + ); + + const row = await rowOf(id); + const history = await revisionsOf(id); + + // However they interleaved: no version was written twice, and the row's + // version equals the highest revision. + expect(new Set(history.map(entry => entry.version)).size).toBe( + history.length, + ); + expect(row.version).toBe( + Math.max(...history.map(entry => entry.version)), + ); + + // The publish either committed or was rolled back whole - never half. + const published = history.some(entry => entry.operation === "publish"); + expect(row.status).toBe(published ? "published" : "draft"); + expect(fulfilledCount(results)).toBeGreaterThan(0); + }); + }); + + // ------------------------------------------------------------------------- + // Translations + // ------------------------------------------------------------------------- + + describe("translations", () => { + const guide = async (title: string) => { + const { row } = await localizedService(h.context).create( + { shared: {}, translation: { body: `Body of ${title}`, title } }, + { actor: ACTOR }, + ); + + return row.id; + }; + + const translationRows = async (itemId: number) => + await h.sql<{ languageId: number; title: string; version: number }[]>` + SELECT "languageId", "title", "version" + FROM "example_localized_articles_translations" + WHERE "itemId" = ${itemId} + ORDER BY "languageId" + `; + + it("lets exactly one of two writers on the same locale win", async () => { + const itemId = await guide("Same Locale"); + await translationEditorial(h.context).create( + itemId, + "pl", + { body: "Tresc", title: "Polski" }, + { actor: ACTOR }, + ); + + const results = await race( + async () => + await translationEditorial(h.context).update( + itemId, + "pl", + { title: "Polski A" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await translationEditorial(h.rivalContext).update( + itemId, + "pl", + { title: "Polski B" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + ); + + expect(fulfilledCount(results)).toBe(1); + expect( + reasons(results).every( + error => error instanceof ContentTranslationVersionConflict, + ), + ).toBe(true); + + const rows = await translationRows(itemId); + expect(rows.find(row => row.languageId === 2)?.version).toBe(2); + }); + + it("lets two locales be written at the same time, independently", async () => { + const itemId = await guide("Two Locales"); + await translationEditorial(h.context).create( + itemId, + "pl", + { body: "Tresc", title: "Polski" }, + { actor: ACTOR }, + ); + + const results = await race( + async () => + await translationEditorial(h.context).update( + itemId, + "pl", + { title: "Polski Nowy" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await translationEditorial(h.rivalContext).update( + itemId, + "en", + { title: "English New" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + ); + + // Two version domains, so both win: somebody editing Polish must never be + // told the English copy moved. + expect(fulfilledCount(results)).toBe(2); + + const rows = await translationRows(itemId); + expect(rows.map(row => row.version)).toEqual([2, 2]); + expect(rows.map(row => row.title).sort()).toEqual([ + "English New", + "Polski Nowy", + ]); + }); + + it("lets a locale write and a shared write both succeed", async () => { + const itemId = await guide("Shared And Local"); + await translationEditorial(h.context).create( + itemId, + "pl", + { body: "Tresc", title: "Polski" }, + { actor: ACTOR }, + ); + + const results = await race( + async () => + await translationEditorial(h.context).update( + itemId, + "pl", + { title: "Polski Zmieniony" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + async () => + await h.rivalDb.execute( + // The shared half, written straight through SQL: what matters here + // is that the two version columns are on two different rows, so + // neither guard can see the other's write. + `UPDATE "example_localized_articles" SET "featured" = true WHERE "id" = ${itemId}`, + ), + ); + + expect(fulfilledCount(results)).toBe(2); + + const [base] = await h.sql<{ featured: boolean }[]>` + SELECT "featured" FROM "example_localized_articles" WHERE "id" = ${itemId} + `; + expect(base.featured).toBe(true); + expect( + (await translationRows(itemId)).find(row => row.languageId === 2) + ?.title, + ).toBe("Polski Zmieniony"); + }); + + it("never resurrects a translation a concurrent delete removed", async () => { + const itemId = await guide("Delete Race"); + await translationEditorial(h.context).create( + itemId, + "pl", + { body: "Tresc", title: "Polski" }, + { actor: ACTOR }, + ); + + const results = await race( + async () => + await translationEditorial(h.context).delete(itemId, "pl", { + actor: ACTOR, + expectedVersion: 1, + }), + async () => + await translationEditorial(h.rivalContext).update( + itemId, + "pl", + { title: "Stale" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + ); + + const rows = await translationRows(itemId); + const polish = rows.find(row => row.languageId === 2); + + const [deleteResult] = results; + if (deleteResult.status === "fulfilled" && deleteResult.value) { + expect(polish).toBeUndefined(); + + return; + } + + // The update won, so the delete was refused - and the translation holds + // the update's value rather than a mixture. + expect(polish?.title).toBe("Stale"); + }); + }); + + // ------------------------------------------------------------------------- + // Advanced collections + // ------------------------------------------------------------------------- + + describe("advanced collections", () => { + let categories: number[] = []; + + const advancedArticle = async () => { + // `title` is localized on this content type, so it is not a shared field + // and never appears in a base create payload. + const outcome = await advanced(h.context).create( + { categories: [] }, + { actor: ACTOR }, + ); + + return { id: outcome.row.id, version: outcome.version }; + }; + + const junctionRows = async (id: number) => + await h.sql<{ position: number; relatedItemId: number }[]>` + SELECT "relatedItemId", "position" + FROM "example_advanced_articles_categories" + WHERE "itemId" = ${id} + ORDER BY "position" + `; + + const relatedRows = async (id: number) => + await h.sql<{ position: number; relatedItemId: number }[]>` + SELECT "relatedItemId", "position" + FROM "example_advanced_articles_related_articles" + WHERE "itemId" = ${id} + ORDER BY "position" + `; + + const versionOfAdvanced = async (id: number) => { + const [row] = await h.sql<{ version: number }[]>` + SELECT "version" FROM "example_advanced_articles" WHERE "id" = ${id} + `; + + return row.version; + }; + + const faqRows = async (id: number) => + await h.sql<{ id: number; position: number; question: string }[]>` + SELECT "id", "position", "question" + FROM "example_advanced_articles_faq" + WHERE "itemId" = ${id} + ORDER BY "position" + `; + + beforeEach(async () => { + const rows = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") + VALUES ('One'), ('Two'), ('Three') RETURNING "id" + `; + categories = rows.map(row => row.id); + }); + + it("refuses one of two racing adds and leaves no half-written set", async () => { + const { id, version } = await advancedArticle(); + + const results = await race( + async () => + await advanced(h.context).relations.categories.add( + id, + categories[0], + { actor: ACTOR, expectedVersion: version }, + ), + async () => + await advanced(h.rivalContext).relations.categories.add( + id, + categories[1], + { actor: ACTOR, expectedVersion: version }, + ), + ); + + expect(fulfilledCount(results)).toBe(1); + + // Exactly one target, at position 0 - never two rows written by two + // writers who each thought the set was empty. + const rows = await junctionRows(id); + expect(rows).toHaveLength(1); + expect(rows[0].position).toBe(0); + }); + + it("refuses a remove racing an add", async () => { + const { id, version } = await advancedArticle(); + const seeded = await advanced(h.context).relations.categories.set( + id, + [categories[0]], + { actor: ACTOR, expectedVersion: version }, + ); + const at = seeded?.version ?? version; + + const results = await race( + async () => + await advanced(h.context).relations.categories.remove( + id, + categories[0], + { actor: ACTOR, expectedVersion: at }, + ), + async () => + await advanced(h.rivalContext).relations.categories.add( + id, + categories[1], + { actor: ACTOR, expectedVersion: at }, + ), + ); + + expect(fulfilledCount(results)).toBe(1); + + const rows = await junctionRows(id); + // Either the removal happened (empty) or the addition did (two targets) - + // never the removal's result with the addition's row in it. + expect([0, 2]).toContain(rows.length); + expect(rows.map(row => row.position)).toEqual( + rows.map((_row, index) => index), + ); + }); + + /** + * A reorder needs an **ordered** relation to be a mutation at all. + * + * `categories` is unordered: the engine stores it in ascending target order, + * so `reorder` there computes the list that is already stored and is a no-op + * by construction. `relatedArticles` is `ordered: true`, which is what makes + * the author's sequence a fact the database holds - and what makes racing a + * reorder against something else a real contest. + */ + it("keeps positions contiguous when a reorder races an add", async () => { + const { id, version } = await advancedArticle(); + const first = await advancedArticle(); + const second = await advancedArticle(); + const third = await advancedArticle(); + + const seeded = await advanced(h.context).relations.relatedArticles.set( + id, + [first.id, second.id], + { actor: ACTOR, expectedVersion: version }, + ); + const at = seeded?.version ?? version; + + const results = await race( + async () => + await advanced(h.context).relations.relatedArticles.reorder( + id, + [second.id, first.id], + { actor: ACTOR, expectedVersion: at }, + ), + async () => + await advanced(h.rivalContext).relations.relatedArticles.add( + id, + third.id, + { actor: ACTOR, expectedVersion: at }, + ), + ); + + // Exactly one real mutation, so exactly one version increment. The loser + // either lost the guard or found its own computation was a no-op; neither + // writes a junction row. + expect(changedCount(results)).toBe(1); + expect(await versionOfAdvanced(id)).toBe(at + 1); + + const rows = await relatedRows(id); + expect(rows.map(row => row.position)).toEqual( + rows.map((_row, index) => index), + ); + expect(new Set(rows.map(row => row.relatedItemId)).size).toBe( + rows.length, + ); + }); + + it("keeps positions contiguous when a reorder races a remove", async () => { + const { id, version } = await advancedArticle(); + const first = await advancedArticle(); + const second = await advancedArticle(); + const third = await advancedArticle(); + + const seeded = await advanced(h.context).relations.relatedArticles.set( + id, + [first.id, second.id, third.id], + { actor: ACTOR, expectedVersion: version }, + ); + const at = seeded?.version ?? version; + + const results = await race( + async () => + await advanced(h.context).relations.relatedArticles.reorder( + id, + [third.id, second.id, first.id], + { actor: ACTOR, expectedVersion: at }, + ), + async () => + await advanced(h.rivalContext).relations.relatedArticles.remove( + id, + first.id, + { actor: ACTOR, expectedVersion: at }, + ), + ); + + expect(changedCount(results)).toBe(1); + expect(await versionOfAdvanced(id)).toBe(at + 1); + + const rows = await relatedRows(id); + expect([2, 3]).toContain(rows.length); + expect(rows.map(row => row.position)).toEqual( + rows.map((_row, index) => index), + ); + }); + + it("refuses a repeatable delete racing an update of the same child", async () => { + const { id, version } = await advancedArticle(); + const seeded = await advanced(h.context).repeatable.faq.set( + id, + [ + { answer: "A1", question: "Question one" }, + { answer: "A2", question: "Question two" }, + ], + { actor: ACTOR, expectedVersion: version }, + ); + const at = seeded?.version ?? version; + const children = await faqRows(id); + + const results = await race( + async () => + await advanced(h.context).repeatable.faq.delete(id, children[0].id, { + actor: ACTOR, + expectedVersion: at, + }), + async () => + await advanced(h.rivalContext).repeatable.faq.update( + id, + children[0].id, + { question: "Question one edited" }, + { actor: ACTOR, expectedVersion: at }, + ), + ); + + // At most one *real* mutation. The loser either lost the version guard or + // discovered its own computation was a no-op - editing a child that is + // already gone changes nothing, and a no-op is deliberately not a + // conflict, because there is nothing to overwrite. + expect(changedCount(results)).toBe(1); + expect(await versionOfAdvanced(id)).toBe(at + 1); + + const rows = await faqRows(id); + // Either the child is gone or it holds the edit - never a resurrected row + // carrying the pre-edit values. + const first = rows.find(row => row.id === children[0].id); + if (first) expect(first.question).toBe("Question one edited"); + expect(rows.map(row => row.position)).toEqual( + rows.map((_row, index) => index), + ); + }); + + it("refuses a child update racing a reorder", async () => { + const { id, version } = await advancedArticle(); + const seeded = await advanced(h.context).repeatable.faq.set( + id, + [ + { answer: "A1", question: "Question one" }, + { answer: "A2", question: "Question two" }, + ], + { actor: ACTOR, expectedVersion: version }, + ); + const at = seeded?.version ?? version; + const children = await faqRows(id); + + const results = await race( + async () => + await advanced(h.context).repeatable.faq.update( + id, + children[0].id, + { question: "Question one edited" }, + { actor: ACTOR, expectedVersion: at }, + ), + async () => + await advanced(h.rivalContext).repeatable.faq.reorder( + id, + [children[1].id, children[0].id], + { actor: ACTOR, expectedVersion: at }, + ), + ); + + expect(fulfilledCount(results)).toBe(1); + + const rows = await faqRows(id); + expect(rows).toHaveLength(2); + expect(rows.map(row => row.position)).toEqual([0, 1]); + // Identity survived whichever way it went: both children are still the + // rows they were, not recreated ones. + expect(new Set(rows.map(row => row.id))).toEqual( + new Set(children.map(row => row.id)), + ); + }); + + it("refuses a collection write racing a scalar write on the same version", async () => { + const { id, version } = await advancedArticle(); + + const results = await race( + async () => + await advanced(h.context).relations.categories.add( + id, + categories[0], + { actor: ACTOR, expectedVersion: version }, + ), + async () => + await advanced(h.rivalContext).update( + id, + { syndication: { indexable: false, priority: 3 } }, + { actor: ACTOR, expectedVersion: version }, + ), + ); + + expect(fulfilledCount(results)).toBe(1); + + const [row] = await h.sql< + { syndicationPriority: number; version: number }[] + >` + SELECT "version", "syndicationPriority" FROM "example_advanced_articles" + WHERE "id" = ${id} + `; + expect(row.version).toBe(version + 1); + + const junction = await junctionRows(id); + // One or the other, never both halves of two different writers. + if (junction.length > 0) { + expect(row.syndicationPriority).toBe(5); + } else { + expect(row.syndicationPriority).toBe(3); + } + }); + + it("keeps two different records independent under load", async () => { + const first = await advancedArticle(); + const second = await advancedArticle(); + + const results = await race( + async () => + await advanced(h.context).relations.categories.add( + first.id, + categories[0], + { actor: ACTOR, expectedVersion: first.version }, + ), + async () => + await advanced(h.rivalContext).relations.categories.add( + second.id, + categories[1], + { actor: ACTOR, expectedVersion: second.version }, + ), + ); + + // The lock is per row, so two records never contend. + expect(fulfilledCount(results)).toBe(2); + expect(await junctionRows(first.id)).toHaveLength(1); + expect(await junctionRows(second.id)).toHaveLength(1); + }); + }); + + // ------------------------------------------------------------------------- + // The plain service, which merges rather than arbitrating + // ------------------------------------------------------------------------- + + describe("the plain service serialises instead of conflicting", () => { + let categories: number[] = []; + + beforeEach(async () => { + const rows = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") + VALUES ('Plain One'), ('Plain Two') RETURNING "id" + `; + categories = rows.map(row => row.id); + }); + + it("keeps both concurrent additions", async () => { + // No version column to guard on, so the row lock does the job instead: + // the second `add` waits, then reads what the first committed. + const outcome = await advanced(h.context).create( + { categories: [] }, + { actor: ACTOR }, + ); + const id = outcome.row.id; + + const results = await race( + async () => + await plainAdvanced(h.context).relations.categories.add( + id, + categories[0], + ), + async () => + await plainAdvanced(h.rivalContext).relations.categories.add( + id, + categories[1], + ), + ); + + expect(fulfilledCount(results)).toBe(2); + + const rows = await h.sql<{ position: number }[]>` + SELECT "position" FROM "example_advanced_articles_categories" + WHERE "itemId" = ${id} ORDER BY "position" + `; + expect(rows.map(row => row.position)).toEqual([0, 1]); + }); + }); +}); diff --git a/plugins/example/src/database/harness.ts b/plugins/example/src/database/harness.ts new file mode 100644 index 000000000..50eb05f29 --- /dev/null +++ b/plugins/example/src/database/harness.ts @@ -0,0 +1,444 @@ +import type { SearchDocument } from "@vitnode/core/api/models/search"; +import type { Context } from "hono"; + +import { core_queue } from "@vitnode/core/database/queue"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import postgres from "postgres"; + +import { CONFIG_PLUGIN, EXAMPLE_MIGRATIONS } from "@/const"; + +import { advancedArticleContent } from "./advanced-articles"; +import { articleContent } from "./articles"; +import { categoryContent } from "./categories"; +import { localizedArticleContent } from "./localized-articles"; + +/** + * The shared Postgres fixture for the Stage 7 hardening suites. + * + * Extracted rather than copied because there are now several suites that need + * the same thing: a schema built from the committed migrations, the core tables + * the Content Engine writes to, and a request context whose event transport, + * search engine, queue and logger all *record* instead of doing. + * + * Every suite runs against the same database and every one of them drops the + * schema in its `beforeAll`, which is why `vitest.config.ts` sets + * `fileParallelism: false`. That is a deliberate trade: one shared, real + * database beats several mocked ones, and a suite that cannot see the + * constraints is not testing the thing it claims to. + */ + +export const DATABASE_TEST_URL = process.env.DATABASE_TEST_URL; + +const databaseName = (() => { + if (!DATABASE_TEST_URL) return ""; + try { + return new URL(DATABASE_TEST_URL).pathname.replace(/^\//, ""); + } catch { + return ""; + } +})(); + +const here = dirname(fileURLToPath(import.meta.url)); + +const migrationSql = (files: readonly string[]): string => + files + .map(file => + readFileSync( + resolve(here, "../../../../apps/docs/migrations", file), + "utf8", + ), + ) + .join("\n--> statement-breakpoint\n"); + +/** + * The core tables the engine writes to, stubbed to the columns it touches. + * + * Core's own migration history is not replayed: one of its migrations builds a + * full-text column from per-language text-search configurations a stock + * Postgres image does not ship, and none of that has anything to do with the + * Content Engine. Pulling it in would make every unrelated core change a reason + * for these suites to break. + */ +const CORE_STUBS = ` + CREATE TABLE "core_users" ( + "id" serial PRIMARY KEY NOT NULL, + "name" varchar(255) NOT NULL + ); + CREATE TABLE "core_queue" ( + "id" serial PRIMARY KEY NOT NULL, + "pluginId" varchar(255) NOT NULL, + "name" varchar(100) NOT NULL, + "queue" varchar(100) DEFAULT 'default' NOT NULL, + "status" varchar(20) DEFAULT 'pending' NOT NULL, + "payload" jsonb DEFAULT '{}'::jsonb NOT NULL, + "priority" integer DEFAULT 0 NOT NULL, + "attempts" integer DEFAULT 0 NOT NULL, + "maxAttempts" integer DEFAULT 3 NOT NULL, + "availableAt" timestamp DEFAULT now() NOT NULL, + "reservedAt" timestamp, + "lastError" text, + "createdAt" timestamp DEFAULT now() NOT NULL, + "updatedAt" timestamp DEFAULT now() NOT NULL, + "completedAt" timestamp + ); + CREATE TABLE "core_languages" ( + "id" serial PRIMARY KEY NOT NULL, + "code" varchar(32) NOT NULL, + "name" varchar(255) NOT NULL, + "default" boolean DEFAULT false NOT NULL, + "protected" boolean DEFAULT false NOT NULL, + CONSTRAINT "core_languages_code_unique" UNIQUE("code") + ); + CREATE TABLE "core_search_index" ( + "id" serial PRIMARY KEY NOT NULL, + "pluginId" varchar(255) NOT NULL, + "itemType" varchar(100) NOT NULL, + "itemId" integer NOT NULL, + "languageCode" varchar(32) DEFAULT '' NOT NULL, + "authorId" integer, + "title" text NOT NULL, + "content" text NOT NULL, + "containerType" varchar(100), + "containerId" integer, + "url" text, + "isPublic" boolean DEFAULT true NOT NULL, + "metadata" jsonb DEFAULT '{}'::jsonb NOT NULL, + "createdAt" timestamp NOT NULL, + "updatedAt" timestamp, + "indexedAt" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "core_search_index_item_key" + UNIQUE("itemType", "itemId", "languageCode") + ); +`; + +/** Who the editorial suites act as. `userId: null` - see `postgres.test.ts`. */ +export const ACTOR = { type: "staff" as const, userId: null }; + +export interface RecordedSearchDelete { + itemId: number; + itemType: string; + locale?: string; +} + +export interface RecordedEvent { + name: string; + payload: unknown; +} + +/** A listener that did not receive an event, in the shape `emit` reports. */ +export interface RecordedEventFailure { + error: string; + listener: string; + module: string; + pluginId: string; +} + +export interface ContentTestHarness { + /** Injected failures, flipped per test. */ + readonly behaviour: { + /** Listeners `emit` should report as having failed. */ + eventFailures: RecordedEventFailure[]; + /** When set, every `search.index`/`search.delete` throws it. */ + searchError: Error | null; + }; + context: Context; + db: ReturnType; + /** Every `search.delete` the engine asked for, in order. */ + deleted: RecordedSearchDelete[]; + /** Every event the engine emitted, in order. */ + emitted: RecordedEvent[]; + end: () => Promise; + /** Every document the engine handed the search engine, in order. */ + indexed: SearchDocument[]; + /** Every line written through `c.get("log").error`. */ + logs: string[]; + /** Clears the recorders and the injected failures. */ + reset: () => void; + /** + * A second connection with its own context. + * + * The main client is `max: 1`, which serialises everything through one + * backend - fine for optimistic locking, useless for row locks, because a + * statement waiting on `FOR UPDATE` would be waiting on itself. + */ + rivalContext: Context; + rivalDb: ReturnType; + /** The Postgres major, for the assertions whose SQLSTATE moved in 18. */ + serverMajor: number; + sql: ReturnType; +} + +/** + * Builds the schema and returns everything a suite needs to drive it. + * + * **Wipes the database it points at**, so the URL has to name one with "test" + * in it. That check is not politeness: the suite runs `DROP SCHEMA public + * CASCADE`. + */ +export const createContentTestHarness = + async (): Promise => { + if (!DATABASE_TEST_URL) { + throw new Error("DATABASE_TEST_URL is not set."); + } + if (!/test/i.test(databaseName)) { + throw new Error( + `DATABASE_TEST_URL points at "${databaseName || DATABASE_TEST_URL}". This suite wipes the database it runs against, so its name must contain "test".`, + ); + } + + const sql = postgres(DATABASE_TEST_URL, { + max: 1, + onnotice: () => undefined, + }); + + const [{ version }] = await sql<{ version: number }[]>` + SELECT current_setting('server_version_num')::int AS version + `; + const serverMajor = Math.floor(version / 10_000); + + await sql.unsafe(` + DROP SCHEMA IF EXISTS public CASCADE; + CREATE SCHEMA public; + `); + await sql.unsafe(CORE_STUBS); + await sql` + INSERT INTO "core_languages" ("code", "name", "default") VALUES + ('en', 'English', true), + ('pl', 'Polski', false), + ('de', 'Deutsch', false) + `; + + for (const statement of migrationSql(EXAMPLE_MIGRATIONS).split( + "--> statement-breakpoint", + )) { + const trimmed = statement.trim(); + if (trimmed) await sql.unsafe(trimmed); + } + + const db = drizzle(sql, { casing: "camelCase" }); + const rival = postgres(DATABASE_TEST_URL, { + max: 1, + onnotice: () => undefined, + }); + const rivalDb = drizzle(rival, { casing: "camelCase" }); + + const indexed: SearchDocument[] = []; + const deleted: RecordedSearchDelete[] = []; + const emitted: RecordedEvent[] = []; + const logs: string[] = []; + const behaviour: ContentTestHarness["behaviour"] = { + eventFailures: [], + searchError: null, + }; + + /** + * Everything the Content Engine reads off a request context. + * + * The queue stands in for `QueueModel.dispatch`, writing the row it would + * and honouring the `tx` it is handed - which is the property the schedule + * tests are about. The search engine and the event transport record rather + * than deliver, and both can be made to fail on demand: that is what makes + * "a committed write survives a downstream outage" testable at all. + */ + const buildContext = (handle: typeof db): Context => + ({ + get: (key: string) => { + if (key === "db") return handle; + if (key === "search") { + return { + delete: async ( + itemType: string, + itemId: number, + locale?: string, + ) => { + if (behaviour.searchError) throw behaviour.searchError; + deleted.push({ itemId, itemType, locale }); + + return await Promise.resolve(); + }, + index: async (document: SearchDocument) => { + if (behaviour.searchError) throw behaviour.searchError; + indexed.push(document); + + return await Promise.resolve(); + }, + }; + } + if (key === "events") { + return { + emit: async (name: string, payload: unknown) => { + emitted.push({ name, payload }); + + return await Promise.resolve({ + delivered: behaviour.eventFailures.length === 0 ? 1 : 0, + eventId: `event-${emitted.length}`, + failures: [...behaviour.eventFailures], + status: "delivered" as const, + }); + }, + }; + } + if (key === "log") { + return { + error: async (message: string) => { + logs.push(message); + + return await Promise.resolve(); + }, + }; + } + if (key === "core") { + return { + contentModels: [ + { model: articleContent, pluginId: CONFIG_PLUGIN.pluginId }, + { model: categoryContent, pluginId: CONFIG_PLUGIN.pluginId }, + { + model: localizedArticleContent, + pluginId: CONFIG_PLUGIN.pluginId, + }, + { + model: advancedArticleContent, + pluginId: CONFIG_PLUGIN.pluginId, + }, + ], + // Which locales this app *serves*. `core_languages` is the + // registry of the ones that exist; a locale listed here with + // `enabled: false` is a deliberate switch-off. + i18n: { + locales: [ + { code: "en", name: "English" }, + { code: "pl", name: "Polski" }, + { code: "de", enabled: false, name: "Deutsch" }, + ], + }, + searchIndexers: [], + }; + } + if (key === "queue") { + return { + dispatch: async ({ + availableAt, + name, + payload, + pluginId, + tx, + }: { + availableAt?: Date; + name: string; + payload?: Record; + pluginId?: string; + tx?: typeof db; + }) => { + const [queued] = await (tx ?? handle) + .insert(core_queue) + .values({ + availableAt: availableAt ?? new Date(), + name, + payload: payload ?? {}, + pluginId: pluginId ?? "@vitnode/core", + }) + .returning({ id: core_queue.id }); + + return queued; + }, + }; + } + + return undefined; + }, + }) as unknown as Context; + + return { + behaviour, + context: buildContext(db), + db, + deleted, + emitted, + end: async () => { + await sql.end(); + await rival.end(); + }, + indexed, + logs, + reset: () => { + indexed.length = 0; + deleted.length = 0; + emitted.length = 0; + logs.length = 0; + behaviour.eventFailures = []; + behaviour.searchError = null; + }, + rivalContext: buildContext(rivalDb), + rivalDb, + serverMajor, + sql, + }; + }; + +/** + * Empties every table the suites write to, in dependency order. + * + * `DELETE` rather than `TRUNCATE ... CASCADE`: the cascade would silently prove + * nothing about the foreign keys, and several suites are specifically about what + * the database refuses. + */ +export const clearContentTables = async ( + sql: ReturnType, +): Promise => { + await sql`DELETE FROM "core_search_index"`; + await sql`DELETE FROM "core_content_schedules"`; + await sql`DELETE FROM "core_content_revisions"`; + await sql`DELETE FROM "core_queue"`; + await sql`DELETE FROM "example_advanced_articles"`; + await sql`DELETE FROM "example_localized_articles"`; + await sql`DELETE FROM "example_articles"`; + await sql`DELETE FROM "example_categories"`; + await sql`DELETE FROM "core_users"`; +}; + +/** The SQLSTATE a failing call reported, or `undefined` if it succeeded. */ +export const pgErrorCode = async ( + run: () => Promise, +): Promise => { + try { + await run(); + } catch (error) { + const cause = (error as { cause?: { code?: string } }).cause; + + return cause?.code ?? (error as { code?: string }).code; + } + + return undefined; +}; + +/** + * Runs two writers at once and reports what each one did. + * + * `Promise.allSettled` rather than `Promise.all`, because the whole point is + * that one of them is expected to lose - and `all` would reject before the + * winner's result could be inspected. + */ +export const race = async ( + first: () => Promise, + second: () => Promise, +): Promise< + [PromiseSettledResult>, PromiseSettledResult>] +> => { + const results = await Promise.allSettled([first(), second()]); + + return results; +}; + +/** How many of a race's two sides succeeded. */ +export const fulfilledCount = ( + results: readonly PromiseSettledResult[], +): number => results.filter(entry => entry.status === "fulfilled").length; + +/** The reasons the losing sides gave. */ +export const reasons = ( + results: readonly PromiseSettledResult[], +): unknown[] => + results.flatMap(entry => (entry.status === "rejected" ? [entry.reason] : [])); From ebd119bb7425bca9d2199f87349c9473d0dbb5c0 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:33:10 +0200 Subject: [PATCH 04/14] test(content): harden downstream failure, idempotency and search consistency What happens to a committed mutation when the systems it has to tell go down. The three fail differently, so they are driven separately and then together: events report `failures`, search throws, and a cache origin refuses the POST. Covered: the write stays committed and readable through all three; the failure is visible on the outcome *and* in the log; a partial cache delivery is a failure rather than a success; every outstanding reason is reported, not just the first; and an effects retry re-announces without ever re-running the transition - same version, same `publishedAt`, one publish revision, and a byte-identical search document. Also: live synchronisation and rebuild produce the same document; a rebuild pages without skipping or repeating; stale documents are removed on unpublish, delete and per-locale translation unpublish; and the new drift diagnostic catches a missing document, a document that outlived its record, and a locale that a rebuild never reached. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/example/src/database/harness.ts | 15 + .../src/database/resilience-postgres.test.ts | 989 ++++++++++++++++++ 2 files changed, 1004 insertions(+) create mode 100644 plugins/example/src/database/resilience-postgres.test.ts diff --git a/plugins/example/src/database/harness.ts b/plugins/example/src/database/harness.ts index 50eb05f29..d94b3fb2b 100644 --- a/plugins/example/src/database/harness.ts +++ b/plugins/example/src/database/harness.ts @@ -141,6 +141,14 @@ export interface ContentTestHarness { readonly behaviour: { /** Listeners `emit` should report as having failed. */ eventFailures: RecordedEventFailure[]; + /** + * Web origins the revalidation bridge should post to. + * + * Empty by default, which is what an API with no `NEXT_PUBLIC_WEB_URL` sees + * - and what makes `attempted: 0` mean "there was nothing to tell" rather + * than "nobody answered". + */ + revalidateOrigins: string[]; /** When set, every `search.index`/`search.delete` throws it. */ searchError: Error | null; }; @@ -231,6 +239,7 @@ export const createContentTestHarness = const logs: string[] = []; const behaviour: ContentTestHarness["behaviour"] = { eventFailures: [], + revalidateOrigins: [], searchError: null, }; @@ -292,6 +301,11 @@ export const createContentTestHarness = } if (key === "core") { return { + // What the revalidation bridge posts to, and the secret it signs + // with. Both live on the context in a real install too. + contentRevalidateOrigins: behaviour.revalidateOrigins, + cronSecret: "content-engine-test-secret", + hasCronAdapter: false, contentModels: [ { model: articleContent, pluginId: CONFIG_PLUGIN.pluginId }, { model: categoryContent, pluginId: CONFIG_PLUGIN.pluginId }, @@ -369,6 +383,7 @@ export const createContentTestHarness = emitted.length = 0; logs.length = 0; behaviour.eventFailures = []; + behaviour.revalidateOrigins = []; behaviour.searchError = null; }, rivalContext: buildContext(rivalDb), diff --git a/plugins/example/src/database/resilience-postgres.test.ts b/plugins/example/src/database/resilience-postgres.test.ts new file mode 100644 index 000000000..6f0a3730e --- /dev/null +++ b/plugins/example/src/database/resilience-postgres.test.ts @@ -0,0 +1,989 @@ +import type { SearchDocument } from "@vitnode/core/api/models/search"; +import type { Context } from "hono"; + +import { executeContentSchedule } from "@vitnode/core/api/modules/content/helpers/execute-content-schedule"; +import { + contentEditorialEffects, + contentEngineDiagnostics, + contentSearchDrift, + createContentLocalizedSearchIndexer, + createContentSearchIndexer, + runContentScheduleEffects, + syncContentSearch, +} from "@vitnode/core/content/server"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +import { CONFIG_PLUGIN } from "@/const"; +import { articleContentType } from "@/content/article"; + +import type { ContentTestHarness } from "./harness"; + +import { articleContent } from "./articles"; +import { + ACTOR, + clearContentTables, + createContentTestHarness, + DATABASE_TEST_URL, +} from "./harness"; +import { localizedArticleContent } from "./localized-articles"; + +/** + * What happens to a **committed** mutation when the things it has to tell go + * down. + * + * The rule the whole stage rests on: a database write that committed did commit. + * No event transport, search engine or cache origin may undo it, and none of + * them may make the engine report it as having failed. What they *may* do is + * leave the announcement outstanding - and Stage 7's job is to make that + * outstanding state visible and repairable rather than silent. + * + * The three downstream systems fail in different ways, so they are tested + * separately and then together: + * + * | System | Fails by | Repaired by | + * | -------- | ------------------------------ | ------------------------------ | + * | events | reporting `failures` | nothing - at-least-once | + * | search | throwing from `index`/`delete` | the next write, or a rebuild | + * | cache | an origin refusing the POST | the effects task's own retry | + */ + +let h: ContentTestHarness; +let categoryId = 0; +let seq = 0; + +const editorial = (on: Context) => { + const build = articleContent.editorialService; + if (!build) throw new Error("example.article has no editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const localizedService = (on: Context) => { + const build = localizedArticleContent.localizedService; + if (!build) throw new Error("no localized service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const translationEditorial = (on: Context) => { + const build = localizedArticleContent.translationEditorialService; + if (!build) throw new Error("no translation editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const article = async () => { + seq += 1; + const outcome = await editorial(h.context).create( + { + category: categoryId, + code: `resilient-${seq}`, + title: `Resilient subject ${seq}`, + }, + { actor: ACTOR }, + ); + + return { id: outcome.row.id, version: outcome.version }; +}; + +const rowOf = async (id: number) => { + const [row] = await h.sql< + { publishedAt: Date | null; status: string; version: number }[] + >` + SELECT "status", "publishedAt", "version" FROM "example_articles" + WHERE "id" = ${id} + `; + + return row; +}; + +/** A published article, ready for the index. */ +const published = async () => { + const created = await article(); + const outcome = await editorial(h.context).publish(created.id, { + actor: ACTOR, + }); + + return { id: created.id, version: outcome?.version ?? created.version }; +}; + +const DEAD_LISTENER = { + error: "Service unavailable", + listener: "send-notification", + module: "notifications", + pluginId: CONFIG_PLUGIN.pluginId, +}; + +describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { + beforeAll(async () => { + h = await createContentTestHarness(); + }, 60_000); + + afterAll(async () => { + await h?.end(); + vi.unstubAllGlobals(); + }); + + beforeEach(async () => { + await clearContentTables(h.sql); + h.reset(); + // A web origin that accepts everything, by default. `originsFor` falls back + // to `NEXT_PUBLIC_WEB_URL` when none is configured, so without this the + // bridge would try to reach a real host and every scheduled run would fail + // on the cache rather than on the thing under test. + vi.stubGlobal( + "fetch", + vi.fn( + async () => await Promise.resolve(new Response("ok", { status: 200 })), + ), + ); + + const [category] = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") VALUES ('Resilience') + RETURNING "id" + `; + categoryId = category.id; + }); + + // ------------------------------------------------------------------------- + // Events + // ------------------------------------------------------------------------- + + describe("a dead event listener", () => { + it("leaves the write committed and reports the failure", async () => { + h.behaviour.eventFailures = [DEAD_LISTENER]; + const { id, version } = await published(); + + const outcome = await editorial(h.context).update( + id, + { title: "Edited despite the outage" }, + { actor: ACTOR, expectedVersion: version }, + ); + if (!outcome) throw new Error("Expected an outcome."); + + const result = await contentEditorialEffects( + h.context, + articleContentType, + outcome, + { model: articleContent, pluginId: CONFIG_PLUGIN.pluginId }, + ); + + // Committed, and readable. + expect((await rowOf(id)).version).toBe(version + 1); + // Visible, rather than swallowed. + expect(result.event?.failures).toHaveLength(1); + expect(h.logs.some(line => line.includes("[content-effects]"))).toBe( + true, + ); + expect(h.logs.join("\n")).toContain("send-notification"); + }); + + it("still writes the search document", async () => { + // Two independent systems: one being down is not a reason to skip the + // other, and by the time either runs the row is already committed. + h.behaviour.eventFailures = [DEAD_LISTENER]; + const { id, version } = await published(); + + const outcome = await editorial(h.context).update( + id, + { title: "Still indexed" }, + { actor: ACTOR, expectedVersion: version }, + ); + if (!outcome) throw new Error("Expected an outcome."); + + await contentEditorialEffects(h.context, articleContentType, outcome, { + model: articleContent, + pluginId: CONFIG_PLUGIN.pluginId, + }); + + expect(h.indexed.map(document => document.itemId)).toContain(id); + }); + + it("logs nothing when every listener received it", async () => { + const { id, version } = await published(); + const outcome = await editorial(h.context).update( + id, + { title: "Quiet" }, + { actor: ACTOR, expectedVersion: version }, + ); + if (!outcome) throw new Error("Expected an outcome."); + + await contentEditorialEffects(h.context, articleContentType, outcome, { + model: articleContent, + pluginId: CONFIG_PLUGIN.pluginId, + }); + + expect(h.logs.filter(line => line.includes("[content-effects]"))).toEqual( + [], + ); + }); + }); + + // ------------------------------------------------------------------------- + // Search + // ------------------------------------------------------------------------- + + describe("a search engine that is down", () => { + it("never rolls the database write back", async () => { + const { id, version } = await published(); + h.behaviour.searchError = new Error("elasticsearch unreachable"); + + const outcome = await editorial(h.context).update( + id, + { title: "Written while search was down" }, + { actor: ACTOR, expectedVersion: version }, + ); + + expect(outcome?.changed).toBe(true); + const [row] = await h.sql<{ title: string }[]>` + SELECT "title" FROM "example_articles" WHERE "id" = ${id} + `; + expect(row.title).toBe("Written while search was down"); + }); + + it("reports the failure on the outcome and in the log", async () => { + const { id } = await published(); + h.behaviour.searchError = new Error("elasticsearch unreachable"); + + const result = await syncContentSearch(h.context, articleContentType, { + changedFields: ["title"], + operation: "update", + pluginId: CONFIG_PLUGIN.pluginId, + row: { ...(await rowOf(id)), id, slug: "x", title: "T" }, + }); + + expect(result.error?.message).toBe("elasticsearch unreachable"); + expect(h.logs.some(line => line.includes("[content-search]"))).toBe(true); + }); + + it("is repaired by the next successful write", async () => { + // "Eventually consistent, bounded by the next publish or the next + // rebuild" - the first half of that, shown. + const { id, version } = await published(); + h.behaviour.searchError = new Error("down"); + + const first = await editorial(h.context).update( + id, + { title: "Lost to the outage" }, + { actor: ACTOR, expectedVersion: version }, + ); + await syncContentSearch(h.context, articleContentType, { + changedFields: first?.changedFields, + operation: "update", + pluginId: CONFIG_PLUGIN.pluginId, + row: first?.row ?? {}, + }); + expect(h.indexed).toHaveLength(0); + + h.behaviour.searchError = null; + const second = await editorial(h.context).update( + id, + { title: "Recovered" }, + { actor: ACTOR, expectedVersion: first?.version ?? version }, + ); + await syncContentSearch(h.context, articleContentType, { + changedFields: second?.changedFields, + operation: "update", + pluginId: CONFIG_PLUGIN.pluginId, + row: second?.row ?? {}, + }); + + expect(h.indexed.at(-1)?.title).toBe("Recovered"); + }); + }); + + // ------------------------------------------------------------------------- + // Scheduled effects, where all three meet + // ------------------------------------------------------------------------- + + describe("scheduled effects", () => { + const schedules = (on: Context) => { + const model = editorial(on).schedules; + if (!model) throw new Error("example.article has no scheduling"); + + return model; + }; + + /** Books a publish that is already due, runs it, and returns the payload. */ + const runTransition = async () => { + const created = await article(); + const booked = await schedules(h.context).schedule({ + action: "publish", + actorUserId: null, + itemId: created.id, + scheduledFor: new Date(Date.now() - 1000), + }); + + await executeContentSchedule(h.context, { + generation: booked.generation, + scheduleId: booked.id, + }); + + const [queued] = await h.sql<{ payload: Record }[]>` + SELECT "payload" FROM "core_queue" + WHERE "name" = 'content-schedule-effects' + ORDER BY "id" DESC LIMIT 1 + `; + + return { + id: created.id, + payload: queued.payload as Parameters< + typeof runContentScheduleEffects + >[1], + scheduleId: booked.id, + }; + }; + + const effectsErrorOf = async (scheduleId: number) => { + const [row] = await h.sql<{ effectsError: null | string }[]>` + SELECT "effectsError" FROM "core_content_schedules" + WHERE "id" = ${scheduleId} + `; + + return row.effectsError; + }; + + it("delivers everything on a healthy run and records no error", async () => { + const { payload, scheduleId } = await runTransition(); + + const outcome = await runContentScheduleEffects(h.context, payload); + + expect(outcome.status).toBe("delivered"); + expect(await effectsErrorOf(scheduleId)).toBeNull(); + expect(h.emitted.map(entry => entry.name)).toContain( + "content.example.article.published", + ); + }); + + it("fails the run and records why when the event transport reports a failure", async () => { + const { payload, scheduleId } = await runTransition(); + h.behaviour.eventFailures = [DEAD_LISTENER]; + + await expect( + runContentScheduleEffects(h.context, payload), + ).rejects.toThrow(/committed, but its effects did not/); + + const error = await effectsErrorOf(scheduleId); + expect(error).toContain("event:"); + expect(error).toContain("send-notification"); + }); + + it("fails the run when the search write is refused", async () => { + const { payload, scheduleId } = await runTransition(); + h.behaviour.searchError = new Error("index refused"); + + await expect( + runContentScheduleEffects(h.context, payload), + ).rejects.toThrow(); + + expect(await effectsErrorOf(scheduleId)).toContain("search:"); + }); + + it("reports every outstanding failure, not just the first", async () => { + // The whole point of combining them: an operator looking at one line has + // to see everything that is still outstanding, or they will fix one + // system, retry, and discover the next. + const { payload, scheduleId } = await runTransition(); + h.behaviour.eventFailures = [DEAD_LISTENER]; + h.behaviour.searchError = new Error("index refused"); + h.behaviour.revalidateOrigins = ["http://web-a.invalid"]; + vi.stubGlobal( + "fetch", + vi.fn( + async () => + await Promise.resolve(new Response("no", { status: 500 })), + ), + ); + + await expect( + runContentScheduleEffects(h.context, payload), + ).rejects.toThrow(); + + const error = await effectsErrorOf(scheduleId); + expect(error).toContain("event:"); + expect(error).toContain("search:"); + expect(error).toContain("cache:"); + }); + + it("treats a partial cache delivery as a failure, not a success", async () => { + // Two web apps behind one API: one of them accepting an unpublish while + // the other does not leaves the withdrawn page cached and readable. + const { payload, scheduleId } = await runTransition(); + h.behaviour.revalidateOrigins = [ + "http://web-a.invalid", + "http://web-b.invalid", + ]; + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL) => { + const url = input instanceof URL ? input.href : input; + + return await Promise.resolve( + url.includes("web-a") + ? new Response("ok", { status: 200 }) + : new Response("no", { status: 500 }), + ); + }), + ); + + await expect( + runContentScheduleEffects(h.context, payload), + ).rejects.toThrow(); + + expect(await effectsErrorOf(scheduleId)).toContain( + "1/2 web origins accepted", + ); + }); + + it("never re-runs the transition when the effects are retried", async () => { + const { id, payload, scheduleId } = await runTransition(); + const before = await rowOf(id); + + h.behaviour.searchError = new Error("index refused"); + await expect( + runContentScheduleEffects(h.context, payload), + ).rejects.toThrow(); + + h.behaviour.searchError = null; + const retried = await runContentScheduleEffects(h.context, payload); + + expect(retried.status).toBe("delivered"); + // Same version, same publication timestamp: the retry announced the + // transition again, it did not perform it again. + expect(await rowOf(id)).toEqual(before); + expect(await effectsErrorOf(scheduleId)).toBeNull(); + + const revisions = await h.sql<{ id: number }[]>` + SELECT "id" FROM "core_content_revisions" + WHERE "itemId" = ${id} AND "operation" = 'publish' + `; + expect(revisions).toHaveLength(1); + }); + + it("re-emits the event on a retry, which is why delivery is at-least-once", async () => { + const { payload } = await runTransition(); + + h.behaviour.searchError = new Error("index refused"); + await expect( + runContentScheduleEffects(h.context, payload), + ).rejects.toThrow(); + + h.behaviour.searchError = null; + await runContentScheduleEffects(h.context, payload); + + const published = h.emitted.filter( + entry => entry.name === "content.example.article.published", + ); + // Twice, with the same `scheduleId` both times - which is the key a + // listener that must act once uses. There is no outbox and no + // exactly-once claim. + expect(published).toHaveLength(2); + expect( + published.map( + entry => (entry.payload as { scheduleId: number }).scheduleId, + ), + ).toEqual([payload.scheduleId, payload.scheduleId]); + }); + + it("indexes the same document on a retry, so the repeat is harmless", async () => { + const { payload } = await runTransition(); + + h.behaviour.eventFailures = [DEAD_LISTENER]; + await expect( + runContentScheduleEffects(h.context, payload), + ).rejects.toThrow(); + const first = h.indexed.at(-1); + + h.behaviour.eventFailures = []; + await runContentScheduleEffects(h.context, payload); + const second = h.indexed.at(-1); + + // An upsert is the same operation however many times it runs, and the + // document it writes is byte-identical. + expect(second).toEqual(first); + }); + + it("gives up rather than retrying forever when the content type is gone", async () => { + const { payload, scheduleId } = await runTransition(); + + const outcome = await runContentScheduleEffects(h.context, { + ...payload, + contentTypeId: "example.removed-by-an-uninstall", + }); + + expect(outcome.status).toBe("unregistered"); + expect(await effectsErrorOf(scheduleId)).toContain( + "no longer registered", + ); + }); + }); + + // ------------------------------------------------------------------------- + // Idempotency + // ------------------------------------------------------------------------- + + describe("idempotency", () => { + it("makes a second publish a no-op with no revision and no event", async () => { + const { id } = await published(); + h.reset(); + + const outcome = await editorial(h.context).publish(id, { actor: ACTOR }); + await contentEditorialEffects( + h.context, + articleContentType, + outcome ?? ({} as never), + { model: articleContent, pluginId: CONFIG_PLUGIN.pluginId }, + ); + + expect(outcome?.changed).toBe(false); + expect(outcome?.revisionId).toBeNull(); + expect(h.emitted).toEqual([]); + expect(h.indexed).toEqual([]); + }); + + it("makes a second unpublish a no-op", async () => { + const { id } = await published(); + await editorial(h.context).unpublish(id, { actor: ACTOR }); + const before = await rowOf(id); + h.reset(); + + const outcome = await editorial(h.context).unpublish(id, { + actor: ACTOR, + }); + + expect(outcome?.changed).toBe(false); + expect(await rowOf(id)).toEqual(before); + expect(h.emitted).toEqual([]); + }); + + it("makes a restore to the values already stored a no-op", async () => { + const created = await article(); + const history = await editorial(h.context).revisions.list(created.id); + const only = history.edges[0]; + + const outcome = await editorial(h.context).restore(created.id, only.id, { + actor: ACTOR, + expectedVersion: created.version, + }); + + expect(outcome?.changed).toBe(false); + expect(outcome?.revisionId).toBeNull(); + expect((await rowOf(created.id)).version).toBe(created.version); + }); + + it("bumps no version for a relation add that is already there", async () => { + const created = await article(); + + const outcome = await editorial(h.context).update( + created.id, + { title: `Resilient subject ${seq}` }, + { actor: ACTOR, expectedVersion: created.version }, + ); + + expect(outcome?.changed).toBe(false); + expect((await rowOf(created.id)).version).toBe(created.version); + }); + }); + + // ------------------------------------------------------------------------- + // Search consistency + // ------------------------------------------------------------------------- + + describe("live synchronisation and rebuild agree", () => { + const indexer = () => + createContentSearchIndexer(articleContent, { + pluginId: CONFIG_PLUGIN.pluginId, + }); + + const rebuild = async (limit = 50): Promise => { + const documents: SearchDocument[] = []; + const build = indexer(); + for (let offset = 0; ;) { + const page = await build.load(h.context, offset, limit); + if (page.itemsRead === 0) break; + documents.push(...page.documents); + offset += page.itemsRead; + } + + return documents; + }; + + it("reproduces the live document byte for byte", async () => { + const created = await article(); + const outcome = await editorial(h.context).publish(created.id, { + actor: ACTOR, + }); + // The live path is the effects layer, not the transition: publishing + // writes the row, and the announcement writes the document. + await contentEditorialEffects( + h.context, + articleContentType, + outcome ?? ({} as never), + { model: articleContent, pluginId: CONFIG_PLUGIN.pluginId }, + ); + const id = created.id; + const live = h.indexed.find(document => document.itemId === id); + expect(live).toBeDefined(); + + // The live path indexes on publish; the rebuild reads the same row + // through a different query. Equality is the invariant. + const rebuilt = (await rebuild()).find( + document => document.itemId === id, + ); + + expect(rebuilt).toEqual(live); + }); + + it("pages a rebuild without skipping or repeating a record", async () => { + const ids: number[] = []; + for (let index = 0; index < 7; index += 1) { + const { id } = await published(); + ids.push(id); + } + + const documents = await rebuild(2); + + const ascending = (a: number, b: number) => a - b; + expect( + documents.map(document => document.itemId).sort(ascending), + ).toEqual([...ids].sort(ascending)); + expect(new Set(documents.map(document => document.itemId)).size).toBe( + ids.length, + ); + }); + + it("counts exactly the records it would index", async () => { + await published(); + await published(); + await article(); // a draft, which is never indexed + + expect(await indexer().count?.(h.context)).toBe(2); + expect(await rebuild()).toHaveLength(2); + }); + }); + + describe("stale documents are cleaned up", () => { + it("removes a record's document when it is unpublished", async () => { + const { id, version } = await published(); + h.reset(); + + const outcome = await editorial(h.context).unpublish(id, { + actor: ACTOR, + expectedVersion: version, + }); + await contentEditorialEffects( + h.context, + articleContentType, + outcome ?? ({} as never), + { model: articleContent, pluginId: CONFIG_PLUGIN.pluginId }, + ); + + expect(h.deleted).toContainEqual({ + itemId: id, + itemType: articleContentType.id, + locale: undefined, + }); + }); + + it("removes it when the record is deleted", async () => { + const { id, version } = await published(); + h.reset(); + + const outcome = await editorial(h.context).delete(id, { + actor: ACTOR, + expectedVersion: version, + }); + await contentEditorialEffects( + h.context, + articleContentType, + outcome ?? ({} as never), + { model: articleContent, pluginId: CONFIG_PLUGIN.pluginId }, + ); + + expect(h.deleted.map(entry => entry.itemId)).toContain(id); + }); + + it("removes only the language a translation was taken down in", async () => { + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: "English body", title: "Stale Cleanup" }, + }, + { actor: ACTOR }, + ); + await translationEditorial(h.context).create( + row.id, + "pl", + { body: "Tresc", title: "Polski" }, + { actor: ACTOR }, + ); + const base = localizedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + row.id, + { actor: ACTOR }, + ); + await translationEditorial(h.context).publish(row.id, "en", { + actor: ACTOR, + }); + await translationEditorial(h.context).publish(row.id, "pl", { + actor: ACTOR, + }); + h.reset(); + + const outcome = await translationEditorial(h.context).unpublish( + row.id, + "pl", + { actor: ACTOR }, + ); + const { contentTranslationEffects } = + await import("@vitnode/core/content/server"); + await contentTranslationEffects( + h.context, + localizedArticleContent.definition, + outcome ?? ({} as never), + { + model: localizedArticleContent, + pluginId: CONFIG_PLUGIN.pluginId, + }, + ); + + // One language out, the other left exactly where it was. + expect(h.deleted).toEqual([ + { + itemId: row.id, + itemType: localizedArticleContent.definition.id, + locale: "pl", + }, + ]); + }); + }); + + // ------------------------------------------------------------------------- + // Drift diagnostics + // ------------------------------------------------------------------------- + + describe("index drift is diagnosable", () => { + /** Writes the canonical index rows a healthy install would hold. */ + const indexPublished = async () => { + const rows = await h.sql<{ id: number; title: string }[]>` + SELECT "id", "title" FROM "example_articles" + WHERE "status" = 'published' AND "publishedAt" IS NOT NULL + `; + for (const row of rows) { + await h.sql` + INSERT INTO "core_search_index" + ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${articleContentType.id}, ${row.id}, + '', ${row.title}, ${row.title}, now() + ) + `; + } + }; + + it("reports a healthy index as healthy", async () => { + await published(); + await published(); + await indexPublished(); + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift).toMatchObject({ + contentTypeId: articleContentType.id, + healthy: true, + }); + expect(drift.locales).toEqual([ + { expected: 2, healthy: true, indexed: 2, locale: "" }, + ]); + }); + + it("reports a document the index never received", async () => { + await published(); + await published(); + await indexPublished(); + // A live sync that threw, simulated at the row level. + await h.sql`DELETE FROM "core_search_index" WHERE "id" = ( + SELECT MIN("id") FROM "core_search_index" + )`; + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.healthy).toBe(false); + expect(drift.locales[0]).toMatchObject({ expected: 2, indexed: 1 }); + }); + + it("reports a document that outlived its record", async () => { + await published(); + await indexPublished(); + await h.sql` + INSERT INTO "core_search_index" + ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${articleContentType.id}, 999999, + '', 'Ghost', 'Ghost', now() + ) + `; + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + // More documents than records is drift in the other direction, and it is + // reported as measured rather than clamped - a stale document is exactly + // what an operator needs to see. + expect(drift.healthy).toBe(false); + expect(drift.locales[0]).toMatchObject({ expected: 1, indexed: 2 }); + }); + + it("counts a localized content type per locale", async () => { + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: "English body", title: "Drift Subject" }, + }, + { actor: ACTOR }, + ); + const base = localizedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + await translationEditorial(h.context).create( + row.id, + "pl", + { body: "Tresc", title: "Polski Drift" }, + { actor: ACTOR }, + ); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + row.id, + { actor: ACTOR }, + ); + await translationEditorial(h.context).publish(row.id, "en", { + actor: ACTOR, + }); + await translationEditorial(h.context).publish(row.id, "pl", { + actor: ACTOR, + }); + + // Only English made it into the index. + await h.sql` + INSERT INTO "core_search_index" + ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${localizedArticleContent.definition.id}, + ${row.id}, 'en', 'Drift Subject', 'Drift Subject', now() + ) + `; + + const drift = await contentSearchDrift(h.context, { + model: localizedArticleContent, + }); + + expect(drift.healthy).toBe(false); + expect(drift.locales).toEqual([ + { expected: 1, healthy: true, indexed: 1, locale: "en" }, + { expected: 1, healthy: false, indexed: 0, locale: "pl" }, + ]); + }); + + it("agrees with the localized rebuild about how many documents there should be", async () => { + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: "English body", title: "Parity Subject" }, + }, + { actor: ACTOR }, + ); + const base = localizedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + await translationEditorial(h.context).create( + row.id, + "pl", + { body: "Tresc", title: "Polski Parity" }, + { actor: ACTOR }, + ); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + row.id, + { actor: ACTOR }, + ); + await translationEditorial(h.context).publish(row.id, "en", { + actor: ACTOR, + }); + await translationEditorial(h.context).publish(row.id, "pl", { + actor: ACTOR, + }); + + const build = createContentLocalizedSearchIndexer( + localizedArticleContent, + { pluginId: CONFIG_PLUGIN.pluginId }, + ); + const drift = await contentSearchDrift(h.context, { + model: localizedArticleContent, + }); + + // The diagnostic and the indexer have to agree about "published", or the + // health check would be measuring something the rebuild does not produce. + expect( + drift.locales.reduce((sum, entry) => sum + entry.expected, 0), + ).toBe(await build.count?.(h.context)); + }); + + it("summarises every registered content type, with schedule failures", async () => { + const { id } = await published(); + const model = editorial(h.context).schedules; + if (!model) throw new Error("no scheduling"); + const booked = await model.schedule({ + action: "unpublish", + actorUserId: null, + itemId: id, + scheduledFor: new Date(Date.now() + 3_600_000), + }); + await h.sql` + UPDATE "core_content_schedules" + SET "effectsError" = 'search: down' + WHERE "id" = ${booked.id} + `; + + const report = await contentEngineDiagnostics(h.context); + const entry = report.contentTypes.find( + item => item.contentTypeId === articleContentType.id, + ); + + expect(report.contentTypes.map(item => item.contentTypeId)).toEqual([ + "example.advanced-article", + "example.article", + "example.category", + "example.localized-article", + ]); + expect(entry?.features).toMatchObject({ + editorial: true, + localization: false, + publicApi: true, + scheduling: true, + search: true, + }); + expect(entry?.schedules).toEqual({ + failedEffects: 1, + pending: 1, + withErrors: 0, + }); + // A content type with no search indexes nothing, so it has no drift to + // report rather than a drift of zero. + expect( + report.contentTypes.find( + item => item.contentTypeId === "example.category", + )?.search, + ).toBeNull(); + }); + }); +}); From 6a43bbf5f17312271850f6fcd0d9bbedceb147c2 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:40:35 +0200 Subject: [PATCH 05/14] fix(content): make cursor pagination's ordering total `withPagination` orders by the requested column and compares the cursor against the primary key. When the order column has ties - a bulk import, a migration backfill, a batch publish all share an `updatedAt` - the ordering is only partial, so Postgres is free to place tied rows differently between the two queries a page turn issues. The cursor is the identifier, so an arbitrary position means a row can be skipped or returned twice for no reason except that the page boundary fell inside a tie. A 103-row admin list paged 25 at a time returned 102 of them. Appending the identifier as a tiebreaker, in the same direction the cursor compares in, makes the ordering total and the two agree. It costs nothing when the order column is already unique, and it is skipped entirely when the order column *is* the identifier. What it does not fix is stated in the tests and the docs: the cursor is the identifier, so ordering by a column with no relationship to it cannot page exactly. Ordering by the identifier, or by a timestamp that moves with it - which is every generated default - is exact. Also adds the scale suite: pagination boundaries, query-count guards that hold across page sizes, batch loading for advanced collections and shared repeatables, index definitions, and EXPLAIN plans over a synthetic table large enough for the planner to have a choice. Co-Authored-By: Claude Opus 5 (1M context) --- .../vitnode/src/api/lib/with-pagination.ts | 28 +- plugins/example/src/database/harness.ts | 35 + .../src/database/performance-postgres.test.ts | 771 ++++++++++++++++++ 3 files changed, 832 insertions(+), 2 deletions(-) create mode 100644 plugins/example/src/database/performance-postgres.test.ts diff --git a/packages/vitnode/src/api/lib/with-pagination.ts b/packages/vitnode/src/api/lib/with-pagination.ts index 5a0964d8f..f81520ef4 100644 --- a/packages/vitnode/src/api/lib/with-pagination.ts +++ b/packages/vitnode/src/api/lib/with-pagination.ts @@ -8,7 +8,7 @@ import type { import type { Context } from "hono"; import { z } from "@hono/zod-openapi"; -import { and, asc, count, desc, gt, ilike, lt, or } from "drizzle-orm"; +import { and, asc, count, desc, gt, ilike, lt, or, sql } from "drizzle-orm"; function parsePaginationParams(params: { query: { cursor?: string; first?: string; last?: string }; @@ -144,7 +144,31 @@ export async function withPagination< const isForward = last === undefined; const orderFn = getOrderFn(isForward, orderByFromParams.order); - const orderBy: SQL = orderFn(table[orderByFromParams.column.name]); + const primary = table[primaryCursor.name]; + const ordered = orderFn(table[orderByFromParams.column.name]); + /** + * The primary key, appended as a tiebreaker. + * + * Without it the ordering is only a partial one: every row that shares an + * `updatedAt` - a bulk import, a migration backfill, a batch publish - sits in + * an arbitrary position that Postgres is free to change between the two + * queries a page turn issues. The cursor is the identifier, so an arbitrary + * position means a row can be skipped or returned twice for no reason except + * that the page boundary fell inside a tie. + * + * Appending the identifier makes the ordering total, in the same direction the + * cursor compares in, so the two agree. It costs nothing when the order column + * is already unique. + * + * What it does **not** fix, and what the pagination docs state plainly: the + * cursor is the identifier, so ordering by a column with no relationship to it + * - a title, say - still cannot page exactly. Ordering by the identifier, or + * by a timestamp that moves with it, is exact. + */ + const orderBy: SQL = + orderByFromParams.column.name === primaryCursor.name + ? ordered + : sql`${ordered}, ${orderFn(primary)}`; const searchWhere = buildSearchWhere(search, params.query.search); const baseWhere = diff --git a/plugins/example/src/database/harness.ts b/plugins/example/src/database/harness.ts index d94b3fb2b..e8ec8bbef 100644 --- a/plugins/example/src/database/harness.ts +++ b/plugins/example/src/database/harness.ts @@ -153,6 +153,22 @@ export interface ContentTestHarness { searchError: Error | null; }; context: Context; + /** + * A third connection that records every statement it issues. + * + * Query *counting* is the only way to state an N+1 guard as an invariant + * rather than as a hope: "one page costs a bounded number of round trips + * whatever the page size" is a fact about the SQL, and the SQL is the only + * place to observe it. Separate from the main handle so an ordinary test pays + * nothing for the instrumentation. + */ + counted: { + context: Context; + db: ReturnType; + /** Every statement since the last `reset`, in order. */ + queries: string[]; + reset: () => void; + }; db: ReturnType; /** Every `search.delete` the engine asked for, in order. */ deleted: RecordedSearchDelete[]; @@ -233,6 +249,16 @@ export const createContentTestHarness = }); const rivalDb = drizzle(rival, { casing: "camelCase" }); + const queries: string[] = []; + const countedSql = postgres(DATABASE_TEST_URL, { + debug: (_connection, query) => { + queries.push(query); + }, + max: 1, + onnotice: () => undefined, + }); + const countedDb = drizzle(countedSql, { casing: "camelCase" }); + const indexed: SearchDocument[] = []; const deleted: RecordedSearchDelete[] = []; const emitted: RecordedEvent[] = []; @@ -368,12 +394,21 @@ export const createContentTestHarness = return { behaviour, context: buildContext(db), + counted: { + context: buildContext(countedDb), + db: countedDb, + queries, + reset: () => { + queries.length = 0; + }, + }, db, deleted, emitted, end: async () => { await sql.end(); await rival.end(); + await countedSql.end(); }, indexed, logs, diff --git a/plugins/example/src/database/performance-postgres.test.ts b/plugins/example/src/database/performance-postgres.test.ts new file mode 100644 index 000000000..c2884ed86 --- /dev/null +++ b/plugins/example/src/database/performance-postgres.test.ts @@ -0,0 +1,771 @@ +import type { SearchDocument } from "@vitnode/core/api/models/search"; +import type { Context } from "hono"; + +import { + createContentLocalizedSearchIndexer, + createContentSearchIndexer, +} from "@vitnode/core/content/server"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import { CONFIG_PLUGIN } from "@/const"; + +import type { ContentTestHarness } from "./harness"; + +import { advancedArticleContent } from "./advanced-articles"; +import { articleContent } from "./articles"; +import { + ACTOR, + clearContentTables, + createContentTestHarness, + DATABASE_TEST_URL, +} from "./harness"; +import { localizedArticleContent } from "./localized-articles"; + +/** + * Pagination, query counts and index use, at a scale that can tell them apart. + * + * Nothing here measures milliseconds. A wall-clock number in CI says more about + * the machine than about the code, and it fails on a busy runner for reasons + * nobody can act on. What is measured instead is **algorithmic**: how many + * round trips one page costs, whether that number moves when the page grows, + * and whether a lookup seeks on an index or reads the whole table. + * + * The dataset is deliberately modest - a few thousand rows rather than the ten + * thousand the plan suggests - because the properties under test are visible at + * any size above "a handful", and a suite nobody waits for is a suite nobody + * runs. Where scale genuinely matters (a sequential scan is cheaper than an + * index on a tiny table, so the planner picks it) the fixture is grown until + * the planner has a real choice to make. + */ + +const PAGE = 25; +/** Enough rows that the planner prefers an index over a sequential scan. */ +const SCALE = 2_000; + +let h: ContentTestHarness; +let categoryId = 0; + +const editorial = (on: Context) => { + const build = articleContent.editorialService; + if (!build) throw new Error("example.article has no editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const localizedService = (on: Context) => { + const build = localizedArticleContent.localizedService; + if (!build) throw new Error("no localized service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const translationEditorial = (on: Context) => { + const build = localizedArticleContent.translationEditorialService; + if (!build) throw new Error("no translation editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +/** + * Bulk-inserts published articles straight through SQL. + * + * The service would write one row per statement and one revision alongside it, + * which at this scale is minutes rather than seconds - and none of these tests + * are about the write path. + */ +let seeded = 0; + +const seedArticles = async (count: number): Promise => { + const from = seeded + 1; + seeded += count; + await h.sql` + INSERT INTO "example_articles" + ("title", "slug", "code", "category", "status", "publishedAt", "version") + SELECT + 'Article ' || i, + 'article-' || i, + 'code-' || i, + ${categoryId}, + 'published', + -- Ascending with the identifier, which is what a real collection looks + -- like: rows are published roughly in the order they were created. The + -- cursor is the identifier, so an order column that moves against it + -- cannot page exactly - see the pagination docs. + now() - ((100000 - i) || ' seconds')::interval, + 1 + FROM generate_series(${from}::int, ${seeded}::int) AS i + `; + await h.sql`ANALYZE "example_articles"`; +}; + +const plan = async (query: string): Promise => { + const rows = await h.sql.unsafe(`EXPLAIN ${query}`); + + return rows.map(row => String(row["QUERY PLAN"])).join("\n"); +}; + +const indexesOn = async (table: string) => + await h.sql<{ indexdef: string; indexname: string }[]>` + SELECT indexname, indexdef FROM pg_indexes WHERE tablename = ${table} + ORDER BY indexname + `; + +/** + * Statements the counted connection issued while `run` was in flight. + * + * The call is made **twice** and only the second is counted. `postgres` + * prepares a statement the first time it sees its shape and reuses it + * afterwards, so a cold call and a warm one legitimately issue different + * numbers of protocol messages - and comparing a cold count against a warm one + * would report a difference that has nothing to do with the query plan. + */ +const countQueries = async (run: () => Promise): Promise => { + await run(); + h.counted.reset(); + await run(); + + return [...h.counted.queries]; +}; + +describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { + beforeAll(async () => { + h = await createContentTestHarness(); + }, 60_000); + + afterAll(async () => { + await h?.end(); + }); + + beforeEach(async () => { + await clearContentTables(h.sql); + h.reset(); + seeded = 0; + + const [category] = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") VALUES ('Scale') RETURNING "id" + `; + categoryId = category.id; + }); + + // ------------------------------------------------------------------------- + // Cursor pagination + // ------------------------------------------------------------------------- + + describe("cursor pagination", () => { + it("answers an empty collection without a cursor", async () => { + const page = await articleContent.service(h.context).findMany(); + + expect(page.edges).toEqual([]); + expect(page.pageInfo).toMatchObject({ + endCursor: null, + hasNextPage: false, + startCursor: null, + totalCount: 0, + }); + }); + + it("answers a single row without offering a next page", async () => { + await seedArticles(1); + + const page = await articleContent.service(h.context).findMany(); + + expect(page.edges).toHaveLength(1); + expect(page.pageInfo.hasNextPage).toBe(false); + }); + + it("stops exactly at the page boundary", async () => { + // The off-by-one that matters: with exactly `first` rows there is no next + // page, and with one more there is. + await seedArticles(PAGE); + const exact = await articleContent + .service(h.context) + .findMany({ query: { first: String(PAGE) } }); + expect(exact.edges).toHaveLength(PAGE); + expect(exact.pageInfo.hasNextPage).toBe(false); + + await seedArticles(1); + const overflowing = await articleContent + .service(h.context) + .findMany({ query: { first: String(PAGE) } }); + expect(overflowing.pageInfo.hasNextPage).toBe(true); + }); + + it("walks every row exactly once across many pages", async () => { + await seedArticles(103); + const seen: number[] = []; + let cursor: string | undefined; + + for (let page = 0; page < 20; page += 1) { + const result = await articleContent.service(h.context).findMany({ + query: { cursor, first: String(PAGE) }, + }); + seen.push(...result.edges.map(row => row.id)); + if (!result.pageInfo.hasNextPage) break; + cursor = String(result.pageInfo.endCursor); + } + + expect(seen).toHaveLength(103); + expect(new Set(seen).size).toBe(103); + }); + + it("never repeats a row because something was inserted between pages", async () => { + // A cursor is a position in an ordering, not a snapshot. Rows that arrive + // behind the cursor are simply not seen; the guarantee is that nothing + // already returned comes back a second time. + await seedArticles(60); + const first = await articleContent + .service(h.context) + .findMany({ query: { first: String(PAGE) } }); + + await seedArticles(10); + + const second = await articleContent.service(h.context).findMany({ + query: { + cursor: String(first.pageInfo.endCursor), + first: String(PAGE), + }, + }); + + const overlap = second.edges + .map(row => row.id) + .filter(id => first.edges.some(row => row.id === id)); + expect(overlap).toEqual([]); + }); + + it("does not loop forever when rows are deleted between pages", async () => { + await seedArticles(60); + const first = await articleContent + .service(h.context) + .findMany({ query: { first: String(PAGE) } }); + + // The list is newest-first, so everything *after* the cursor has a + // smaller identifier than it. + await h.sql` + DELETE FROM "example_articles" + WHERE "id" < ${Number(first.pageInfo.endCursor)} + `; + + const second = await articleContent.service(h.context).findMany({ + query: { + cursor: String(first.pageInfo.endCursor), + first: String(PAGE), + }, + }); + + expect(second.edges).toEqual([]); + expect(second.pageInfo.hasNextPage).toBe(false); + }); + + it("caps a public page however large a caller asks for", async () => { + // An anonymous caller controls `first`, so the ceiling has to be the + // server's rather than theirs. + await seedArticles(200); + const service = articleContent.publicService; + if (!service) throw new Error("no public service"); + + const page = await service(h.context).findMany({ + query: { first: "10000" }, + }); + + expect(page.edges.length).toBeLessThanOrEqual(100); + }); + }); + + // ------------------------------------------------------------------------- + // Query counts + // ------------------------------------------------------------------------- + + describe("query counts stay bounded per page", () => { + /** + * Upper bounds rather than exact numbers. + * + * A planner change, a different Postgres major or a Drizzle release can all + * move the exact count by one without anything being wrong. What must never + * move is the *shape*: a page of 25 and a page of 100 cost the same number + * of round trips, and that is what an N+1 would break. + */ + const boundedAcrossPageSizes = async ( + run: (size: number) => Promise, + bound: number, + ) => { + const small = await countQueries(async () => await run(5)); + const large = await countQueries(async () => await run(60)); + + expect(small.length).toBeLessThanOrEqual(bound); + expect(large.length).toBeLessThanOrEqual(bound); + // The invariant an N+1 breaks: twelve times the rows, the same number of + // statements. + expect(large.length).toBe(small.length); + }; + + it("keeps the admin list bounded, labels included", async () => { + await seedArticles(200); + + await boundedAcrossPageSizes( + async size => + await articleContent + .service(h.counted.context) + .findMany({ query: { first: String(size) } }), + 4, + ); + }); + + it("keeps the public list bounded", async () => { + await seedArticles(200); + const service = articleContent.publicService; + if (!service) throw new Error("no public service"); + + await boundedAcrossPageSizes( + async size => + await service(h.counted.context).findMany({ + query: { first: String(size) }, + }), + 4, + ); + }); + + it("keeps a localized public list bounded across languages", async () => { + const service = localizedArticleContent.publicService; + if (!service) throw new Error("no public service"); + const base = localizedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + + for (let index = 0; index < 30; index += 1) { + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: `Body ${index}`, title: `Localized ${index}` }, + }, + { actor: ACTOR }, + ); + await translationEditorial(h.context).create( + row.id, + "pl", + { body: `Tresc ${index}`, title: `Polski ${index}` }, + { actor: ACTOR }, + ); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + row.id, + { actor: ACTOR }, + ); + await translationEditorial(h.context).publish(row.id, "en", { + actor: ACTOR, + }); + await translationEditorial(h.context).publish(row.id, "pl", { + actor: ACTOR, + }); + } + + await boundedAcrossPageSizes( + async size => + await service(h.counted.context).findMany({ + locale: "pl", + query: { first: String(size) }, + }), + 6, + ); + }); + + it("loads a page of advanced collections in batches, not per row", async () => { + // The whole reason a to-many relation is absent from `ContentSelect`: a + // list that carried one would issue a query per row. + const service = advancedArticleContent.publicService; + if (!service) throw new Error("no public service"); + const base = advancedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + const categories = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") + VALUES ('A'), ('B') RETURNING "id" + `; + const translations = advancedArticleContent.translationEditorialService; + if (!translations) throw new Error("no translation editorial service"); + + for (let index = 0; index < 20; index += 1) { + const created = await base(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).create( + { categories: categories.map(row => row.id) }, + { + actor: ACTOR, + }, + ); + await base(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).repeatable.faq.set( + created.row.id, + [ + { answer: "Answer one", question: `Question one ${index}` }, + { answer: "Answer two", question: `Question two ${index}` }, + ], + { actor: ACTOR, expectedVersion: created.version }, + ); + await translations(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).create( + created.row.id, + "en", + { title: `Advanced ${index}` }, + { actor: ACTOR }, + ); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + created.row.id, + { actor: ACTOR }, + ); + await translations(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).publish(created.row.id, "en", { actor: ACTOR }); + } + + const small = await countQueries( + async () => + await service(h.counted.context).findMany({ + locale: "en", + query: { first: "3" }, + }), + ); + const large = await countQueries( + async () => + await service(h.counted.context).findMany({ + locale: "en", + query: { first: "20" }, + }), + ); + + expect(large.length).toBe(small.length); + // Two exposed collections - `categories` and `faq` - so two batch reads + // for the whole page, however many rows are on it. + expect( + large.filter(query => + query.includes("example_advanced_articles_categories"), + ), + ).toHaveLength(1); + expect( + large.filter(query => query.includes("example_advanced_articles_faq")), + ).toHaveLength(1); + }); + + it("fetches no collection the public projection does not expose", async () => { + // `relatedArticles` is private on this content type, so a public read + // must not touch its junction table at all - querying it to discard the + // rows afterwards is work with no answer attached. + const service = advancedArticleContent.publicService; + if (!service) throw new Error("no public service"); + + const queries = await countQueries( + async () => + await service(h.counted.context).findMany({ + locale: "en", + query: { first: "20" }, + }), + ); + + expect( + queries.filter(query => + query.includes("example_advanced_articles_related_articles"), + ), + ).toEqual([]); + }); + + it("keeps a revision history page bounded", async () => { + const created = await editorial(h.context).create( + { category: categoryId, code: "history", title: "History subject" }, + { actor: ACTOR }, + ); + let version = created.version; + for (let index = 0; index < 12; index += 1) { + const outcome = await editorial(h.context).update( + created.row.id, + { title: `History subject ${index}` }, + { actor: ACTOR, expectedVersion: version }, + ); + version = outcome?.version ?? version; + } + + const small = await countQueries( + async () => + await editorial(h.counted.context).revisions.list(created.row.id, { + limit: 2, + }), + ); + const large = await countQueries( + async () => + await editorial(h.counted.context).revisions.list(created.row.id, { + limit: 13, + }), + ); + + expect(large.length).toBe(small.length); + expect(large.length).toBeLessThanOrEqual(2); + }); + }); + + // ------------------------------------------------------------------------- + // Search rebuild + // ------------------------------------------------------------------------- + + describe("the search rebuild reads in batches", () => { + it("keeps a page of the non-localized rebuild to a bounded number of queries", async () => { + await seedArticles(200); + const indexer = createContentSearchIndexer(articleContent, { + pluginId: CONFIG_PLUGIN.pluginId, + }); + + const small = await countQueries( + async () => await indexer.load(h.counted.context, 0, 5), + ); + const large = await countQueries( + async () => await indexer.load(h.counted.context, 0, 100), + ); + + expect(large.length).toBe(small.length); + expect(large.length).toBeLessThanOrEqual(2); + }); + + it("loads a shared repeatable once for a page, not once per locale", async () => { + // The localized rebuild emits one document per published translation, so + // a record with three languages appears three times on a page. Its FAQ is + // shared, and loading it three times would be an N+1 hiding behind a + // correct result. + const base = advancedArticleContent.editorialService; + const translations = advancedArticleContent.translationEditorialService; + if (!base || !translations) throw new Error("no editorial services"); + + for (let index = 0; index < 5; index += 1) { + const created = await base(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).create({}, { actor: ACTOR }); + await base(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).repeatable.faq.set( + created.row.id, + [{ answer: "Answer", question: `Question ${index}` }], + { actor: ACTOR, expectedVersion: created.version }, + ); + for (const locale of ["en", "pl"] as const) { + await translations(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).create( + created.row.id, + locale, + { title: `Advanced ${locale} ${index}` }, + { actor: ACTOR }, + ); + await translations(h.context, { + pluginId: CONFIG_PLUGIN.pluginId, + }).publish(created.row.id, locale, { actor: ACTOR }); + } + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + created.row.id, + { actor: ACTOR }, + ); + } + + const indexer = createContentLocalizedSearchIndexer( + advancedArticleContent, + { pluginId: CONFIG_PLUGIN.pluginId }, + ); + + const queries = await countQueries( + async () => await indexer.load(h.counted.context, 0, 50), + ); + + expect( + queries.filter(query => + query.includes("example_advanced_articles_faq"), + ), + ).toHaveLength(1); + expect(queries.length).toBeLessThanOrEqual(4); + }); + + it("pages the localized rebuild by translation, without repeating one", async () => { + const base = localizedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + + for (let index = 0; index < 6; index += 1) { + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: `Body ${index}`, title: `Paged ${index}` }, + }, + { actor: ACTOR }, + ); + await translationEditorial(h.context).create( + row.id, + "pl", + { body: `Tresc ${index}`, title: `Polski ${index}` }, + { actor: ACTOR }, + ); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + row.id, + { actor: ACTOR }, + ); + for (const locale of ["en", "pl"] as const) { + await translationEditorial(h.context).publish(row.id, locale, { + actor: ACTOR, + }); + } + } + + const indexer = createContentLocalizedSearchIndexer( + localizedArticleContent, + { pluginId: CONFIG_PLUGIN.pluginId }, + ); + const documents: SearchDocument[] = []; + for (let offset = 0; ;) { + const page = await indexer.load(h.context, offset, 4); + if (page.itemsRead === 0) break; + documents.push(...page.documents); + offset += page.itemsRead; + } + + const keys = documents.map( + document => `${document.itemId}:${document.languageCode ?? ""}`, + ); + expect(keys).toHaveLength(12); + expect(new Set(keys).size).toBe(12); + expect(await indexer.count?.(h.context)).toBe(12); + }); + }); + + // ------------------------------------------------------------------------- + // Indexes and plans + // ------------------------------------------------------------------------- + + describe("the generated indexes exist and are the ones the queries need", () => { + it("indexes the slug uniquely", async () => { + const indexes = await indexesOn("example_articles"); + const slug = indexes.find(entry => entry.indexname.includes("slug")); + + expect(slug?.indexdef).toContain("CREATE UNIQUE INDEX"); + expect(slug?.indexdef).toContain("slug"); + }); + + it("indexes the publication predicate the public list orders by", async () => { + const indexes = await indexesOn("example_articles"); + + expect( + indexes.some( + entry => + entry.indexdef.includes("status") && + entry.indexdef.includes("publishedAt"), + ), + ).toBe(true); + }); + + it("indexes a revision history by record and version", async () => { + const indexes = await indexesOn("core_content_revisions"); + + expect( + indexes.some( + entry => + entry.indexname === "core_content_revisions_item_version_unique", + ), + ).toBe(true); + }); + + it("indexes a junction from both ends", async () => { + const indexes = await indexesOn("example_advanced_articles_categories"); + + // The primary key covers `(itemId, relatedItemId)`, which is what the + // membership `EXISTS` seeks on; the second index covers the reverse + // lookup, which Postgres does not create for a foreign key on its own. + expect(indexes.some(entry => entry.indexname.endsWith("_pk"))).toBe(true); + expect( + indexes.some(entry => entry.indexname.endsWith("_related_item_id_idx")), + ).toBe(true); + }); + + it("indexes a repeatable's position uniquely per parent", async () => { + const indexes = await indexesOn("example_advanced_articles_faq"); + + const position = indexes.find(entry => + entry.indexname.endsWith("_position_key"), + ); + expect(position?.indexdef).toContain("CREATE UNIQUE INDEX"); + expect(position?.indexdef).toContain("position"); + }); + + it("seeks rather than scans for a slug lookup", async () => { + await seedArticles(SCALE); + + const explained = await plan( + `SELECT "id" FROM "example_articles" WHERE "slug" = 'article-1234'`, + ); + + // A unique index over two thousand rows is not a close call for the + // planner, which is why this one is safe to assert. + expect(explained).toContain("Index"); + expect(explained).not.toContain("Seq Scan"); + }); + + it("seeks rather than scans for a lookup by identifier", async () => { + await seedArticles(SCALE); + const [row] = await h.sql<{ id: number }[]>` + SELECT "id" FROM "example_articles" LIMIT 1 + `; + + const explained = await plan( + `SELECT "id" FROM "example_articles" WHERE "id" = ${row.id}`, + ); + + expect(explained).toContain("Index"); + expect(explained).not.toContain("Seq Scan"); + }); + + it("seeks rather than scans for one record's revision history", async () => { + await h.sql` + INSERT INTO "core_content_revisions" + ("pluginId", "contentTypeId", "itemId", "version", "operation", "snapshot") + SELECT + ${CONFIG_PLUGIN.pluginId}, 'example.article', i / 20 + 1, + i % 20 + 1, 'update', '{}'::jsonb + FROM generate_series(1, ${SCALE}::int) AS i + `; + await h.sql`ANALYZE "core_content_revisions"`; + + const explained = await plan( + `SELECT "id" FROM "core_content_revisions" + WHERE "contentTypeId" = 'example.article' AND "itemId" = 7 + AND "languageId" IS NULL + ORDER BY "version" DESC LIMIT 25`, + ); + + expect(explained).not.toContain("Seq Scan"); + }); + }); + + // ------------------------------------------------------------------------- + // Memory + // ------------------------------------------------------------------------- + + describe("reads stay page-bound", () => { + it("never materialises more rows than the page asked for", async () => { + // The property that keeps a large collection usable: a page is a page + // whatever the table holds behind it. + await seedArticles(SCALE); + + const page = await articleContent + .service(h.context) + .findMany({ query: { first: "25" } }); + + expect(page.edges).toHaveLength(25); + expect(page.pageInfo.totalCount).toBe(SCALE); + }); + + it("counts the whole collection without reading it", async () => { + await seedArticles(SCALE); + + const queries = await countQueries( + async () => + await articleContent + .service(h.counted.context) + .findMany({ query: { first: "5" } }), + ); + + // The count is an aggregate, not a fetch: no statement in the page's set + // asks for every row. + expect(queries.some(query => /count\(/i.test(query))).toBe(true); + expect(queries.length).toBeLessThanOrEqual(4); + }); + }); +}); From 04fdf4d4a04edf38660eddb8465538520dd3f89c Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:46:58 +0200 Subject: [PATCH 06/14] test(content): add integrity, schema-evolution and migration upgrade coverage Three things Stage 1-6 never proved, all of them about a record outliving the assumptions it was written under. **Delete integrity.** Translations, junction rows and child rows go with the record; a category still referenced refuses to go at all; a self-relation's reference is dropped without taking the record that held it; an author's account can be removed without removing their articles; and no orphan is left in either direction. Asserted against Postgres rather than against the service, because a check in application code is one a direct DELETE walks past. **Schema evolution.** Revisions are written straight into the table in the envelope the engine really uses - which is the only honest way to model a history full of rows written by code that no longer exists. A dropped field is ignored, a field added since is left alone, a value that no longer validates is a structured refusal with the field named and nothing written, a missing relation target is fatal, and a repeatable child whose identifier is gone comes back with its values and a new one. **Migration upgrades.** Every pattern the docs describe, run against real rows: adding a group to a populated table, to-one to to-many, JSON array to repeatable, and non-localized to localized including its verification step. The rule they share is checked each time - the destructive statement is in a different migration from the copy - along with what a failed migration leaves behind, which in Postgres is nothing at all. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/database/integrity-postgres.test.ts | 733 ++++++++++++++++++ .../src/database/migration-postgres.test.ts | 680 ++++++++++++++++ 2 files changed, 1413 insertions(+) create mode 100644 plugins/example/src/database/integrity-postgres.test.ts create mode 100644 plugins/example/src/database/migration-postgres.test.ts diff --git a/plugins/example/src/database/integrity-postgres.test.ts b/plugins/example/src/database/integrity-postgres.test.ts new file mode 100644 index 000000000..11b1b6a51 --- /dev/null +++ b/plugins/example/src/database/integrity-postgres.test.ts @@ -0,0 +1,733 @@ +import type { Context } from "hono"; + +import { + ContentLanguageError, + ContentRevisionNotRestorable, +} from "@vitnode/core/content"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import { CONFIG_PLUGIN } from "@/const"; + +import type { ContentTestHarness } from "./harness"; + +import { advancedArticleContent } from "./advanced-articles"; +import { articleContent } from "./articles"; +import { + ACTOR, + clearContentTables, + createContentTestHarness, + DATABASE_TEST_URL, + pgErrorCode, +} from "./harness"; +import { localizedArticleContent } from "./localized-articles"; + +/** + * What the **database** guarantees, and what the engine does when a definition + * has moved on since a revision was written. + * + * Two halves that look unrelated and are not: both are about a record outliving + * the assumptions it was written under. A delete has to take exactly the rows + * that belong to the record and refuse exactly the ones that belong to somebody + * else; a restore has to apply a snapshot written against an older shape, or + * refuse it whole. + * + * Wherever Postgres can enforce something, the assertion is against Postgres + * rather than against the service - a check in application code is one a direct + * `DELETE` walks straight past. + */ + +let h: ContentTestHarness; +let categoryId = 0; +let seq = 0; + +const editorial = (on: Context) => { + const build = articleContent.editorialService; + if (!build) throw new Error("example.article has no editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const advanced = (on: Context) => { + const build = advancedArticleContent.editorialService; + if (!build) throw new Error("no advanced editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const localizedService = (on: Context) => { + const build = localizedArticleContent.localizedService; + if (!build) throw new Error("no localized service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const translationEditorial = (on: Context) => { + const build = localizedArticleContent.translationEditorialService; + if (!build) throw new Error("no translation editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const advancedTranslations = (on: Context) => { + const build = advancedArticleContent.translationEditorialService; + if (!build) throw new Error("no advanced translation editorial service"); + + return build(on, { pluginId: CONFIG_PLUGIN.pluginId }); +}; + +const article = async () => { + seq += 1; + const outcome = await editorial(h.context).create( + { + category: categoryId, + code: `integrity-${seq}`, + title: `Integrity subject ${seq}`, + }, + { actor: ACTOR }, + ); + + return { id: outcome.row.id, version: outcome.version }; +}; + +const countOf = async (table: string): Promise => { + const [row] = await h.sql.unsafe( + `SELECT count(*)::int AS count FROM "${table}"`, + ); + + return Number(row.count); +}; + +describe.skipIf(!DATABASE_TEST_URL)("Content Engine integrity", () => { + beforeAll(async () => { + h = await createContentTestHarness(); + }, 60_000); + + afterAll(async () => { + await h?.end(); + }); + + beforeEach(async () => { + await clearContentTables(h.sql); + h.reset(); + + const [category] = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") VALUES ('Integrity') + RETURNING "id" + `; + categoryId = category.id; + }); + + // ------------------------------------------------------------------------- + // Delete integrity + // ------------------------------------------------------------------------- + + describe("deleting a record", () => { + it("takes its translations with it", async () => { + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: "Body", title: "Cascade Subject" }, + }, + { actor: ACTOR }, + ); + await translationEditorial(h.context).create( + row.id, + "pl", + { body: "Tresc", title: "Polski" }, + { actor: ACTOR }, + ); + expect(await countOf("example_localized_articles_translations")).toBe(2); + + await h.sql` + DELETE FROM "example_localized_articles" WHERE "id" = ${row.id} + `; + + expect(await countOf("example_localized_articles_translations")).toBe(0); + }); + + it("takes its junction and child rows with it", async () => { + const created = await advanced(h.context).create( + { categories: [categoryId] }, + { actor: ACTOR }, + ); + await advanced(h.context).repeatable.faq.set( + created.row.id, + [{ answer: "An answer", question: "A question" }], + { actor: ACTOR, expectedVersion: created.version }, + ); + expect(await countOf("example_advanced_articles_categories")).toBe(1); + expect(await countOf("example_advanced_articles_faq")).toBe(1); + + await h.sql` + DELETE FROM "example_advanced_articles" WHERE "id" = ${created.row.id} + `; + + expect(await countOf("example_advanced_articles_categories")).toBe(0); + expect(await countOf("example_advanced_articles_faq")).toBe(0); + }); + + it("keeps its history, which outlives it deliberately", async () => { + const created = await article(); + + await editorial(h.context).delete(created.id, { + actor: ACTOR, + expectedVersion: created.version, + }); + + const revisions = await h.sql<{ operation: string }[]>` + SELECT "operation" FROM "core_content_revisions" + WHERE "contentTypeId" = 'example.article' AND "itemId" = ${created.id} + ORDER BY "version" + `; + // "Who removed this, and what did it say" is only answerable if the + // history is not a foreign key to the row it describes. + expect(revisions.map(row => row.operation)).toEqual(["create", "delete"]); + }); + + it("refuses to remove a category that content still points at", async () => { + await article(); + + const code = await pgErrorCode( + async () => + await h.sql`DELETE FROM "example_categories" WHERE "id" = ${categoryId}`, + ); + + // `onDelete: "restrict"`, enforced by Postgres rather than by a check in + // service code that a direct `DELETE` would walk past. + expect(code).toBe(h.serverMajor >= 18 ? "23001" : "23503"); + }); + + it("refuses to remove a category a to-many relation still points at", async () => { + const created = await advanced(h.context).create( + { categories: [categoryId] }, + { actor: ACTOR }, + ); + expect(created.row.id).toBeGreaterThan(0); + + const code = await pgErrorCode( + async () => + await h.sql`DELETE FROM "example_categories" WHERE "id" = ${categoryId}`, + ); + + expect(code).toBe(h.serverMajor >= 18 ? "23001" : "23503"); + }); + + it("drops a self-relation's reference when its target goes", async () => { + // `relatedArticles` is `onDelete: "cascade"`: forgetting the reference is + // the honest analogue of nulling a column, because a junction row has no + // column to null. + const source = await advanced(h.context).create({}, { actor: ACTOR }); + const target = await advanced(h.context).create({}, { actor: ACTOR }); + await advanced(h.context).relations.relatedArticles.set( + source.row.id, + [target.row.id], + { actor: ACTOR, expectedVersion: source.version }, + ); + expect(await countOf("example_advanced_articles_related_articles")).toBe( + 1, + ); + + await h.sql` + DELETE FROM "example_advanced_articles" WHERE "id" = ${target.row.id} + `; + + expect(await countOf("example_advanced_articles_related_articles")).toBe( + 0, + ); + // And the source record is still there: a cascade on the reference is not + // a cascade on the record that held it. + const [row] = await h.sql<{ id: number }[]>` + SELECT "id" FROM "example_advanced_articles" WHERE "id" = ${source.row.id} + `; + expect(row).toBeDefined(); + }); + + it("leaves no orphaned junction row behind, in either direction", async () => { + const created = await advanced(h.context).create( + { categories: [categoryId] }, + { actor: ACTOR }, + ); + await advanced(h.context).relations.relatedArticles.set( + created.row.id, + [created.row.id], + { actor: ACTOR, expectedVersion: created.version }, + ); + + await h.sql` + DELETE FROM "example_advanced_articles" WHERE "id" = ${created.row.id} + `; + + const orphans = await h.sql<{ count: number }[]>` + SELECT ( + (SELECT count(*) FROM "example_advanced_articles_categories" j + LEFT JOIN "example_advanced_articles" a ON a."id" = j."itemId" + WHERE a."id" IS NULL) + + + (SELECT count(*) FROM "example_advanced_articles_related_articles" r + LEFT JOIN "example_advanced_articles" a ON a."id" = r."itemId" + WHERE a."id" IS NULL) + )::int AS count + `; + expect(orphans[0].count).toBe(0); + }); + + it("clears a user reference rather than removing the record", async () => { + // `onDelete: "set null"` on a nullable user field: an article does not + // stop existing because its author's account did. + const [user] = await h.sql<{ id: number }[]>` + INSERT INTO "core_users" ("name") VALUES ('Ada') RETURNING "id" + `; + seq += 1; + const created = await editorial(h.context).create( + { + author: user.id, + category: categoryId, + code: `authored-${seq}`, + title: "Authored subject", + }, + { actor: ACTOR }, + ); + + await h.sql`DELETE FROM "core_users" WHERE "id" = ${user.id}`; + + const [row] = await h.sql<{ author: null | number }[]>` + SELECT "author" FROM "example_articles" WHERE "id" = ${created.row.id} + `; + expect(row.author).toBeNull(); + }); + }); + + // ------------------------------------------------------------------------- + // Schema evolution + // ------------------------------------------------------------------------- + + /** + * A revision written under an older definition, applied to today's. + * + * The snapshots are written straight into `core_content_revisions`, which is + * the honest way to model this: a real installation's history is full of rows + * written by code that no longer exists, and there is no way to get one except + * by having been there. + */ + describe("restoring a revision written under an older definition", () => { + /** + * A revision row in the envelope the engine really writes. + * + * `fields` is the whole point: a snapshot is a versioned envelope around + * the declared field values, and `projectRevisionSnapshot` reads that half + * rather than the row it came from. Writing a flat object here would test a + * shape no revision has ever had. + */ + const writeRevision = async ( + itemId: number, + version: number, + fields: Record, + contentTypeId = "example.article", + ) => { + const snapshot = { + contentTypeId, + createdAt: new Date(0).toISOString(), + fields, + id: itemId, + schemaVersion: 1, + updatedAt: new Date(0).toISOString(), + version, + }; + + const [row] = await h.sql<{ id: number }[]>` + INSERT INTO "core_content_revisions" + ("pluginId", "contentTypeId", "itemId", "version", "operation", + "actorType", "snapshot") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${contentTypeId}, ${itemId}, ${version}, + 'update', 'staff', ${JSON.stringify(snapshot)}::jsonb + ) + RETURNING "id" + `; + + return row.id; + }; + + it("ignores a field the content type has since dropped", async () => { + const created = await article(); + const revisionId = await writeRevision(created.id, 900, { + // `subtitle` was a field once. It is not one now, and a restore has to + // drop it rather than hand it to a strict schema that will refuse it. + subtitle: "A field that no longer exists", + title: "Restored from an older shape", + }); + + const outcome = await editorial(h.context).restore( + created.id, + revisionId, + { actor: ACTOR, expectedVersion: created.version }, + ); + + expect(outcome?.changed).toBe(true); + const [row] = await h.sql<{ title: string }[]>` + SELECT "title" FROM "example_articles" WHERE "id" = ${created.id} + `; + expect(row.title).toBe("Restored from an older shape"); + }); + + it("leaves a field added since the snapshot exactly as it stands", async () => { + // The update schema is partial, so a field the snapshot never carried is + // simply not written - which is the only answer that does not invent a + // value nobody chose. + const created = await article(); + await editorial(h.context).update( + created.id, + { excerpt: "Written after the snapshot" }, + { actor: ACTOR, expectedVersion: created.version }, + ); + const revisionId = await writeRevision(created.id, 901, { + title: "Older still", + }); + + await editorial(h.context).restore(created.id, revisionId, { + actor: ACTOR, + expectedVersion: created.version + 1, + }); + + const [row] = await h.sql<{ excerpt: null | string; title: string }[]>` + SELECT "title", "excerpt" FROM "example_articles" WHERE "id" = ${created.id} + `; + expect(row.title).toBe("Older still"); + expect(row.excerpt).toBe("Written after the snapshot"); + }); + + it("refuses a snapshot whose value no longer validates, and writes nothing", async () => { + const created = await article(); + const before = await h.sql<{ title: string; version: number }[]>` + SELECT "title", "version" FROM "example_articles" WHERE "id" = ${created.id} + `; + const revisionId = await writeRevision(created.id, 902, { + // `title` has a three-character minimum today. It did not always. + title: "No", + }); + + await expect( + editorial(h.context).restore(created.id, revisionId, { + actor: ACTOR, + expectedVersion: created.version, + }), + ).rejects.toBeInstanceOf(ContentRevisionNotRestorable); + + // All or nothing: the record is byte-identical to what it was. + const after = await h.sql<{ title: string; version: number }[]>` + SELECT "title", "version" FROM "example_articles" WHERE "id" = ${created.id} + `; + expect(after).toEqual(before); + }); + + it("names the field, and nothing internal, when it refuses", async () => { + const created = await article(); + const revisionId = await writeRevision(created.id, 903, { + title: "No", + }); + + try { + await editorial(h.context).restore(created.id, revisionId, { + actor: ACTOR, + expectedVersion: created.version, + }); + throw new Error("Expected the restore to be refused."); + } catch (error) { + expect(error).toBeInstanceOf(ContentRevisionNotRestorable); + const refusal = error as ContentRevisionNotRestorable; + expect(refusal.fields).toEqual(["title"]); + // Never a Zod issue tree: it names internal paths, and the route's + // OpenAPI schema already describes the contract. + expect(JSON.stringify(refusal.fields)).not.toContain("_zod"); + } + }); + + it("refuses when a relation target in the snapshot is gone", async () => { + const created = await advanced(h.context).create( + { categories: [categoryId] }, + { actor: ACTOR }, + ); + const [spare] = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") VALUES ('Doomed') RETURNING "id" + `; + const revisionId = await writeRevision( + created.row.id, + 904, + { categories: [spare.id] }, + "example.advanced-article", + ); + await h.sql`DELETE FROM "example_categories" WHERE "id" = ${spare.id}`; + + await expect( + advanced(h.context).restore(created.row.id, revisionId, { + actor: ACTOR, + expectedVersion: created.version, + }), + ).rejects.toBeInstanceOf(ContentRevisionNotRestorable); + + // Nothing partial: the relation it *could* have restored is untouched. + const rows = await h.sql<{ relatedItemId: number }[]>` + SELECT "relatedItemId" FROM "example_advanced_articles_categories" + WHERE "itemId" = ${created.row.id} + `; + expect(rows.map(row => row.relatedItemId)).toEqual([categoryId]); + }); + + it("recreates a repeatable child whose identifier is gone", async () => { + // The other rule, and the reason the two kinds differ: a child's values + // are all in the snapshot, so recreating it loses nothing but its + // identifier. A relation target's values were never there to begin with. + const created = await advanced(h.context).create({}, { actor: ACTOR }); + const seeded = await advanced(h.context).repeatable.faq.set( + created.row.id, + [{ answer: "The answer", question: "The question" }], + { actor: ACTOR, expectedVersion: created.version }, + ); + const [child] = await h.sql<{ id: number }[]>` + SELECT "id" FROM "example_advanced_articles_faq" + WHERE "itemId" = ${created.row.id} + `; + + await advanced(h.context).repeatable.faq.delete( + created.row.id, + child.id, + { actor: ACTOR, expectedVersion: seeded?.version ?? created.version }, + ); + + const revisionId = await writeRevision( + created.row.id, + 905, + { + faq: [ + { answer: "The answer", id: child.id, question: "The question" }, + ], + }, + "example.advanced-article", + ); + + const [current] = await h.sql<{ version: number }[]>` + SELECT "version" FROM "example_advanced_articles" + WHERE "id" = ${created.row.id} + `; + const outcome = await advanced(h.context).restore( + created.row.id, + revisionId, + { actor: ACTOR, expectedVersion: current.version }, + ); + + expect(outcome?.changed).toBe(true); + const rows = await h.sql<{ id: number; question: string }[]>` + SELECT "id", "question" FROM "example_advanced_articles_faq" + WHERE "itemId" = ${created.row.id} + `; + expect(rows).toHaveLength(1); + expect(rows[0].question).toBe("The question"); + // A new identifier, because the old row is gone. The values came back; + // the identity did not, and could not. + expect(rows[0].id).not.toBe(child.id); + }); + + it("keeps the restored-from revision untouched", async () => { + const created = await article(); + const revisionId = await writeRevision(created.id, 906, { + title: "Immutable source", + }); + const [before] = await h.sql<{ snapshot: unknown; version: number }[]>` + SELECT "snapshot", "version" FROM "core_content_revisions" + WHERE "id" = ${revisionId} + `; + + await editorial(h.context).restore(created.id, revisionId, { + actor: ACTOR, + expectedVersion: created.version, + }); + + const [after] = await h.sql<{ snapshot: unknown; version: number }[]>` + SELECT "snapshot", "version" FROM "core_content_revisions" + WHERE "id" = ${revisionId} + `; + expect(after).toEqual(before); + }); + + it("moves the record forward rather than backward", async () => { + const created = await article(); + const revisionId = await writeRevision(created.id, 907, { + title: "Rolled forward", + }); + + const outcome = await editorial(h.context).restore( + created.id, + revisionId, + { actor: ACTOR, expectedVersion: created.version }, + ); + + // A restore is an edit, not a rewind: the version increases and the + // history gains an entry rather than losing one. + expect(outcome?.version).toBe(created.version + 1); + expect(outcome?.restoredFromRevisionId).toBe(revisionId); + }); + }); + + // ------------------------------------------------------------------------- + // Disabled locales + // ------------------------------------------------------------------------- + + /** + * The Stage 5 policy, unchanged and now pinned on both kinds of localized + * content type: + * + * | create | update | restore | publish | unpublish | delete | read | + * | ------ | ------ | ------- | ------- | --------- | ------ | ---- | + * | refuse | refuse | refuse | refuse | allow | allow | allow| + * + * The asymmetry is the point. Switching a language off must stop new content + * going into it, and must **not** trap the content that is already there: + * taking a page down and deleting it are exactly the operations an + * administrator needs after switching the language off. + */ + describe("a locale the installation has switched off", () => { + const DISABLED = "de"; + + /** A record with a `de` translation already written, before the switch-off. */ + const withGermanTranslation = async () => { + const { row } = await localizedService(h.context).create( + { shared: {}, translation: { body: "Body", title: "Locale Policy" } }, + { actor: ACTOR }, + ); + const [german] = await h.sql<{ id: number }[]>` + SELECT "id" FROM "core_languages" WHERE "code" = ${DISABLED} + `; + await h.sql` + INSERT INTO "example_localized_articles_translations" + ("itemId", "languageId", "title", "slug", "body", "version", "status") + VALUES (${row.id}, ${german.id}, 'Deutsch', 'deutsch', 'Körper', 1, 'published') + `; + + return { itemId: row.id, languageId: german.id }; + }; + + const isDisabled = (error: unknown): boolean => + error instanceof ContentLanguageError && error.reason === "disabled"; + + it("refuses a create", async () => { + const { row } = await localizedService(h.context).create( + { shared: {}, translation: { body: "Body", title: "Refused Create" } }, + { actor: ACTOR }, + ); + + await expect( + translationEditorial(h.context).create( + row.id, + DISABLED, + { body: "Körper", title: "Deutsch" }, + { actor: ACTOR }, + ), + ).rejects.toSatisfy(isDisabled); + }); + + it("refuses an update", async () => { + const { itemId } = await withGermanTranslation(); + + await expect( + translationEditorial(h.context).update( + itemId, + DISABLED, + { title: "Deutsch Neu" }, + { actor: ACTOR, expectedVersion: 1 }, + ), + ).rejects.toSatisfy(isDisabled); + }); + + it("refuses a publish", async () => { + const { itemId } = await withGermanTranslation(); + + await expect( + translationEditorial(h.context).publish(itemId, DISABLED, { + actor: ACTOR, + }), + ).rejects.toSatisfy(isDisabled); + }); + + it("refuses a restore", async () => { + const { itemId, languageId } = await withGermanTranslation(); + const [revision] = await h.sql<{ id: number }[]>` + INSERT INTO "core_content_revisions" + ("pluginId", "contentTypeId", "itemId", "languageId", "version", + "operation", "actorType", "snapshot") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, 'example.localized-article', ${itemId}, + ${languageId}, 1, 'create', 'staff', + ${JSON.stringify({ body: "Alt", slug: "alt", title: "Alt", version: 1 })}::jsonb + ) + RETURNING "id" + `; + + await expect( + translationEditorial(h.context).restore(itemId, DISABLED, revision.id, { + actor: ACTOR, + expectedVersion: 1, + }), + ).rejects.toSatisfy(isDisabled); + }); + + it("still allows an unpublish, which is how a page comes down", async () => { + const { itemId } = await withGermanTranslation(); + + const outcome = await translationEditorial(h.context).unpublish( + itemId, + DISABLED, + { actor: ACTOR }, + ); + + expect(outcome?.changed).toBe(true); + }); + + it("still allows a delete", async () => { + const { itemId } = await withGermanTranslation(); + + const outcome = await translationEditorial(h.context).delete( + itemId, + DISABLED, + { actor: ACTOR, expectedVersion: 1 }, + ); + + expect(outcome?.changed).toBe(true); + const rows = await h.sql<{ count: number }[]>` + SELECT count(*)::int AS count + FROM "example_localized_articles_translations" + WHERE "itemId" = ${itemId} + `; + expect(rows[0].count).toBe(1); + }); + + it("still allows a read and a history read", async () => { + const { itemId } = await withGermanTranslation(); + const build = localizedArticleContent.translationService; + if (!build) throw new Error("no translation service"); + + const translation = await build(h.context).findByLocale(itemId, DISABLED); + const history = await translationEditorial(h.context).listRevisions( + itemId, + DISABLED, + ); + + expect(translation?.locale).toBe(DISABLED); + expect(history.edges).toEqual([]); + }); + + it("applies the same policy to an advanced localized content type", async () => { + // The rule is the language resolver's, not the content type's - so a + // content type with groups and repeatables gets exactly the same answers. + const created = await advanced(h.context).create({}, { actor: ACTOR }); + + await expect( + advancedTranslations(h.context).create( + created.row.id, + DISABLED, + { title: "Deutsch" }, + { actor: ACTOR }, + ), + ).rejects.toSatisfy(isDisabled); + }); + }); +}); diff --git a/plugins/example/src/database/migration-postgres.test.ts b/plugins/example/src/database/migration-postgres.test.ts new file mode 100644 index 000000000..9dd4f50bd --- /dev/null +++ b/plugins/example/src/database/migration-postgres.test.ts @@ -0,0 +1,680 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import type { ContentTestHarness } from "./harness"; + +import { + createContentTestHarness, + DATABASE_TEST_URL, + pgErrorCode, +} from "./harness"; + +/** + * The documented migration patterns, run against real data. + * + * Stage 1-6 prove the schema a *fresh* install gets. What they never proved is + * the thing an existing install actually does: take a table with rows in it and + * move it onto the newer shape. Every pattern below is copied from + * `apps/docs/content/docs/dev/content-engine/` - so this suite is what stops the + * docs describing a migration that quietly loses rows. + * + * The rule the patterns share, and the one every test here checks: **the + * destructive statement is in a different migration from the copy.** A backfill + * that silently dropped three rows and then deleted its source is not something + * anybody can notice afterwards. + * + * The tables are built here rather than taken from the committed migrations, + * because what is under test is the *shape* of the upgrade rather than one + * install's history - and a Stage 1-era table no longer exists anywhere to + * borrow. + */ + +let h: ContentTestHarness; + +/** Runs a script as one statement per `--> statement-breakpoint`. */ +const migrate = async (script: string): Promise => { + for (const statement of script.split("--> statement-breakpoint")) { + const trimmed = statement.trim(); + if (trimmed) await h.sql.unsafe(trimmed); + } +}; + +const countOf = async (table: string): Promise => { + const [row] = await h.sql.unsafe( + `SELECT count(*)::int AS count FROM "${table}"`, + ); + + return Number(row.count); +}; + +const columnsOf = async (table: string) => + await h.sql<{ column_name: string; is_nullable: string }[]>` + SELECT column_name, is_nullable + FROM information_schema.columns + WHERE table_name = ${table} + ORDER BY column_name + `; + +/** A Stage 1-era flat table: no publication, no editorial, no translations. */ +const STAGE_ONE = ` + CREATE TABLE "legacy_articles" ( + "id" serial PRIMARY KEY NOT NULL, + "createdAt" timestamp DEFAULT now() NOT NULL, + "updatedAt" timestamp DEFAULT now() NOT NULL, + "title" varchar(200) NOT NULL, + "slug" varchar(160) NOT NULL, + "seoTitle" varchar(200), + "category" integer NOT NULL, + "faqJson" jsonb, + CONSTRAINT "legacy_articles_slug_key" UNIQUE("slug") + ); +`; + +const seedLegacy = async (): Promise => { + const [category] = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") VALUES ('Legacy') RETURNING "id" + `; + await h.sql` + INSERT INTO "legacy_articles" ("title", "slug", "seoTitle", "category", "faqJson") + VALUES + ('First', 'first', 'First SEO', ${category.id}, + '[{"question":"Q1","answer":"A1"},{"question":"Q2","answer":"A2"}]'::jsonb), + ('Second', 'second', NULL, ${category.id}, + '[{"question":"Q3","answer":"A3"}]'::jsonb), + ('Third', 'third', NULL, ${category.id}, NULL) + `; +}; + +describe.skipIf(!DATABASE_TEST_URL)("Content Engine migration patterns", () => { + beforeAll(async () => { + h = await createContentTestHarness(); + }, 60_000); + + afterAll(async () => { + await h?.end(); + }); + + beforeEach(async () => { + await h.sql`DROP TABLE IF EXISTS "legacy_articles_translations"`; + await h.sql`DROP TABLE IF EXISTS "legacy_articles_categories"`; + await h.sql`DROP TABLE IF EXISTS "legacy_articles_faq"`; + await h.sql`DROP TABLE IF EXISTS "legacy_articles"`; + await h.sql`DELETE FROM "example_categories"`; + await h.sql.unsafe(STAGE_ONE); + await seedLegacy(); + }); + + // ------------------------------------------------------------------------- + // Additive upgrades + // ------------------------------------------------------------------------- + + describe("adding a structured group to a populated table", () => { + it("gives every existing row a null group without touching its other values", async () => { + const before = await countOf("legacy_articles"); + + await migrate(` + ALTER TABLE "legacy_articles" + ADD COLUMN "syndicationIndexable" boolean DEFAULT true NOT NULL, + ADD COLUMN "syndicationPriority" integer DEFAULT 5 NOT NULL; + `); + + expect(await countOf("legacy_articles")).toBe(before); + + const rows = await h.sql< + { syndicationIndexable: boolean; syndicationPriority: number }[] + >` + SELECT "syndicationIndexable", "syndicationPriority" FROM "legacy_articles" + `; + // A defaulted leaf is what makes this additive at all: a `NOT NULL` + // column with no default cannot be added to a table with rows in it. + expect(rows.every(row => row.syndicationPriority === 5)).toBe(true); + expect(rows.every(row => row.syndicationIndexable)).toBe(true); + }); + + it("regroups an existing column with no data migration at all", async () => { + // `seoTitle` as a top-level field and `seo.title` as a group leaf compile + // to the same column, so the upgrade is a definition change and nothing + // else. The test is that the column and its values are still there. + const before = await h.sql<{ id: number; seoTitle: null | string }[]>` + SELECT "id", "seoTitle" FROM "legacy_articles" ORDER BY "id" + `; + + await migrate(` + ALTER TABLE "legacy_articles" ADD COLUMN "seoDescription" text; + `); + + const after = await h.sql<{ id: number; seoTitle: null | string }[]>` + SELECT "id", "seoTitle" FROM "legacy_articles" ORDER BY "id" + `; + expect(after).toEqual(before); + }); + + it("refuses a non-null leaf with no default, rather than inventing values", async () => { + const code = await pgErrorCode( + async () => + await h.sql.unsafe(` + ALTER TABLE "legacy_articles" + ADD COLUMN "syndicationOwner" varchar(100) NOT NULL + `), + ); + + // 23502: not_null_violation. `defineContentType` refuses this shape at + // definition time, and Postgres refuses it here - which is what makes the + // rule a fact rather than a convention. + expect(code).toBe("23502"); + }); + }); + + // ------------------------------------------------------------------------- + // To-one to to-many + // ------------------------------------------------------------------------- + + describe("moving a to-one relation onto a junction table", () => { + const CREATE_JUNCTION = ` + CREATE TABLE "legacy_articles_categories" ( + "itemId" integer NOT NULL, + "relatedItemId" integer NOT NULL, + "position" integer NOT NULL, + "createdAt" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "legacy_articles_categories_pk" + PRIMARY KEY("itemId","relatedItemId") + );--> statement-breakpoint + CREATE UNIQUE INDEX "legacy_articles_categories_position_key" + ON "legacy_articles_categories" ("itemId","position");--> statement-breakpoint + INSERT INTO "legacy_articles_categories" ("itemId", "relatedItemId", "position") + SELECT "id", "category", 0 FROM "legacy_articles" WHERE "category" IS NOT NULL; + `; + + it("copies every reference, at position zero, before anything is dropped", async () => { + const before = await countOf("legacy_articles"); + + await migrate(CREATE_JUNCTION); + + expect(await countOf("legacy_articles_categories")).toBe(before); + const rows = await h.sql<{ position: number }[]>` + SELECT "position" FROM "legacy_articles_categories" + `; + expect(rows.every(row => row.position === 0)).toBe(true); + + // The source column is still there. That is the pattern: the destructive + // statement is a *second* migration, run after somebody has looked at the + // counts. + expect( + (await columnsOf("legacy_articles")).map(row => row.column_name), + ).toContain("category"); + }); + + it("keeps the foreign key honest once it is added", async () => { + await migrate(` + ${CREATE_JUNCTION}--> statement-breakpoint + ALTER TABLE "legacy_articles_categories" + ADD CONSTRAINT "legacy_articles_categories_related_fk" + FOREIGN KEY ("relatedItemId") REFERENCES "example_categories"("id") + ON DELETE restrict; + `); + + const code = await pgErrorCode( + async () => + await h.sql` + INSERT INTO "legacy_articles_categories" ("itemId", "relatedItemId", "position") + VALUES (1, 999999, 1) + `, + ); + + expect(code).toBe("23503"); + }); + + it("drops the source column only in the second migration", async () => { + await migrate(CREATE_JUNCTION); + const copied = await countOf("legacy_articles_categories"); + const source = await countOf("legacy_articles"); + + // The pause in the middle, made explicit: the drop is guarded by the very + // comparison the docs tell an operator to make by hand. + expect(copied).toBe(source); + + await migrate(`ALTER TABLE "legacy_articles" DROP COLUMN "category";`); + + expect( + (await columnsOf("legacy_articles")).map(row => row.column_name), + ).not.toContain("category"); + expect(await countOf("legacy_articles_categories")).toBe(copied); + }); + + it("aborts the copy whole when one row cannot be copied", async () => { + // Postgres runs a migration statement in a transaction, so a backfill + // that fails halfway leaves nothing behind - which is what makes the + // "verify before you drop" pattern safe to retry. + await migrate(` + CREATE TABLE "legacy_articles_categories" ( + "itemId" integer NOT NULL, + "relatedItemId" integer NOT NULL, + "position" integer NOT NULL, + CONSTRAINT "legacy_articles_categories_pk" + PRIMARY KEY("itemId","relatedItemId"), + CONSTRAINT "legacy_articles_categories_position_check" + CHECK ("position" >= 0) + ); + `); + + const code = await pgErrorCode( + async () => + await h.sql` + INSERT INTO "legacy_articles_categories" ("itemId", "relatedItemId", "position") + SELECT "id", "category", "id" - 100 FROM "legacy_articles" + `, + ); + + expect(code).toBe("23514"); + expect(await countOf("legacy_articles_categories")).toBe(0); + }); + }); + + // ------------------------------------------------------------------------- + // JSON array to repeatable + // ------------------------------------------------------------------------- + + describe("moving a JSON array onto a repeatable child table", () => { + const CREATE_CHILD = ` + CREATE TABLE "legacy_articles_faq" ( + "id" serial PRIMARY KEY NOT NULL, + "itemId" integer NOT NULL, + "position" integer NOT NULL, + "createdAt" timestamp DEFAULT now() NOT NULL, + "updatedAt" timestamp DEFAULT now() NOT NULL, + "question" varchar(200) NOT NULL, + "answer" text NOT NULL + );--> statement-breakpoint + CREATE UNIQUE INDEX "legacy_articles_faq_position_key" + ON "legacy_articles_faq" ("itemId","position");--> statement-breakpoint + INSERT INTO "legacy_articles_faq" ("itemId", "position", "question", "answer") + SELECT + a."id", + entry.ordinality - 1, + entry.value ->> 'question', + entry.value ->> 'answer' + FROM "legacy_articles" a, + jsonb_array_elements(a."faqJson") + WITH ORDINALITY AS entry(value, ordinality) + WHERE a."faqJson" IS NOT NULL; + `; + + it("copies every entry and preserves its order", async () => { + await migrate(CREATE_CHILD); + + const [{ expected }] = await h.sql<{ expected: number }[]>` + SELECT coalesce(sum(jsonb_array_length("faqJson")), 0)::int AS expected + FROM "legacy_articles" + `; + expect(await countOf("legacy_articles_faq")).toBe(expected); + + const rows = await h.sql<{ position: number; question: string }[]>` + SELECT f."position", f."question" FROM "legacy_articles_faq" f + JOIN "legacy_articles" a ON a."id" = f."itemId" + WHERE a."slug" = 'first' + ORDER BY f."position" + `; + // `WITH ORDINALITY` is what carries the order across, and the engine reads + // a repeatable back in `position` order - so getting this wrong reorders + // somebody's FAQ silently. + expect(rows).toEqual([ + { position: 0, question: "Q1" }, + { position: 1, question: "Q2" }, + ]); + }); + + it("starts positions at zero, which is where the engine reads from", async () => { + await migrate(CREATE_CHILD); + + const [row] = await h.sql<{ min: number }[]>` + SELECT min("position")::int AS min FROM "legacy_articles_faq" + `; + expect(row.min).toBe(0); + }); + + it("copies nothing for a record whose array was null", async () => { + await migrate(CREATE_CHILD); + + const rows = await h.sql<{ count: number }[]>` + SELECT count(*)::int AS count FROM "legacy_articles_faq" f + JOIN "legacy_articles" a ON a."id" = f."itemId" + WHERE a."slug" = 'third' + `; + expect(rows[0].count).toBe(0); + }); + + it("leaves the source column in place for the operator to check", async () => { + await migrate(CREATE_CHILD); + + expect( + (await columnsOf("legacy_articles")).map(row => row.column_name), + ).toContain("faqJson"); + }); + + it("refuses two entries in one position once the index is there", async () => { + await migrate(CREATE_CHILD); + + const code = await pgErrorCode( + async () => + await h.sql` + INSERT INTO "legacy_articles_faq" ("itemId", "position", "question", "answer") + SELECT "itemId", "position", 'Dup', 'Dup' FROM "legacy_articles_faq" LIMIT 1 + `, + ); + + expect(code).toBe("23505"); + }); + }); + + // ------------------------------------------------------------------------- + // Non-localized to localized + // ------------------------------------------------------------------------- + + describe("localizing a table that already has rows", () => { + const CREATE_TRANSLATIONS = ` + CREATE TABLE "legacy_articles_translations" ( + "itemId" integer NOT NULL, + "languageId" integer NOT NULL, + "version" integer DEFAULT 1 NOT NULL, + "createdAt" timestamp DEFAULT now() NOT NULL, + "updatedAt" timestamp DEFAULT now() NOT NULL, + "title" varchar(200) NOT NULL, + "slug" varchar(160) NOT NULL + );--> statement-breakpoint + INSERT INTO "legacy_articles_translations" + ("itemId", "languageId", "title", "slug", "createdAt", "updatedAt") + SELECT + a."id", + (SELECT "id" FROM "core_languages" WHERE "code" = 'en'), + a."title", + a."slug", + a."createdAt", + a."updatedAt" + FROM "legacy_articles" a; + `; + + /** Step 4 of the documented six, verbatim in shape. */ + const VERIFY = ` + DO $$ + DECLARE + source_count integer; + copied_count integer; + language_id integer; + BEGIN + SELECT "id" INTO language_id FROM "core_languages" WHERE "code" = 'en'; + IF language_id IS NULL THEN + RAISE EXCEPTION 'No core_languages row for the default locale "en".'; + END IF; + + SELECT count(*) INTO source_count FROM "legacy_articles"; + SELECT count(*) INTO copied_count FROM "legacy_articles_translations"; + + IF source_count <> copied_count THEN + RAISE EXCEPTION 'Copied % of % rows; refusing to drop the source columns.', + copied_count, source_count; + END IF; + END $$; + `; + + const CONSTRAIN = ` + ALTER TABLE "legacy_articles_translations" + ADD CONSTRAINT "legacy_articles_translations_pk" + PRIMARY KEY ("itemId", "languageId");--> statement-breakpoint + ALTER TABLE "legacy_articles_translations" + ADD CONSTRAINT "legacy_articles_translations_item_fk" + FOREIGN KEY ("itemId") REFERENCES "legacy_articles"("id") + ON DELETE cascade ON UPDATE cascade;--> statement-breakpoint + ALTER TABLE "legacy_articles_translations" + ADD CONSTRAINT "legacy_articles_translations_language_fk" + FOREIGN KEY ("languageId") REFERENCES "core_languages"("id") + ON DELETE restrict ON UPDATE cascade;--> statement-breakpoint + CREATE UNIQUE INDEX "legacy_articles_translations_language_id_slug_key" + ON "legacy_articles_translations" ("languageId","slug"); + `; + + it("copies every row into the default language, timestamps included", async () => { + const before = await h.sql< + { createdAt: Date; id: number; slug: string; title: string }[] + >` + SELECT "id", "title", "slug", "createdAt" FROM "legacy_articles" ORDER BY "id" + `; + + await migrate(CREATE_TRANSLATIONS); + + const after = await h.sql< + { createdAt: Date; itemId: number; slug: string; title: string }[] + >` + SELECT "itemId", "title", "slug", "createdAt" + FROM "legacy_articles_translations" ORDER BY "itemId" + `; + + expect(after).toHaveLength(before.length); + expect(after.map(row => [row.itemId, row.title, row.slug])).toEqual( + before.map(row => [row.id, row.title, row.slug]), + ); + // The original timestamps travel with the values. A translation stamped + // `now()` would tell every editor the whole collection was rewritten on + // deployment day. + expect(after.map(row => row.createdAt)).toEqual( + before.map(row => row.createdAt), + ); + }); + + it("resolves the language rather than hardcoding an identifier", async () => { + await migrate(CREATE_TRANSLATIONS); + + const [english] = await h.sql<{ id: number }[]>` + SELECT "id" FROM "core_languages" WHERE "code" = 'en' + `; + const rows = await h.sql<{ languageId: number }[]>` + SELECT DISTINCT "languageId" FROM "legacy_articles_translations" + `; + + // A literal `1` is right on the machine it was written on and wrong on + // every other install. + expect(rows).toEqual([{ languageId: english.id }]); + }); + + it("passes its own verification step and only then drops the columns", async () => { + await migrate(CREATE_TRANSLATIONS); + await migrate(VERIFY); + await migrate(CONSTRAIN); + await migrate(` + ALTER TABLE "legacy_articles" DROP COLUMN "title";--> statement-breakpoint + ALTER TABLE "legacy_articles" DROP COLUMN "slug"; + `); + + const columns = (await columnsOf("legacy_articles")).map( + row => row.column_name, + ); + expect(columns).not.toContain("title"); + expect(columns).not.toContain("slug"); + expect(await countOf("legacy_articles_translations")).toBe(3); + }); + + it("aborts rather than dropping the source when a row did not copy", async () => { + // The failure the verification exists for, produced deliberately: one + // source row that the copy missed. + await migrate(CREATE_TRANSLATIONS); + await h.sql` + DELETE FROM "legacy_articles_translations" + WHERE "itemId" = (SELECT min("itemId") FROM "legacy_articles_translations") + `; + + await expect(migrate(VERIFY)).rejects.toThrow( + /refusing to drop the source columns/, + ); + + // And the source is untouched, which is the whole point. + const columns = (await columnsOf("legacy_articles")).map( + row => row.column_name, + ); + expect(columns).toContain("title"); + expect(columns).toContain("slug"); + }); + + it("moves uniqueness from global to per language", async () => { + await migrate(CREATE_TRANSLATIONS); + await migrate(CONSTRAIN); + + const [polish] = await h.sql<{ id: number }[]>` + SELECT "id" FROM "core_languages" WHERE "code" = 'pl' + `; + + // The same slug in another language is now legal - it was not before, + // when the column carried a global unique index. + await h.sql` + INSERT INTO "legacy_articles_translations" + ("itemId", "languageId", "title", "slug") + VALUES ( + (SELECT min("id") FROM "legacy_articles"), ${polish.id}, 'Pierwszy', 'first' + ) + `; + + const code = await pgErrorCode( + async () => + await h.sql` + INSERT INTO "legacy_articles_translations" + ("itemId", "languageId", "title", "slug") + VALUES ( + (SELECT max("id") FROM "legacy_articles"), ${polish.id}, 'Drugi', 'first' + ) + `, + ); + expect(code).toBe("23505"); + }); + + it("surfaces a pre-existing duplicate as a named failure with the data intact", async () => { + // Step 5 is where a collision shows up, deliberately after the copy: the + // rows are still there to look at, rather than half-migrated. + await migrate(CREATE_TRANSLATIONS); + await h.sql` + UPDATE "legacy_articles_translations" SET "slug" = 'first' + WHERE "itemId" = (SELECT max("itemId") FROM "legacy_articles_translations") + `; + + const code = await pgErrorCode(async () => await migrate(CONSTRAIN)); + + expect(code).toBe("23505"); + expect(await countOf("legacy_articles_translations")).toBe(3); + }); + + it("takes the translations with the record it belongs to", async () => { + await migrate(CREATE_TRANSLATIONS); + await migrate(CONSTRAIN); + + await h.sql` + DELETE FROM "legacy_articles" + WHERE "id" = (SELECT min("id") FROM "legacy_articles") + `; + + expect(await countOf("legacy_articles_translations")).toBe(2); + }); + + it("refuses to remove a language that content is written in", async () => { + await migrate(CREATE_TRANSLATIONS); + await migrate(CONSTRAIN); + + const code = await pgErrorCode( + async () => + await h.sql`DELETE FROM "core_languages" WHERE "code" = 'en'`, + ); + + // `ON DELETE restrict`, which Postgres 18 reports as `23001` and earlier + // majors as `23503`. The version decides which is correct rather than the + // assertion accepting either. + expect(code).toBe(h.serverMajor >= 18 ? "23001" : "23503"); + }); + }); + + // ------------------------------------------------------------------------- + // Transactional behaviour + // ------------------------------------------------------------------------- + + describe("a failed migration leaves nothing half-applied", () => { + it("rolls a multi-statement data migration back whole", async () => { + // The migrator wraps a file in a transaction, so this is what an operator + // gets when statement three of four fails: the schema and the data exactly + // as they were. + await expect( + h.sql.begin(async transaction => { + await transaction.unsafe(` + CREATE TABLE "legacy_articles_faq" ( + "id" serial PRIMARY KEY NOT NULL, + "itemId" integer NOT NULL, + "position" integer NOT NULL, + "question" varchar(200) NOT NULL, + "answer" text NOT NULL + ) + `); + await transaction.unsafe(` + INSERT INTO "legacy_articles_faq" ("itemId", "position", "question", "answer") + SELECT "id", 0, 'Q', 'A' FROM "legacy_articles" + `); + await transaction.unsafe( + `ALTER TABLE "legacy_articles_faq" ADD COLUMN "answer" text`, + ); + }), + ).rejects.toThrow(); + + // DDL is transactional in Postgres, so even the `CREATE TABLE` is gone. + const tables = await h.sql<{ table_name: string }[]>` + SELECT table_name FROM information_schema.tables + WHERE table_name = 'legacy_articles_faq' + `; + expect(tables).toEqual([]); + }); + + it("keeps a verification failure and its copy in one transaction", async () => { + await expect( + h.sql.begin(async transaction => { + await transaction.unsafe(` + CREATE TABLE "legacy_articles_translations" ( + "itemId" integer NOT NULL, + "languageId" integer NOT NULL, + "title" varchar(200) NOT NULL, + "slug" varchar(160) NOT NULL + ) + `); + await transaction.unsafe(` + INSERT INTO "legacy_articles_translations" + ("itemId", "languageId", "title", "slug") + SELECT a."id", + (SELECT "id" FROM "core_languages" WHERE "code" = 'en'), + a."title", a."slug" + FROM "legacy_articles" a LIMIT 1 + `); + await transaction.unsafe(` + DO $$ + DECLARE source_count integer; copied_count integer; + BEGIN + SELECT count(*) INTO source_count FROM "legacy_articles"; + SELECT count(*) INTO copied_count FROM "legacy_articles_translations"; + IF source_count <> copied_count THEN + RAISE EXCEPTION 'Copied % of % rows.', copied_count, source_count; + END IF; + END $$; + `); + }), + ).rejects.toThrow(/Copied 1 of 3 rows/); + + const tables = await h.sql<{ table_name: string }[]>` + SELECT table_name FROM information_schema.tables + WHERE table_name = 'legacy_articles_translations' + `; + expect(tables).toEqual([]); + }); + + it("cannot roll back a CREATE INDEX CONCURRENTLY, which is why none is generated", async () => { + // The documented exception: `CONCURRENTLY` cannot run inside a + // transaction at all, so a migration using it is not atomic. Nothing the + // engine generates does, and this pins that. + await expect( + h.sql.begin(async transaction => { + await transaction.unsafe( + `CREATE INDEX CONCURRENTLY "legacy_articles_title_idx" ON "legacy_articles" ("title")`, + ); + }), + ).rejects.toThrow(); + }); + }); +}); From dd173d1d817cd5e140be875090b818597c117198 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:52:49 +0200 Subject: [PATCH 07/14] docs(content): document the production hardening guarantees Six pages under the Content Engine section, each one a promise written next to the test that proves it - because a guarantee nobody can point at is a hope. - **Production hardening** - the guarantees in one table, and the list of things that are deliberately *not* promised, which is where most production surprises come from. - **Concurrency** - the full race matrix, plus the two rules that surprise people: a no-op is never a conflict, and a reorder on an unordered relation does nothing. - **Failure and retries** - what is inside the transaction and what is after it, what each downstream outage costs, and why delivery is at-least-once with no exactly-once claim anywhere. - **Security** - the permission matrix over every generated route, translator isolation, what a public response may contain, and why every bad preview link is the same 404. - **Observability** - the two questions an operator actually asks, and the aggregate route that answers them. - **Performance and scaling** - round trips per page, where cursor pagination is exact and where it is not, and the index behind each generated query. - **Migration hardening** - the invariants every documented upgrade holds, and the one statement PostgreSQL cannot roll back. Also updates `limitations.mdx`, which had two entries this stage changed: the cursor ties it described are gone, and the generated-name check now covers junction and repeatable tables and their constraints. Co-Authored-By: Claude Opus 5 (1M context) --- .../docs/dev/content-engine/concurrency.mdx | 136 +++++++++++++ .../content-engine-observability.mdx | 165 ++++++++++++++++ .../content-engine-security.mdx | 159 +++++++++++++++ .../content-engine/failure-and-retries.mdx | 186 ++++++++++++++++++ .../content/docs/dev/content-engine/index.mdx | 5 + .../docs/dev/content-engine/limitations.mdx | 32 ++- .../content/docs/dev/content-engine/meta.json | 7 + .../content-engine/migration-hardening.mdx | 175 ++++++++++++++++ .../performance-and-scaling.mdx | 167 ++++++++++++++++ .../content-engine/production-hardening.mdx | 106 ++++++++++ 10 files changed, 1132 insertions(+), 6 deletions(-) create mode 100644 apps/docs/content/docs/dev/content-engine/concurrency.mdx create mode 100644 apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx create mode 100644 apps/docs/content/docs/dev/content-engine/content-engine-security.mdx create mode 100644 apps/docs/content/docs/dev/content-engine/failure-and-retries.mdx create mode 100644 apps/docs/content/docs/dev/content-engine/migration-hardening.mdx create mode 100644 apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx create mode 100644 apps/docs/content/docs/dev/content-engine/production-hardening.mdx diff --git a/apps/docs/content/docs/dev/content-engine/concurrency.mdx b/apps/docs/content/docs/dev/content-engine/concurrency.mdx new file mode 100644 index 000000000..716d647d9 --- /dev/null +++ b/apps/docs/content/docs/dev/content-engine/concurrency.mdx @@ -0,0 +1,136 @@ +--- +title: Concurrency +description: Who wins when two writers race, what the loser is told, and why a no-op is never a conflict. +icon: GitFork +--- + +Two editors open the same article. One fixes a typo, the other rewrites the +introduction, and both press save within the same second. Exactly one of them +should win, and the other should be told - clearly enough that the AdminCP can +reload the newer record and offer to overwrite it. + +That is the whole of this page, applied to every mutation the engine has. + +## The mechanism + +Two primitives, and which one applies depends on whether the content type has an +editorial workflow. + +**Optimistic locking**, on an editorial content type. Every write carries the +version the caller read, and the write is a single guarded statement: + +```sql +UPDATE "example_articles" + SET "title" = $1, "version" = "version" + 1 + WHERE "id" = $2 AND "version" = $3 +``` + +There is no read-then-write window, because there is no read. Two racing writers +produce one statement that matches and one that does not; the one that does not +gets a [`CONTENT_VERSION_CONFLICT`](/docs/dev/content-engine/editorial) carrying +both versions. + +**A row lock**, on a content type without one. There is no version column to +guard, so `SELECT ... FOR UPDATE` takes the source row before the collection is +read - which makes two concurrent `add` calls run one after the other, and the +second sees the first's result rather than the state they both started from. + +The lock is the database's, not the process's. A second API instance is +serialised by exactly the same primitive. + +## The matrix + +Every row below is a test on two separate PostgreSQL connections. + +| Race | Outcome | +| ---- | ------- | +| update vs update | One winner, one `CONTENT_VERSION_CONFLICT`. Version moves once, one revision. | +| update vs delete | Delete first: the update answers `null` (a 404). Update first: the delete conflicts. Never a resurrection. | +| update vs publish (same version) | One winner. The loser conflicts. | +| update vs publish (no version) | Both may land. A publish writes `status` only, so no field value is reverted. | +| restore vs update | One winner. A restore never silently overwrites a newer edit. | +| restore vs delete | Delete first: the restore answers `null`. It cannot recreate a removed record. | +| scheduled vs manual publish | The second finds nothing to do and is skipped. One revision, one announcement. | +| PL update vs PL update | One winner, one `CONTENT_TRANSLATION_VERSION_CONFLICT`. | +| PL update vs EN update | **Both win.** Separate version domains, separate rows. | +| PL update vs shared update | **Both win.** The translation and the base row are two rows with two versions. | +| PL delete vs stale PL update | No resurrection, either way round. | +| relation add vs add | One winner. Exactly one junction row, at position 0. | +| relation remove vs add | One winner. Positions stay contiguous from zero. | +| reorder vs add / remove | One real mutation. Positions stay contiguous, targets stay unique. | +| repeatable create vs create | One winner (editorial) or both merged (plain service). | +| child update vs reorder | One winner. Child identity survives either way. | +| child delete vs update | One real mutation, and never a resurrected child. | +| collection vs scalar | One winner. Never one writer's categories under another's version. | +| two different records | **Both win.** The lock is per row. | + +## Two rules that surprise people + +### A no-op is not a conflict + +An editorial write that changes nothing succeeds, bumps no version and leaves no +revision - and it does **not** check `expectedVersion`. There is nothing to +overwrite, so there is nothing to conflict about. An editor who pressed save +twice has not created two versions of anything. + +This reaches further than it looks. `repeatable.update` for a child a concurrent +writer already deleted computes a list identical to the stored one, so it is a +successful no-op rather than a conflict. The invariant worth holding is therefore +**one race, one version increment** - not "one race, one rejected promise". + +### A reorder on an unordered relation does nothing + +`field.relation({ multiple: true })` without `ordered: true` is a *set*. The +engine stores it in ascending target order, so `set([9, 2])` and `set([2, 9])` +are the same state - and `reorder` computes the list that is already stored. + +If a sequence matters, say so: + +```ts +relatedArticles: field.relation({ + multiple: true, + ordered: true, // Now `UNIQUE (itemId, position)` makes the order a fact. + self: true, +}), +``` + +## Why positions never collide + +Rewriting an ordered collection cannot be done in place: moving row A from slot 0 +to slot 1 while row B still sits in slot 1 violates `UNIQUE (itemId, position)` +*during* the statement, even though the final state is fine. + +So every surviving row is first parked at a negative slot - a space no settled +row ever occupies - and one final `UPDATE` maps the whole set back to `0..n-1` at +once. No deferrable constraint, no delete-and-recreate, and every child keeps its +identifier through a reorder. + +## Scheduled transitions + +A booked publication is claimed with `SELECT ... FOR UPDATE`, and the lock is +held from the claim through the transition, its revision, the settlement *and* +the queue row that will announce it. Four conditions are re-read from the +database under that lock rather than trusted from the queue payload: the row +exists, it is still `pending`, its generation matches, and its time has come. + +That is what makes a cancel honest. It either wins outright - before the claim - +or waits, then finds the schedule already `completed` and answers a truthful 404. +There is no window in which an administrator is told the cancel worked and then +watches the article go live anyway. + +A stale booking cannot override newer manual state either, because the transition +guards on the *state* it is leaving: a scheduled publish of an already-published +record changes nothing, writes no revision, and announces nothing. + +## Writing concurrent code against the engine + +Three things worth knowing if you call the services directly: + +1. **Pass `expectedVersion`.** Every editorial write requires it, including a + collection mutation, and that is deliberate - defaulting it would make that the + one write which silently overwrites whatever it finds. +2. **Join the transaction, do not open a second one.** Every service method takes + `{ tx }`. Two transactions on two connections is a deadlock waiting for a + quiet afternoon. +3. **Call the effects after the write returns.** Never inside the transaction + callback - see [failure and retries](/docs/dev/content-engine/failure-and-retries). diff --git a/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx b/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx new file mode 100644 index 000000000..8fa5573f5 --- /dev/null +++ b/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx @@ -0,0 +1,165 @@ +--- +title: Observability +description: The two questions an operator asks at three in the morning, and where the Content Engine answers them. +icon: Activity +--- + +This is not a monitoring product. There is no time series, no Prometheus and no +dashboard, because the install has none of those and the Content Engine is not +the place to introduce them. + +What it does have is answers to the two questions that actually get asked when +something looks wrong. + +## Is the search index telling the truth? + +The database is the source of truth. So "how many documents should there be" is a +`COUNT` over published rows, and "how many are there" is a `COUNT` over +`core_search_index`. A difference is drift. + +```http +GET /api/@vitnode/core/admin/debug/content/status +``` + +```json +{ + "healthy": false, + "contentTypes": [ + { + "contentTypeId": "example.localized-article", + "pluginId": "@vitnode/example", + "features": { + "editorial": true, + "localization": true, + "publicApi": true, + "publication": true, + "scheduling": false, + "search": true + }, + "search": { + "contentTypeId": "example.localized-article", + "healthy": false, + "locales": [ + { "locale": "en", "expected": 421, "indexed": 421, "healthy": true }, + { "locale": "pl", "expected": 421, "indexed": 419, "healthy": false } + ] + }, + "schedules": null + } + ] +} +``` + +Two aggregates per content type, whatever its size, so this is safe to open on an +install with a large table. + + +The existing search status page reports what is *in* the index, measured against +the indexer's own count. A collection can read as fully covered there and still +be missing every Polish document. This measures against the **rows**. + + +Three things it can tell you: + +- **`indexed` below `expected`** - a live synchronisation that threw, or a + rebuild that stopped part-way. Fix the engine, then rebuild. +- **`indexed` above `expected`** - documents that outlived their records. Reported + as measured rather than clamped, because a stale document is exactly what needs + seeing. +- **One locale healthy and another not** - the shape a half-finished localized + rebuild makes, and the one a single total cannot show. + +The expected side uses the **same** `publishedCondition` the indexer uses. Re- +deriving "published" here would let the diagnostic disagree with the thing it is +diagnosing, which is the one way a health check is worse than none. + +### Repairing drift + +```http +POST /api/@vitnode/core/admin/debug/search/rebuild +{ "itemType": "example.localized-article" } +``` + +Scoped to one collection so a single-collection reindex never wipes the rest of +the index. A rebuild reproduces exactly what live synchronisation would have +written, so repairing drift never changes what search returns beyond making it +correct. + +## Did anything scheduled fail to announce itself? + +A scheduled transition that committed but whose event, index write or cache +expiry did not is recorded on the schedule row - and counted per content type on +the same status route: + +```json +"schedules": { "pending": 3, "withErrors": 0, "failedEffects": 1 } +``` + +| Field | Means | +| ----- | ----- | +| `pending` | Bookings still waiting to fire. | +| `withErrors` | Pending bookings whose last run threw. The transition has not happened. | +| `failedEffects` | Transitions that **did** happen and were never announced. | + +`failedEffects` is the one worth alerting on: the record *is* published, and +nobody has been told. The effects task retries on the queue's backoff, so a +non-zero value that stays non-zero is an outage rather than a blip. Per booking, +the AdminCP schedule panel shows the reason. + +## The logs + +Two greppable prefixes in `core_logs`, visible in AdminCP → Advanced → Logs. + +| Prefix | Written when | +| ------ | ------------ | +| `[content-search]` | An index write or delete threw after a committed mutation. | +| `[content-effects]` | An event was emitted and a listener did not receive it. | +| `[content-revalidate]` | A web origin refused or could not be reached. | + +Each is a single JSON object behind the prefix, with the content type, the item, +the operation and the underlying error - enough to find the record without +correlating three systems by timestamp. + +Deliberately **not** logged as errors: a `404`, a no-op mutation, and a draft that +was not indexed. All three are the engine working, and a log full of them is a log +nobody reads. + +## Using the diagnostics from code + +Everything the route returns is available directly, so a plugin can build its own +panel or a cron job can alert on it: + +```ts +import { + contentEngineDiagnostics, + contentScheduleHealth, + contentSearchDrift, +} from "@vitnode/core/content/server"; + +// Everything, sorted by content type id so two calls - and two processes - +// report the same order. +const report = await contentEngineDiagnostics(c); + +// Or one content type at a time. +const drift = await contentSearchDrift(c, { model: articleContent }); +if (!drift.healthy) { + await c.get("log").error(`[my-plugin] ${JSON.stringify(drift.locales)}`); +} + +// Or just the schedules, in one grouped query for many content types. +const schedules = await contentScheduleHealth(c, ["example.article"]); +``` + +The route is behind `system: can_view`, like the rest of the debug surface. There +is nothing in the response an administrator could not read elsewhere, but there is +nothing in it a visitor should read either. + +## What this deliberately does not do + +- **It does not repair anything.** Drift is detected by counting; correcting it is + a rebuild, and a rebuild is a decision an operator makes. +- **It does not compare documents.** A count is not a checksum: two stale + documents still count as two. It catches the failure that actually happens - a + write that threw, a rebuild that stopped - for the price of two aggregates. +- **It keeps no history.** Every number is computed on demand. Trending them over + time is what a monitoring system is for, and this is not one. diff --git a/apps/docs/content/docs/dev/content-engine/content-engine-security.mdx b/apps/docs/content/docs/dev/content-engine/content-engine-security.mdx new file mode 100644 index 000000000..dc93da8cb --- /dev/null +++ b/apps/docs/content/docs/dev/content-engine/content-engine-security.mdx @@ -0,0 +1,159 @@ +--- +title: Security +description: The permission matrix over every generated route, what a public response may contain, and why every bad preview link is the same 404. +icon: Lock +--- + +Three questions, and the engine answers each of them in exactly one place so +there is no second implementation to disagree with the first: + +1. **Who may do this?** A staff permission on every generated route. +2. **What may they see?** An allowlist, applied where the `SELECT` is built. +3. **Who is this anonymous caller?** A signed token, or nobody. + +## The permission matrix + +Every generated route carries an explicit `adminStaffPermission`, checked before +validation - so a request without the permission is a `403` whatever its body +says. The matrix is enumerated from the route builder in the test suite, which is +what stops a new endpoint joining the set without one. + +| Route | Permission | +| ----- | ---------- | +| `GET /` | `can_view` | +| `GET /{id}` | `can_view` | +| `GET /options/{field}` | `can_view` | +| `POST /` | `can_create` | +| `PUT /{id}` | `can_edit` | +| `DELETE /{id}` | `can_delete` | +| `POST /{id}/publish` · `/unpublish` | `can_publish` | +| `GET /{id}/revisions` · `/{revisionId}` | `can_view` | +| `POST /{id}/revisions/{revisionId}/restore` | `can_restore` | +| `POST /{id}/preview` | `can_view` | +| `GET /{id}/schedules` | `can_view` | +| `POST /{id}/schedule` · `/{scheduleId}/cancel` | `can_publish` | +| `GET /{id}/translations` · `/{locale}` · `/public-locales` | `can_view` | +| `POST` · `PUT /{id}/translations/{locale}` | `can_translate` | +| `DELETE /{id}/translations/{locale}` | `can_delete` | +| `POST /{id}/translations/{locale}/publish` · `/unpublish` | `can_publish` | +| `GET /{id}/translations/{locale}/revisions` · `/{revisionId}` | `can_view` | +| `POST /{id}/translations/{locale}/revisions/{revisionId}/restore` | `can_restore` | + +Two choices in there are worth the sentence they take: + +- **`can_publish` for scheduling.** Booking a publication *is* publishing, just + later. A role trusted to write drafts is not automatically trusted to put one + on the internet at 9am on Monday. +- **`can_restore` depends on `can_edit`.** Restoring rewrites many fields at once + from a source the editor did not type, so somebody who may not edit must not + reach the same outcome through the history. + +## The translator + +`can_translate` depends on `can_view` and deliberately **not** on `can_edit`, +which is what makes "writes Polish and nothing else" expressible. Give a role +`can_view + can_translate` and it can: + +- read the record, every locale tab and every locale's history; +- create and edit a translation in any enabled locale. + +It cannot: + +- edit a shared field (`PUT /{id}` is `can_edit`); +- publish or unpublish anything, record or translation (`can_publish`); +- restore a shared revision *or* a locale's own (`can_restore`, which needs + `can_edit`); +- delete the record or a translation (`can_delete`). + +Existing roles simply do not have `can_translate` - permissions are stored as +JSON per role, so a new one denies by default and needs no migration. + +## Advanced collections have no second door + +There is no per-relation or per-repeatable endpoint. A collection is written +through the ordinary `PUT /{id}`, gated on `can_edit`, in the same guarded +transaction as a field edit - so there is no route that hands somebody a write +primitive they could not have used anyway. + +The relation picker (`GET /options/{field}`) is a read of display labels, gated on +`can_view` like every other read, and it writes nothing. + +## Cross-plugin isolation + +Two plugins can name a permission module the same thing - `articles` is not an +unusual choice - and the registry allows it precisely because the plugin id is +part of the key. That only holds because the **route** carries its own plugin id +into the check rather than reading whichever plugin is handling the request. + +Underneath, everything shared is scoped the same way. The revisions table and the +schedules table are shared by every content type in the install, so every +statement filters on `pluginId`, `contentTypeId` *and* `itemId`. A revision id on +its own proves nothing about ownership, and cancelling somebody else's +publication would be a strange way to find that out. + +## Public responses are an allowlist + +`publicApi.fields` is not a filter applied to a full row. It decides the `SELECT` +itself, so a private column is never fetched - which means it cannot be leaked by +a mistake further downstream. + +What a public response contains, exactly: + +- the fields the allowlist names, and nothing else; +- a group carrying only the **leaves** the allowlist named - exposing `seo.title` + does not expose `seo.robots`; +- a repeatable child carrying its public leaves plus its `id`; +- a to-many relation as **identifiers**, never expanded rows - a target has its + own public API, its own allowlist and its own publication state; +- `locale`, on a localized content type, because a response has to say which + language it is. + +What it never contains: `id` unless the allowlist names it, `status`, `version`, +`createdAt`, `updatedAt`, `languageId`, any revision metadata, any private leaf, +any private collection, and any flattened storage column name (`seo.title` is +stored as `seoTitle`, and a response that carried that would publish an internal +detail *and* give a client two spellings of one value). + +## Preview fails closed + +A preview link is an unpublished record behind a short-lived credential, so every +part of it is written to fail rather than to be helpful. + +- **The token is the authorization.** HMAC-SHA256, bound to one plugin, one + content type, one record, one revision and - for a localized content type - one + locale. No session is consulted, which is the point: a reviewer has no account. +- **Every failure is the same 404.** A forged signature, an expired link, a token + for another record, a deleted revision and a record that never existed are + indistinguishable. A 401 or a 403 would confirm the record exists, which is + precisely what a draft URL must not do. +- **No fallback.** A `pl` link opened on the English URL is a 404, not the English + copy. Falling back would hand a reviewer a different language from the one they + were sent. +- **A weak secret disables it entirely.** An install whose `CONTENT_PREVIEW_SECRET` + is missing, too short, or still the published placeholder can have its tokens + forged by anyone, so no token is honoured - and the answer is the same 404, + because "preview is misconfigured here" is not something an anonymous request + needs to learn. +- **Nothing caches it.** `Cache-Control: private, no-store` and + `X-Robots-Tag: noindex, nofollow`, and the response carries no cache tag at all. +- **The projection is the public one.** The same function the detail route uses, + so a field cannot be private on one and public on the other. + +## Cache keys cannot mix public and private + +An AdminCP read forwards the staff cookie and is never cached - no `force-cache`, +no tags. A preview is `no-store`. Only the generated public read is cached, and +only under tags built from `publicApi.path`. + +Locale spellings are normalised where the tag is built, so `pl`, `PL` and `" pl "` +address one cache entry and expire together - a tag is a string comparison, and +hoping every call site agrees on casing is not a plan. + +## Errors say nothing they should not + +A driver error carries the constraint name, often the column, and sometimes the +value that clashed. None of it reaches a response body; every expected failure is +mapped to a status and, where a client has to branch, a stable code. See +[the error contracts](/docs/dev/content-engine/editorial). Anything unrecognised +becomes a bare `500` - the detail goes to `core_logs`, and production answers +`Internal Server Error` and nothing else. diff --git a/apps/docs/content/docs/dev/content-engine/failure-and-retries.mdx b/apps/docs/content/docs/dev/content-engine/failure-and-retries.mdx new file mode 100644 index 000000000..b9394ab75 --- /dev/null +++ b/apps/docs/content/docs/dev/content-engine/failure-and-retries.mdx @@ -0,0 +1,186 @@ +--- +title: Failure and retries +description: What survives an outage, what gets retried, and why event delivery is at-least-once rather than exactly-once. +icon: TriangleAlert +--- + +A content mutation touches four systems, and only one of them can be rolled +back. The database write either committed or it did not. The event, the search +document and the cache invalidation are calls to things that can be down for a +minute - and once the transaction has closed, nothing can un-send them. + +So the engine draws a hard line, and everything on this page follows from it. + +## Inside the transaction, and after it + +```text +inside the transaction after the commit +------------------------ ---------------------------- +the row lock the event +validation that needs locked state the search document +the content write the cache invalidation +the version increment the remote revalidation +the revision insert +the schedule transition +the collection rows +the queue row that announces it +``` + +Two things about that split are worth stating outright. + +**Nothing external is inside.** No `fetch`, no search call, no `next/cache`. A +rolled-back transaction cannot un-emit an event, so an event emitted inside one +is a lie waiting to happen. + +**The queue row is inside.** The task that announces a scheduled publication is +written in the same transaction as the publication, so it exists if and only if +the transition committed. A crash a millisecond later loses nothing: the row is +durable, and the queue will drain it. + + +`syncContentSearch`, `contentEditorialEffects` and `revalidateContent` must be +called **after** the write has returned, never inside a `db.transaction()` +callback. The generated routes already do this; hand-written code has to. + + +## When the event transport fails + +`EventsModel.emit` reports rather than throws, so `failures` is the only place a +dead listener or a broker outage is visible at all. + +The write has already committed by then, and two things follow: + +1. **The request still succeeds.** Answering 500 would tell the client its edit + was lost when it was not, and invite a retry that creates a second version of + everything. +2. **The failure is never swallowed.** It goes to `core_logs` behind + `[content-effects]` with the content type, the item and the listener that + failed: + +```text +[content-effects] {"action":"published","contentTypeId":"example.article", +"delivered":0,"eventId":"...","failures":[{"error":"Service unavailable", +"listener":"@acme/plugin:notifications:send-notification"}],"itemId":7} +``` + +Search it in AdminCP → Advanced → Logs. On the scheduled path the same failure is +also written onto the schedule row as `effectsError`, and retried. + +## When the search engine fails + +The index write throws, the mutation stays committed, and the failure is +reported three ways: on the outcome (`ContentSearchSyncOutcome.error`), in +`core_logs` behind `[content-search]`, and - for a scheduled transition - on the +schedule row. + +The index is eventually consistent, and "eventually" is bounded by two things: + +- **the next write** that touches an indexed field on that record, which + rewrites the document; +- **a rebuild**, which rewrites all of them. + +A rebuild reproduces exactly what live synchronisation would have written - +that equality is itself a test - so repairing drift never changes what search +returns beyond making it correct. + +## When a web origin refuses its cache invalidation + +The [revalidation bridge](/docs/dev/content-engine/caching) posts to every +configured web origin, retries each one twice, and **reports rather than +throws**. The caller decides what counts as delivered, and the scheduled effects +task requires *all* of them: + +```text +attempted: 0 nothing needed telling. Not a failure. +delivered === attempted delivered. +delivered < attempted a partial. The run fails and is retried. +``` + +A partial is the dangerous case, not the acceptable one. With two web apps behind +one API, one of them accepting an unpublish while the other does not leaves the +withdrawn page cached and readable - and "at least one worked" would call that a +success and never try the other again. + +## Retrying the announcements without republishing + +A scheduled publication is deliberately **two units of work**: + +```text +content-schedule moves the database. Commits, or does not. +content-schedule-effects announces what committed. Retried on its own. +``` + +Retrying them together would re-run the publish - which is idempotent, so the +second run would find nothing changed and skip the announcements entirely. That +is exactly how a scheduled unpublish ends up permanently serving a cached page it +should have expired, and splitting them is what removes it. + +So a retry re-announces and never re-transitions. After a failed effects run and +a successful retry, the record holds the same version, the same `publishedAt` and +exactly one `publish` revision. + +Every reason is reported, not just the first: + +```text +Scheduled publish of example.article#7 committed, but its effects did not +(event: @acme/plugin:notifications:send (down); search: index refused; + cache: 1/2 web origins accepted the invalidation). +``` + +An operator who fixes one system and retries should not discover the next one on +the following run. + +## At-least-once, and what to do about it + + +A retried effects run emits its event again. A listener can see the same +`content.example.article.published` twice, and there is no outbox and no +deduplication in the transport. + + +Search and cache are idempotent by construction - an upsert and an expiry are the +same operation however many times they run, and the document a retry writes is +byte-identical. Events are not, because a listener can do anything. + +A listener that must act once keys off the identifiers the payload carries: + +```ts +buildEventListener({ + name: "announce-on-slack", + event: "content.example.article.published", + handler: async (c, { payload }) => { + // Present only when a schedule caused this, and stable across every retry + // of that booking. + if (payload.scheduleId && (await alreadyAnnounced(c, payload.scheduleId))) { + return; + } + // ... + }, +}); +``` + +## Idempotency, everywhere else + +| Operation | Repeating it | +| --------- | ------------ | +| publish / unpublish | A no-op. No write, no version bump, no revision, no event, no index write. | +| restore to the current values | A no-op, with `revisionId: null`. | +| relation `add` of a target already present | A no-op. | +| relation `remove` of one that is not there | A no-op. | +| repeatable `set` to the same rows and order | A no-op. | +| search upsert | The same document, rewritten. | +| search delete | Removes nothing that is not there. | +| cache expiry | Expires an already-expired tag. | +| scheduled task delivered twice | The claim refuses anything not still `pending`. | +| effects task retried | Re-announces; never re-transitions. | + +That list falls out of one decision rather than ten special cases: every +collection mutation computes the whole next state and diffs it against the +stored one, so "nothing moved" is a single check that every operation shares. + +## What is left outstanding, and where to see it + +- **Per record:** the AdminCP schedule panel shows `effectsError` on the booking. +- **Per install:** `GET /admin/debug/content/status` counts outstanding + scheduled-effect failures per content type, alongside search index drift. See + [observability](/docs/dev/content-engine/content-engine-observability). diff --git a/apps/docs/content/docs/dev/content-engine/index.mdx b/apps/docs/content/docs/dev/content-engine/index.mdx index 0eaa0e221..3a8158ce7 100644 --- a/apps/docs/content/docs/dev/content-engine/index.mdx +++ b/apps/docs/content/docs/dev/content-engine/index.mdx @@ -112,6 +112,11 @@ one object instead of two that drift apart. title="Database & migrations" description="How Drizzle Kit finds generated tables - and what happens when you rename a field." /> + The destructive statement lives in a different migration from the copy. + +Two files, because between them somebody has to look at the counts. A backfill +that silently dropped three rows and then deleted its source is not something +anybody can notice afterwards - the evidence went with the source. + +That is why the to-one → to-many pattern is two migrations, why the JSON array → +repeatable pattern is two, and why the localization pattern has a verification +step that aborts rather than continuing. + +## The invariants + +Every upgrade in the test suite checks the same six things. Yours should too. + +| Invariant | How to check it | +| --------- | --------------- | +| **Row count** | `count(*)` on the source and the destination, before the drop. | +| **Nullability** | Every new `NOT NULL` column has a default, or the `ALTER` fails on a populated table. | +| **Foreign keys** | Add them *after* the copy, so a bad reference is a named failure with the data intact. | +| **Uniqueness** | Same: a pre-existing duplicate surfaces at the index, not halfway through a backfill. | +| **Order** | `WITH ORDINALITY` for a JSON array; `position` starting at zero, which is where the engine reads from. | +| **Source intact** | The old column is still there when the first migration ends. | + +## What PostgreSQL gives you for free + +DDL is transactional. A migration file that fails on its third statement leaves +**nothing** behind - not the rows it inserted, and not the table it created. So a +failed migration is always safe to fix and re-run, and the verification step can +`RAISE EXCEPTION` knowing it will take the whole file with it: + +```sql +DO $$ +DECLARE source_count integer; copied_count integer; +BEGIN + SELECT count(*) INTO source_count FROM "example_articles"; + SELECT count(*) INTO copied_count FROM "example_articles_translations"; + + IF source_count <> copied_count THEN + RAISE EXCEPTION 'Copied % of % rows; refusing to drop the source columns.', + copied_count, source_count; + END IF; +END $$; +``` + + +`CREATE INDEX CONCURRENTLY` cannot run inside a transaction at all, so a +migration that uses one is **not** atomic - a failure leaves an invalid index +behind, and you have to drop it by hand. Nothing the Content Engine generates +uses `CONCURRENTLY`; if you add one for a large table, put it in a migration of +its own and know what it costs. + + +## The upgrades that are covered + +Each of these runs against a populated table in +`plugins/example/src/database/migration-postgres.test.ts`. + +### Adding a structured group + +Additive and safe, because `defineContentType` already requires every leaf of an +optional group to be nullable or defaulted: + +```sql +ALTER TABLE "example_articles" + ADD COLUMN "syndicationIndexable" boolean DEFAULT true NOT NULL, + ADD COLUMN "syndicationPriority" integer DEFAULT 5 NOT NULL; +``` + +Every existing row gets the default. A `NOT NULL` column with no default is a +`23502` on a table with rows in it - which is the same rule the definition +enforces, arrived at from the other side. + +### Regrouping columns you already have + +Moving `seoTitle` and `seoDescription` into a `seo` group needs **no data +migration**: `field.text()` named `seoTitle` and `seo.title` compile to the same +column. Generate the migration and check it is empty before believing it. + +### To-one → to-many + +Copy at position 0, verify, then drop in a second migration: + +```sql +INSERT INTO "example_articles_categories" ("itemId", "relatedItemId", "position") +SELECT "id", "category", 0 FROM "example_articles" WHERE "category" IS NOT NULL; +``` + +The junction's foreign key goes on afterwards, so a stale reference is a `23503` +you can look at rather than a half-finished copy. + +### JSON array → repeatable + +`WITH ORDINALITY` is what carries the order across, and `ordinality - 1` is what +puts it where the engine reads from: + +```sql +INSERT INTO "example_articles_faq" ("itemId", "position", "question", "answer") +SELECT a."id", entry.ordinality - 1, + entry.value ->> 'question', entry.value ->> 'answer' +FROM "example_articles" a, + jsonb_array_elements(a."faqJson") WITH ORDINALITY AS entry(value, ordinality) +WHERE a."faqJson" IS NOT NULL; +``` + +Get that wrong and nothing fails - somebody's FAQ is silently reordered. + +### Non-localized → localized + +The longest one, and the only one with a mandatory verification step. Six stages: +create the table, resolve the language id, copy, **verify**, constrain, drop. + +Two details the tests pin: + +- **The timestamps travel with the values.** A translation stamped `now()` would + tell every editor the whole collection was rewritten on deployment day. +- **The language is resolved, not hardcoded.** A literal `1` is right on the + machine it was written on and wrong on every other install. + +And one consequence worth expecting: uniqueness moves from *global* to *per +language*. The base table's old unique slug index goes away with the column, and +`/en/about` and `/pl/about` become two legal rows. + +## Before you run it anywhere real + +1. **Against a copy of production first.** The verification step turns a data + loss into a failed migration, which is the whole point - but a failed + migration is still better discovered on a copy. +2. **Check the counts by hand between the two files.** That pause is the entire + safety mechanism. Automating it away is a product decision made by a script. +3. **Take constraint and index names from the generated file.** They are derived + from your table name and clamped to 63 characters; a name that does not match + makes every future diff noisy. +4. **Never regenerate an applied migration.** It bumps the journal's `when` and + the migrator replays the whole file, which fails on `relation already exists` + at best. Add a new one. + +## Testing your own migration + +Copy the shape of the example suite: build the old table, put rows in it that are +awkward on purpose, run your migration script, and assert the invariants before +the destructive step. + +```ts +it("copies every entry and preserves its order", async () => { + await migrate(CREATE_CHILD); + + const [{ expected }] = await sql` + SELECT coalesce(sum(jsonb_array_length("faqJson")), 0)::int AS expected + FROM "legacy_articles" + `; + expect(await countOf("legacy_articles_faq")).toBe(expected); +}); +``` + +The awkward rows are the ones worth having: a `NULL` array, a duplicate that will +collide with the new unique index, a row whose title normalises to nothing. Every +one of those has been a real migration bug in something, somewhere. diff --git a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx new file mode 100644 index 000000000..2eb73b310 --- /dev/null +++ b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx @@ -0,0 +1,167 @@ +--- +title: Performance and scaling +description: What a page costs in round trips, why that number does not move when the page grows, and where cursor pagination is exact. +icon: Gauge +--- + +None of the numbers here are milliseconds. A wall-clock figure says more about +the machine than about the code, and a test built on one fails on a busy CI +runner for reasons nobody can act on. + +What is measured instead is **algorithmic**: how many round trips one page costs, +whether that number moves when the page grows, and whether a lookup seeks on an +index or reads the whole table. Those are properties of the SQL, and they are the +ones that decide whether a collection stays usable at a hundred thousand rows. + +## A page costs a bounded number of queries + +The invariant an N+1 breaks is not "few queries" - it is "the *same* number of +queries for five rows and for sixty". + +| Read | Round trips per page | +| ---- | -------------------- | +| Admin list, labels included | bounded - one count, one select, joins inside it | +| Public list | bounded | +| Localized public list | bounded, whatever the locale | +| Public list with advanced collections | bounded, **plus one batch per exposed collection** | +| Revision history page | bounded | +| Search rebuild page | bounded | + +Two things make that hold. + +**Labels are joined, not looked up.** One `LEFT JOIN` per reference field +resolves every display name in the same round trip - there is no per-row lookup +anywhere in the list path. + +**Collections are batched by parent id.** A to-many relation is deliberately +absent from `ContentSelect` for exactly this reason: a list that carried one +would issue a query per row, and a table of 25 rows with two collections would be +50 round trips. `loadMany` takes the whole page's ids and issues one query per +collection field. + +## Only what the projection asks for + +A public read loads the collections the allowlist actually exposes, and no +others. `relatedArticles` is private on the example content type, so a public +list never touches its junction table at all - querying it to discard the rows +afterwards is work with no answer attached. + +The same rule runs through search: `contentSearchAdvancedValues` loads only the +collections the search configuration names. A content type that indexes none - +every Stage 1-5 one - pays a boolean check and no query. + +## Shared collections are loaded once, not once per locale + +The localized rebuild emits one document per published translation, so a record +with three languages appears three times on a page. Its FAQ is *shared*, and +loading it three times would be an N+1 hiding behind a perfectly correct result. + +So the batch loader deduplicates parent ids before it queries. Three translations +of one record read the FAQ once. + +## Cursor pagination + +The cursor is the primary key, and pages are keyset reads rather than `OFFSET` - +so a page deep into a large collection seeks on the index instead of counting +past every earlier row. + +**What is guaranteed:** + +- a row already returned never comes back; +- the cursor always advances, so a loop always terminates; +- deleting everything past the cursor ends the walk rather than looping; +- ordering is **total** - the primary key is appended as a tiebreaker, so rows + sharing an `updatedAt` are not free to move between the two queries a page turn + issues; +- a public page is capped at the server's ceiling however large a `first` an + anonymous caller sends. + +**What is not:** + + +Rows inserted behind the cursor are not seen until the next pass. That is the +trade every keyset pagination makes, and it is the right one - the alternative is +holding a transaction open across requests. + + + +The cursor **is** the identifier. Ordering by a column with no relationship to it +- a title, say - cannot page exactly: rows whose identifiers fall on the wrong +side of the cursor are skipped. Every generated default (`updatedAt`, +`publishedAt`, `createdAt`, `id`) moves with the identifier and is exact; a +custom `orderableFields` entry that does not is a deep-pagination hazard. + + +## Indexes + +Everything a generated query needs is generated with it, and the definitions are +asserted rather than assumed: + +| Query | Index | +| ----- | ----- | +| `findById` | the primary key | +| `findBySlug` | the slug's unique index | +| localized slug lookup | `UNIQUE (languageId, slug)` on the translation table | +| public list | `(status, publishedAt)` from the publication block | +| revision history | `UNIQUE (contentTypeId, itemId, version)` | +| translation history | the partial unique index on `(…, languageId, version)` | +| relation membership `EXISTS` | the junction's primary key `(itemId, relatedItemId)` | +| reverse relation lookup | `(relatedItemId)` - Postgres does not index the child side of a foreign key on its own | +| repeatable `loadMany` | `UNIQUE (itemId, position)` | +| schedule claims | `(status, scheduledFor)` and `(contentTypeId, itemId)` | + +The plans are checked too, on a table large enough for the planner to have a real +choice: a slug lookup and an identifier lookup both seek, and a revision history +page does not scan. Nothing asserts a *cost* - planner costs move between majors +for reasons that are none of the engine's business. + +## Adding an index of your own + +```ts +indexes: [ + { on: ["status", "createdAt"] }, + // Leaf paths work, and compile against the generated column. + { on: ["syndication.priority"] }, +], +``` + +Names are derived from the table and clamped to Postgres' 63 characters with a +fingerprint, and a collision anywhere in the schema is a **boot failure** rather +than a migration that fails at deploy time. That includes the names nobody wrote +down: a junction's primary key, its position constraint, its target index and a +repeatable's position constraint. + +## Reads stay page-bound + +A page is a page whatever the table holds behind it: `first: 25` materialises 25 +rows, and `totalCount` is an aggregate rather than a fetch. Nothing in the list +path reads a collection whole. + +The one read that deliberately loads everything is `findDetail`, which is what an +edit form needs - one record, with its collections attached. A list must never +call it, and does not. + +## Where the limits actually are + +- **A repeatable is a handful of rows a person edits in one form.** The ceiling + is 200 and the default is 50; model a content type for anything larger. +- **A to-many relation is capped** for the same reason - it is edited as a set in + a picker, not paged. +- **A rebuild is a queue task**, paged, and safe to run against a large table. + It is not free; run it when you need it rather than on a schedule. +- **`totalCount` is a `COUNT(*)`** on every list page. On a very large table with + a filter that no index covers, that is the expensive part of the request - and + the fix is an index on the filter, not a change to the pagination. + +## Running the scale suite + +```bash +DATABASE_TEST_URL=postgres://postgres:postgres@localhost:5432/vitnode_test \ + pnpm --filter @vitnode/example test +``` + +`performance-postgres.test.ts` seeds a few thousand rows rather than the ten +thousand a plan might suggest, because every property above is visible at any size +above "a handful" - and a suite nobody waits for is a suite nobody runs. Where +scale genuinely matters, because a sequential scan is cheaper than an index on a +tiny table, the fixture is grown until the planner has a real choice to make. diff --git a/apps/docs/content/docs/dev/content-engine/production-hardening.mdx b/apps/docs/content/docs/dev/content-engine/production-hardening.mdx new file mode 100644 index 000000000..602e0ae7f --- /dev/null +++ b/apps/docs/content/docs/dev/content-engine/production-hardening.mdx @@ -0,0 +1,106 @@ +--- +title: Production hardening +description: What the Content Engine guarantees under concurrency, partial failure and scale - and, just as importantly, what it does not. +icon: ShieldCheck +--- + +Stages 1-6 built the Content Engine. This page is the other half of the story: +what happens when two editors press save at the same second, when the search +engine is down, when a table has a hundred thousand rows in it, and when the +definition a revision was written under no longer exists. + +None of it is new functionality. It is the set of promises the engine makes, each +one written down next to the test that proves it - because a guarantee nobody +can point at is a hope. + +## The promises, in one place + +| Area | Guarantee | +| ---- | --------- | +| **Concurrency** | Two writers carrying the same `expectedVersion` produce exactly one winner and one structured conflict. | +| **Atomicity** | The content write, its version bump, its revision and its collection rows are one transaction. | +| **Effects** | Events, search and cache invalidation run **after** the commit and can never undo it. | +| **Delivery** | At-least-once where a retry is involved, best-effort otherwise. Never exactly-once. | +| **Search** | The database is the source of truth. Live synchronisation and a rebuild produce the same document. | +| **Cache** | A mutation expires exactly the tags it touched, per locale, and nothing global. | +| **Security** | Every generated route carries a staff permission; a public response is an allowlist, not a filter. | +| **Errors** | Every expected constraint failure has a stable status and code. No SQLSTATE reaches a client. | +| **Revisions** | History is append-only and per-record monotonic, across deletes and recreations. | +| **Boot** | A generated table, index or constraint name that would collide is a boot failure, not a Tuesday. | + +Each one has a page of its own: + +- [Concurrency](/docs/dev/content-engine/concurrency) - the race matrix and who wins. +- [Failure and retries](/docs/dev/content-engine/failure-and-retries) - what survives an outage, and what gets retried. +- [Security](/docs/dev/content-engine/content-engine-security) - permissions, privacy, preview. +- [Observability](/docs/dev/content-engine/content-engine-observability) - what an operator can diagnose. +- [Migration hardening](/docs/dev/content-engine/migration-hardening) - upgrading an install that has rows in it. +- [Performance and scaling](/docs/dev/content-engine/performance-and-scaling) - pagination, query counts, N+1. + +## What is *not* promised + +Worth reading before the rest, because most production surprises come from a +guarantee somebody assumed rather than one that broke. + + +There is no outbox. A retried scheduled effect re-emits its event, so a listener +can see the same `published` twice. Payloads carry a `scheduleId` precisely so a +listener that must act once can key off it. See +[failure and retries](/docs/dev/content-engine/failure-and-retries). + + +- **No frozen dataset across pages.** A cursor is a position in an ordering, not + a snapshot. Rows inserted behind the cursor are not seen; rows already returned + never come back. +- **No cross-request transaction.** A mutation is atomic. A *workflow* made of + several mutations is not, and nothing here pretends otherwise. +- **No automatic schema migration.** Migrations are generated by `drizzle-kit` + and committed. The engine creates no schema at runtime. +- **No field-level permissions.** Permissions are per content type and per + operation. A translator is limited by *locale*, not by field. +- **No repair of a search index that is merely stale.** Drift is *detected* by + counting; correcting it is a rebuild, which is a decision an operator makes. + +## Running the hardening suites + +Everything on these pages is tested against a real PostgreSQL, because none of +it can be shown with a mock - a lock wait, a guarded `UPDATE` that matches +nothing and a `DELETE` that commits mid-transaction are all database behaviour. + +```bash +DATABASE_TEST_URL=postgres://postgres:postgres@localhost:5432/vitnode_test \ + pnpm --filter @vitnode/example test +``` + + +The suites drop and recreate the schema, and refuse to start unless the database +name contains "test". + + +The files, and what each one is for: + +| File | Covers | +| ---- | ------ | +| `concurrency-postgres.test.ts` | Every race in the matrix, on two connections. | +| `resilience-postgres.test.ts` | Event, search and cache failures; idempotency; drift. | +| `integrity-postgres.test.ts` | Delete cascades, schema-evolution restore, disabled locales. | +| `performance-postgres.test.ts` | Pagination, query counts, index use, batch loading. | +| `migration-postgres.test.ts` | The documented upgrade patterns, against real rows. | + +Without `DATABASE_TEST_URL` they skip, loudly, rather than passing quietly. + +## Supported PostgreSQL versions + +PostgreSQL 17 and 18 are both supported, and the suites are **version-aware** +rather than lenient. The one difference that reaches an API contract: + +```text +ON DELETE RESTRICT violated + PostgreSQL 18 and later -> SQLSTATE 23001 (restrict_violation) + earlier majors -> SQLSTATE 23503 (foreign_key_violation) +``` + +Both map to the same `409`, so upgrading the database does not change what a +client sees. The tests assert the *correct* code for the server they are running +against - "one of these two" would still pass if a future major stopped refusing +the delete entirely. From b1c942cb2c9773e43f81ace6bee8f11522a1741d Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 19:59:20 +0200 Subject: [PATCH 08/14] style(content): apply prettier to the new test suites Co-Authored-By: Claude Opus 5 (1M context) --- .../src/content/server/openapi-parity.test.ts | 23 ++++++++++--------- .../content/server/permission-matrix.test.ts | 3 ++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/vitnode/src/content/server/openapi-parity.test.ts b/packages/vitnode/src/content/server/openapi-parity.test.ts index b57d0bde7..b2179e1c3 100644 --- a/packages/vitnode/src/content/server/openapi-parity.test.ts +++ b/packages/vitnode/src/content/server/openapi-parity.test.ts @@ -768,17 +768,18 @@ describe("public routes match their OpenAPI document", () => { ).mockReturnValue(() => ({ findById: async () => await Promise.resolve(findById), findBySlug: async () => await Promise.resolve(findBySlug), - findMany: async () => await Promise.resolve({ - edges: [publicRow], - pageInfo: { - count: 1, - endCursor: 7, - hasNextPage: false, - hasPreviousPage: false, - startCursor: 7, - totalCount: 1, - }, - }), + findMany: async () => + await Promise.resolve({ + edges: [publicRow], + pageInfo: { + count: 1, + endCursor: 7, + hasNextPage: false, + hasPreviousPage: false, + startCursor: 7, + totalCount: 1, + }, + }), })); return mount(buildContentPublicRoutes(posts, { pluginId: PLUGIN_ID })); diff --git a/packages/vitnode/src/content/server/permission-matrix.test.ts b/packages/vitnode/src/content/server/permission-matrix.test.ts index 0896b4fb5..e50345959 100644 --- a/packages/vitnode/src/content/server/permission-matrix.test.ts +++ b/packages/vitnode/src/content/server/permission-matrix.test.ts @@ -387,7 +387,8 @@ describe("the generated permission matrix", () => { asked = []; granted = new Set([`${MODULE}:can_view`]); vi.spyOn(model, "service").mockReturnValue({ - findMany: async () => await Promise.resolve({ edges: [], pageInfo: {} }), + findMany: async () => + await Promise.resolve({ edges: [], pageInfo: {} }), } as never); const res = await other.request("/"); vi.restoreAllMocks(); From 6040d9db4f1d9c0efd5ab11571484a38259b05a8 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 20:02:25 +0200 Subject: [PATCH 09/14] docs(content): state the one redundant cache invalidation rather than hide it The generated PUT answers with the row, not with a `changed` flag, so the AdminCP Server Action cannot tell a saved-but-identical edit from a real one and treats it as an ordinary update of a published record. That expires three tags stale-while-revalidate, so the responses stay served while they refresh - a refresh nobody needed rather than a cache miss. Widening a public response contract to save it would be the wrong trade. Publish, unpublish and restore all carry `changed` already and are guarded. Co-Authored-By: Claude Opus 5 (1M context) --- .../content/docs/dev/content-engine/caching.mdx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/docs/content/docs/dev/content-engine/caching.mdx b/apps/docs/content/docs/dev/content-engine/caching.mdx index 8d838e3ce..b0c4e390f 100644 --- a/apps/docs/content/docs/dev/content-engine/caching.mdx +++ b/apps/docs/content/docs/dev/content-engine/caching.mdx @@ -158,10 +158,23 @@ Two things worth stating out loud: free. - **A no-op touches nothing.** Publishing something already published transitioned nothing, so a double-clicked button costs one 200 and no cache. + The same holds for an unpublish, and for a restore that changed nothing - + each of those routes answers with `changed`, and the Server Action reads it. Nothing global is ever expired, and one content type's mutation never touches another's tags. + +An **update** that changed nothing is the exception: the generated `PUT` answers +with the row, not with a `changed` flag, so the Server Action cannot tell a +saved-but-identical edit from a real one and treats it as +`update published, same slug`. That expires three tags +stale-while-revalidate - the responses stay served while they refresh, so the +cost is a refresh nobody needed rather than a cache miss. Widening a public +response contract to save it would be the wrong trade; correctness first, hit +rate second. + + [Restore](/docs/dev/content-engine/revisions#restore) has no rules of its own - it lands on the update rows above, because as far as a visitor is concerned a restore *is* an update. It cannot appear on the publish or unpublish rows at From a242ad4add6d56f4b3824ba1ec9b20ec82266099 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 21:20:16 +0200 Subject: [PATCH 10/14] fix(pagination): make the cursor the ordered tuple, not the row id The tiebreaker added earlier made the *ordering* total but left the cursor predicate comparing only the identifier, which is not a keyset at all. With `ORDER BY title ASC, id ASC` over `(1,"Z") (2,"A") (3,"M")`, page one returns id=2 and the next query asks for `id > 2` - so id=1 is skipped permanently, and a short page looks exactly like the end of the collection. The cursor is now the whole tuple, `base64url(JSON)` of `{column, value, id}`: WHERE col > :value OR (col = :value AND id > :id) -- ascending WHERE col < :value OR (col = :value AND id < :id) -- descending with the direction flipped for `last`, so the `ORDER BY` and the predicate always describe the same sequence. Null blocks are named explicitly, because Postgres sorts NULLS LAST ascending and NULLS FIRST descending and `col > NULL` is NULL - a nullable `publishedAt` ordering would otherwise end its walk at the first null. Two things worth calling out: - **Timestamps are compared against the stored value**, through an uncorrelated subquery on the cursor's id, with the cursor's own copy as the fallback for a row that has since been deleted. A Postgres `timestamp` keeps microseconds and a JavaScript `Date` keeps milliseconds, so a value that has been out to a client and back is strictly smaller than the one still in the table - and since `now()` stamps every row in one statement identically, comparing against it would end a bulk-imported collection's walk after page one. - **A legacy numeric cursor is honoured only when the list is ordered by its identifier**, where the number really is the whole tuple. Anywhere else it is a 400 rather than a guess. Invalid input is refused rather than repaired: `first=0` used to clamp into a one-row page reporting `hasNextPage: true`, `first=abc` became `NaN` and fell through to the default size, and `first`+`last` threw a bare Error that surfaced as a 500. All of them are 400 now, most from the route schema itself. An empty page no longer claims a neighbour it cannot hand out a cursor for. `pageInfo.startCursor`/`endCursor` are opaque strings. The search route keeps its own numeric page info - it paginates the index, not a table. Co-Authored-By: Claude Opus 5 (1M context) --- .../docs/dev/content-engine/limitations.mdx | 27 +- .../performance-and-scaling.mdx | 71 +- .../content-engine/production-hardening.mdx | 4 +- apps/docs/src/examples/data-table.tsx | 5 +- .../src/api/lib/pagination-cursor.test.ts | 200 ++++++ .../vitnode/src/api/lib/pagination-cursor.ts | 196 ++++++ .../vitnode/src/api/lib/with-pagination.ts | 447 ++++++++++--- .../api/modules/search/routes/search.route.ts | 28 +- .../src/components/table/content.test.tsx | 5 +- .../src/components/table/pagination.tsx | 8 +- .../src/content/server/openapi-parity.test.ts | 10 +- .../content/server/pagination-routes.test.ts | 151 +++++ .../src/content/server/service.test.ts | 18 +- .../vitnode/src/content/server/service.ts | 11 +- .../src/database/pagination-postgres.test.ts | 625 ++++++++++++++++++ .../src/database/performance-postgres.test.ts | 15 +- 16 files changed, 1671 insertions(+), 150 deletions(-) create mode 100644 packages/vitnode/src/api/lib/pagination-cursor.test.ts create mode 100644 packages/vitnode/src/api/lib/pagination-cursor.ts create mode 100644 packages/vitnode/src/content/server/pagination-routes.test.ts create mode 100644 plugins/example/src/database/pagination-postgres.test.ts diff --git a/apps/docs/content/docs/dev/content-engine/limitations.mdx b/apps/docs/content/docs/dev/content-engine/limitations.mdx index 55ac91d99..da8342567 100644 --- a/apps/docs/content/docs/dev/content-engine/limitations.mdx +++ b/apps/docs/content/docs/dev/content-engine/limitations.mdx @@ -303,21 +303,18 @@ the rows orphaned by `contentTypeId` - the same story as an orphaned search collection, and with the same fix: a one-line `UPDATE` when it was a rename, and a `DELETE` when it was not. -## The public cursor is always the row id - -`withPagination` paginates on the primary key, so a list sorted by `publishedAt` -still pages by `id`. That is pre-existing behaviour shared with every admin list, -not a public-API quirk. - -Ties no longer matter: the primary key is appended to the ordering as a -tiebreaker, so two rows sharing a `publishedAt` have a fixed position rather than -whichever one Postgres happened to return first. What still matters is -**correlation** - the cursor is the identifier, so an order column with no -relationship to it cannot page exactly, and a row whose identifier falls on the -wrong side of the cursor is skipped. Every generated default (`updatedAt`, -`publishedAt`, `createdAt`, `id`) moves with the identifier and is exact; a -custom `orderableFields` entry that does not - a title, say - is a deep-pagination -hazard. See +## Pagination is a position, not a snapshot + +The cursor encodes the **ordered tuple** - the sort column's value and the row's +identifier - so any orderable column pages exactly, ties included and nulls +included. What it deliberately does not do is freeze the collection: rows +inserted behind the cursor are not seen until the next pass. Holding a +transaction open across requests is the only alternative, and it is a worse one. + +The cursor is opaque and belongs to the ordering that produced it. Replaying one +against a different `orderBy` is a `400` rather than a page of wrong rows, and a +legacy numeric cursor is honoured only where the identifier really is the whole +tuple - a list ordered by `id`. See [performance and scaling](/docs/dev/content-engine/performance-and-scaling#cursor-pagination). ## Content types are code diff --git a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx index 2eb73b310..69fd5be2e 100644 --- a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx +++ b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx @@ -61,18 +61,47 @@ of one record read the FAQ once. ## Cursor pagination -The cursor is the primary key, and pages are keyset reads rather than `OFFSET` - -so a page deep into a large collection seeks on the index instead of counting -past every earlier row. +Pages are keyset reads rather than `OFFSET`, so a page deep into a large +collection seeks on the index instead of counting past every earlier row. + +The cursor is the **ordered tuple**: the sort column's value *and* the row's +identifier. That is not a detail - it is the whole correctness argument. A list +ordered by `title` and a cursor that is only an identifier describe two +different sequences, and a page boundary between them skips rows permanently, +because a short page looks exactly like the end of a collection. + +```text +ORDER BY updatedAt DESC, id DESC + +WHERE updatedAt < :cursorValue + OR (updatedAt = :cursorValue AND id < :cursorId) +``` + +Ascending flips both comparisons; backward pagination (`last`) runs the whole +thing in reverse and flips the page back afterwards. The `ORDER BY` and the +predicate are built from the same tuple in the same direction, which is the +invariant everything else rests on. + + +It is `base64url(JSON)` carrying the column, the value and the identifier - so +it is meaningless outside the ordering that produced it, and it says so: a +cursor minted while a list was ordered by `updatedAt` and replayed against the +same list ordered by `title` is a `400`, not a page of wrong rows. Hand it back +unchanged; never parse it, and never build one. + **What is guaranteed:** -- a row already returned never comes back; -- the cursor always advances, so a loop always terminates; -- deleting everything past the cursor ends the walk rather than looping; -- ordering is **total** - the primary key is appended as a tiebreaker, so rows - sharing an `updatedAt` are not free to move between the two queries a page turn - issues; +- **any** orderable column pages exactly - a title, a nullable `publishedAt`, a + custom field, whatever its relationship to the identifier; +- rows sharing a sort value are returned exactly once, because the identifier is + the tiebreaker in both the ordering and the predicate; +- a nullable order column walks through its null block correctly - Postgres sorts + `NULLS LAST` ascending and `NULLS FIRST` descending, and the predicate names + the block rather than letting `column > NULL` end the walk early; +- a row already returned never comes back, and the cursor always advances, so a + loop always terminates; +- a page never claims a neighbour it cannot hand out a cursor for; - a public page is capped at the server's ceiling however large a `first` an anonymous caller sends. @@ -84,13 +113,23 @@ trade every keyset pagination makes, and it is the right one - the alternative i holding a transaction open across requests. - -The cursor **is** the identifier. Ordering by a column with no relationship to it -- a title, say - cannot page exactly: rows whose identifiers fall on the wrong -side of the cursor are skipped. Every generated default (`updatedAt`, -`publishedAt`, `createdAt`, `id`) moves with the identifier and is exact; a -custom `orderableFields` entry that does not is a deep-pagination hazard. - +### Invalid input is refused, not repaired + +`first=0` used to clamp its way into a one-row page that reported +`hasNextPage: true`; `first=abc` became `NaN` and fell through to the default +page size. Both are requests nobody made, answered as if they had. Every one of +these is now a `400`, most of them from the route's own schema: + +```text +first=0 last=0 first=-1 last=-1 +first=abc last=abc first=1.5 +first=5&last=5 +cursor= +``` + +A legacy numeric cursor still works in the one place it was ever correct - +a list ordered by its identifier, where the number really is the whole tuple. +Anywhere else it is refused rather than guessed at. ## Indexes diff --git a/apps/docs/content/docs/dev/content-engine/production-hardening.mdx b/apps/docs/content/docs/dev/content-engine/production-hardening.mdx index 602e0ae7f..ea0e42967 100644 --- a/apps/docs/content/docs/dev/content-engine/production-hardening.mdx +++ b/apps/docs/content/docs/dev/content-engine/production-hardening.mdx @@ -25,6 +25,7 @@ can point at is a hope. | **Cache** | A mutation expires exactly the tags it touched, per locale, and nothing global. | | **Security** | Every generated route carries a staff permission; a public response is an allowlist, not a filter. | | **Errors** | Every expected constraint failure has a stable status and code. No SQLSTATE reaches a client. | +| **Pagination** | The cursor is the ordered tuple, so any orderable column pages exactly - ties and nulls included. | | **Revisions** | History is append-only and per-record monotonic, across deletes and recreations. | | **Boot** | A generated table, index or constraint name that would collide is a boot failure, not a Tuesday. | @@ -51,7 +52,8 @@ listener that must act once can key off it. See - **No frozen dataset across pages.** A cursor is a position in an ordering, not a snapshot. Rows inserted behind the cursor are not seen; rows already returned - never come back. + never come back. The *ordering itself* is exact for any orderable column - the + cursor encodes the whole ordered tuple. - **No cross-request transaction.** A mutation is atomic. A *workflow* made of several mutations is not, and nothing here pretends otherwise. - **No automatic schema migration.** Migrations are generated by `drizzle-kit` diff --git a/apps/docs/src/examples/data-table.tsx b/apps/docs/src/examples/data-table.tsx index d0d86a2c0..0de593d86 100644 --- a/apps/docs/src/examples/data-table.tsx +++ b/apps/docs/src/examples/data-table.tsx @@ -73,8 +73,9 @@ export default function DataTableExample() { pageInfo={{ hasNextPage: false, hasPreviousPage: false, - startCursor: 1, - endCursor: 1, + // Opaque cursors, as the API mints them - never row identifiers. + startCursor: null, + endCursor: null, count: 1, totalCount: 1, }} diff --git a/packages/vitnode/src/api/lib/pagination-cursor.test.ts b/packages/vitnode/src/api/lib/pagination-cursor.test.ts new file mode 100644 index 000000000..3b7967873 --- /dev/null +++ b/packages/vitnode/src/api/lib/pagination-cursor.test.ts @@ -0,0 +1,200 @@ +// @vitest-environment node +import { HTTPException } from "hono/http-exception"; +import { describe, expect, it } from "vitest"; + +import { core_users } from "@/database/users"; + +import { + cursorValueForColumn, + cursorValueOf, + decodePaginationCursor, + encodePaginationCursor, + isCursorSortableColumn, +} from "./pagination-cursor"; + +/** + * The cursor is the ordered tuple, or it is nothing. + * + * An identifier on its own only describes a position when the list is ordered + * by the identifier. For any other column it names a row whose place in the + * sequence nobody knows - which is how a list ordered by `updatedAt` used to + * skip every row whose id happened to fall on the wrong side of it. + */ + +const statusOf = (error: unknown): number => + error instanceof HTTPException ? error.status : 0; + +describe("encoding", () => { + it("round-trips a cursor", () => { + const cursor = { + column: "updatedAt", + id: 42, + value: "2026-08-08T12:00:00.000Z", + }; + + expect( + decodePaginationCursor(encodePaginationCursor(cursor), { + column: "updatedAt", + primaryKey: "id", + }), + ).toEqual(cursor); + }); + + it("is opaque, so nothing downstream starts parsing it", () => { + const encoded = encodePaginationCursor({ + column: "updatedAt", + id: 42, + value: "2026-08-08T12:00:00.000Z", + }); + + // base64url: URL-safe, and not a number somebody will be tempted to read. + expect(encoded).toMatch(/^[A-Za-z0-9_-]+$/); + expect(Number.isNaN(Number(encoded))).toBe(true); + }); + + it("round-trips a null order value, which is a real position", () => { + const cursor = { column: "publishedAt", id: 9, value: null }; + + expect( + decodePaginationCursor(encodePaginationCursor(cursor), { + column: "publishedAt", + primaryKey: "id", + }), + ).toEqual(cursor); + }); + + it.each([ + ["a string", "Zebra"], + ["a number", 12], + ["a boolean", true], + ])("round-trips %s value", (_why, value) => { + const cursor = { column: "name", id: 3, value }; + + expect( + decodePaginationCursor(encodePaginationCursor(cursor), { + column: "name", + primaryKey: "id", + }).value, + ).toEqual(value); + }); +}); + +describe("decoding refuses what it cannot trust", () => { + const decode = (raw: string, column = "updatedAt") => + decodePaginationCursor(raw, { column, primaryKey: "id" }); + + it.each([ + ["garbage", "not-a-cursor!!"], + ["an empty string", " "], + [ + "valid base64 that is not JSON", + Buffer.from("hello").toString("base64url"), + ], + [ + "JSON that is not an object", + Buffer.from("[1,2,3]").toString("base64url"), + ], + [ + "an object with no id", + Buffer.from(JSON.stringify({ column: "updatedAt", value: 1 })).toString( + "base64url", + ), + ], + [ + "a non-integer id", + Buffer.from( + JSON.stringify({ column: "updatedAt", id: 1.5, value: 1 }), + ).toString("base64url"), + ], + [ + "a zero id", + Buffer.from( + JSON.stringify({ column: "updatedAt", id: 0, value: 1 }), + ).toString("base64url"), + ], + [ + "an object value", + Buffer.from( + JSON.stringify({ column: "updatedAt", id: 1, value: { a: 1 } }), + ).toString("base64url"), + ], + ])("answers 400 for %s", (_why, raw) => { + expect(() => decode(raw)).toThrow(HTTPException); + try { + decode(raw); + } catch (error) { + expect(statusOf(error)).toBe(400); + } + }); + + it("refuses a cursor minted for another ordering", () => { + // The two describe different sequences, so the position means nothing - + // and using it anyway is exactly how rows get skipped. + const encoded = encodePaginationCursor({ + column: "updatedAt", + id: 42, + value: "2026-08-08T12:00:00.000Z", + }); + + expect(() => decode(encoded, "title")).toThrow(/different ordering/); + }); +}); + +describe("legacy numeric cursors", () => { + it("still works when the list is ordered by its identifier", () => { + // There the identifier really is the whole ordered tuple, so an old + // bookmark keeps working. + expect( + decodePaginationCursor("42", { column: "id", primaryKey: "id" }), + ).toEqual({ column: "id", id: 42, value: 42 }); + }); + + it("is refused for any other ordering rather than guessed at", () => { + // The regression this whole change exists for: a bare number says nothing + // about where `updatedAt` was, so interpreting it as one would silently + // skip rows. + expect(() => + decodePaginationCursor("42", { column: "updatedAt", primaryKey: "id" }), + ).toThrow(/cannot be used with the "updatedAt" ordering/); + }); +}); + +describe("column values", () => { + it("flattens a Date to an ISO string and back", () => { + const date = new Date("2026-08-08T12:00:00.000Z"); + const flattened = cursorValueOf(core_users.createdAt, date); + + expect(flattened).toBe("2026-08-08T12:00:00.000Z"); + // Back as a `Date`, because that is what the column compares against - + // handing Postgres the string would compare a timestamp with text. + expect(cursorValueForColumn(core_users.createdAt, flattened)).toEqual(date); + }); + + it("keeps a string a string", () => { + expect(cursorValueOf(core_users.name, "Ada")).toBe("Ada"); + expect(cursorValueForColumn(core_users.name, "Ada")).toBe("Ada"); + }); + + it("keeps a number a number", () => { + expect(cursorValueOf(core_users.id, 7)).toBe(7); + expect(cursorValueForColumn(core_users.id, 7)).toBe(7); + }); + + it("treats null as null on both sides", () => { + expect(cursorValueOf(core_users.createdAt, null)).toBeNull(); + expect(cursorValueForColumn(core_users.createdAt, null)).toBeNull(); + }); + + it("refuses an unparseable date rather than comparing against Invalid Date", () => { + expect(() => + cursorValueForColumn(core_users.createdAt, "not-a-date"), + ).toThrow(HTTPException); + }); + + it("accepts the sortable column kinds", () => { + expect(isCursorSortableColumn(core_users.id)).toBe(true); + expect(isCursorSortableColumn(core_users.name)).toBe(true); + expect(isCursorSortableColumn(core_users.createdAt)).toBe(true); + expect(isCursorSortableColumn(core_users.newsletter)).toBe(true); + }); +}); diff --git a/packages/vitnode/src/api/lib/pagination-cursor.ts b/packages/vitnode/src/api/lib/pagination-cursor.ts new file mode 100644 index 000000000..0de70e45a --- /dev/null +++ b/packages/vitnode/src/api/lib/pagination-cursor.ts @@ -0,0 +1,196 @@ +import type { PgColumn } from "drizzle-orm/pg-core"; + +import { HTTPException } from "hono/http-exception"; + +/** + * The opaque cursor a paginated list hands out, and takes back. + * + * A cursor has to describe a **position in an ordering**, and an ordering is + * `(orderColumn, id)` - so the cursor is that pair. An identifier on its own is + * only a position when the list is ordered by the identifier; for any other + * column it names a row whose place in the sequence nobody knows, which is how a + * list ordered by `updatedAt` used to skip rows whose ids happened to fall on + * the wrong side of it. + * + * The wire form is `base64url(JSON)`: opaque, so no client starts depending on + * the shape, and self-describing, so a cursor minted for one order column is + * refused by a request that has since changed to another rather than silently + * producing nonsense. + */ + +/** What an order column's value can be, once it has been through JSON. */ +export type PaginationCursorValue = boolean | null | number | string; + +export interface PaginationCursor { + /** The order column this cursor was minted for. */ + column: string; + /** The row's primary key - the tiebreaker half of the ordered tuple. */ + id: number; + /** The order column's value on that row. `null` is a real position. */ + value: PaginationCursorValue; +} + +/** Postgres types whose ordering a cursor can describe. */ +const SORTABLE_DATA_TYPES = new Set([ + "bigint", + "boolean", + "date", + "number", + "string", +]); + +const badRequest = (message: string): HTTPException => + new HTTPException(400, { message }); + +/** + * Whether a column can be paged through at all. + * + * A `json`, `array` or custom column has no total order Postgres and JavaScript + * agree on, so a cursor over one would be a value the next page cannot compare + * against. Refused rather than approximated. + */ +export const isCursorSortableColumn = (column: PgColumn): boolean => + SORTABLE_DATA_TYPES.has(column.dataType); + +/** + * One column value, flattened to something JSON gives back unchanged. + * + * A `Date` becomes an ISO string and a `bigint` becomes a decimal string, + * because neither survives `JSON.stringify` as itself. + * {@link cursorValueForColumn} is the exact inverse. + */ +export const cursorValueOf = ( + column: PgColumn, + value: unknown, +): PaginationCursorValue => { + if (value === null || value === undefined) return null; + if (value instanceof Date) return value.toISOString(); + if (typeof value === "bigint") return value.toString(); + + const type = typeof value; + if (type === "boolean" || type === "number" || type === "string") { + return value as PaginationCursorValue; + } + + throw badRequest( + `The "${column.name}" column cannot be used as a pagination cursor.`, + ); +}; + +/** + * The cursor value, back in the shape the **column** compares against. + * + * This is the step that keeps the predicate honest: Drizzle maps a parameter + * through the column it is compared with, and a `timestamp` column expects a + * `Date`. Handing it the ISO string the cursor carries would compare a timestamp + * against text, which Postgres refuses - so the string is turned back into a + * `Date` here rather than coerced somewhere further down. + */ +export const cursorValueForColumn = ( + column: PgColumn, + value: PaginationCursorValue, +): unknown => { + if (value === null) return null; + + switch (column.dataType) { + case "bigint": + return BigInt(String(value)); + case "boolean": + return Boolean(value); + case "date": { + const date = new Date(String(value)); + if (Number.isNaN(date.getTime())) { + throw badRequest("Invalid pagination cursor."); + } + + return date; + } + case "number": { + const parsed = Number(value); + if (!Number.isFinite(parsed)) + throw badRequest("Invalid pagination cursor."); + + return parsed; + } + default: + return String(value); + } +}; + +export const encodePaginationCursor = (cursor: PaginationCursor): string => + Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); + +/** A bare integer, which is what every cursor was before this. */ +const LEGACY_CURSOR = /^[1-9]\d{0,14}$/; + +const isCursorValue = (value: unknown): value is PaginationCursorValue => + value === null || + typeof value === "boolean" || + typeof value === "number" || + typeof value === "string"; + +/** + * Reads a cursor, or refuses the request. + * + * Three ways this says no, and each of them is a `400` rather than a page of + * wrong rows: + * + * 1. **garbage** - not base64url, not JSON, or not the shape; + * 2. **the wrong column** - a cursor minted while the list was ordered by + * `updatedAt`, replayed against a list now ordered by `title`. The two + * describe different sequences, so the position means nothing; + * 3. **a legacy numeric cursor on a non-primary-key ordering** - the exact case + * that used to skip rows. A bare number is still accepted when the list is + * ordered by its identifier, because there it really is the whole tuple. + */ +export const decodePaginationCursor = ( + raw: string, + { column, primaryKey }: { column: string; primaryKey: string }, +): PaginationCursor => { + const trimmed = raw.trim(); + if (trimmed === "") throw badRequest("Invalid pagination cursor."); + + if (LEGACY_CURSOR.test(trimmed)) { + if (column !== primaryKey) { + throw badRequest( + `This cursor cannot be used with the "${column}" ordering. Start from the first page.`, + ); + } + const id = Number(trimmed); + + return { column, id, value: id }; + } + + let parsed: unknown; + try { + parsed = JSON.parse( + Buffer.from(trimmed, "base64url").toString("utf8"), + ) as unknown; + } catch { + throw badRequest("Invalid pagination cursor."); + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw badRequest("Invalid pagination cursor."); + } + + const candidate = parsed as Record; + const id = candidate.id; + if ( + typeof candidate.column !== "string" || + typeof id !== "number" || + !Number.isSafeInteger(id) || + id <= 0 || + !isCursorValue(candidate.value) + ) { + throw badRequest("Invalid pagination cursor."); + } + + if (candidate.column !== column) { + throw badRequest( + `This cursor was issued for a different ordering. Start from the first page.`, + ); + } + + return { column, id, value: candidate.value }; +}; diff --git a/packages/vitnode/src/api/lib/with-pagination.ts b/packages/vitnode/src/api/lib/with-pagination.ts index f81520ef4..b1a1853d9 100644 --- a/packages/vitnode/src/api/lib/with-pagination.ts +++ b/packages/vitnode/src/api/lib/with-pagination.ts @@ -8,65 +8,212 @@ import type { import type { Context } from "hono"; import { z } from "@hono/zod-openapi"; -import { and, asc, count, desc, gt, ilike, lt, or, sql } from "drizzle-orm"; +import { + and, + asc, + count, + desc, + eq, + gt, + ilike, + inArray, + isNotNull, + isNull, + lt, + or, + sql, +} from "drizzle-orm"; +import { HTTPException } from "hono/http-exception"; +import type { PaginationCursor } from "./pagination-cursor"; + +import { + cursorValueForColumn, + cursorValueOf, + decodePaginationCursor, + encodePaginationCursor, + isCursorSortableColumn, +} from "./pagination-cursor"; + +/** Nobody may ask for more than this in one page, whatever they send. */ +const MAX_PAGE_SIZE = 100; + +/** + * Reads `first`, `last` and `cursor`, or refuses the request with a 400. + * + * Refusing rather than repairing is the change worth noting. `first=0` used to + * clamp its way into a one-row page that reported `hasNextPage: true`, and + * `first=abc` became `NaN` and fell through to the default page size - both of + * them a request nobody made, answered as if they had. Every one of these is now + * a stable 400, and the route schema rejects most of them a step earlier. + */ function parsePaginationParams(params: { query: { cursor?: string; first?: string; last?: string }; -}): { cursor?: number; first?: number; last?: number } { - const cursor = params.query.cursor - ? parseInt(params.query.cursor, 10) - : undefined; - const first = params.query.first - ? Math.min(parseInt(params.query.first, 10), 100) - : undefined; - const last = params.query.last - ? Math.min(parseInt(params.query.last, 10), 100) - : undefined; +}): { cursor?: string; first?: number; last?: number } { + const size = (raw: string | undefined, name: string): number | undefined => { + if (raw === undefined || raw === "") return undefined; + + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new HTTPException(400, { + message: `"${name}" must be a whole number greater than zero.`, + }); + } + + return Math.min(parsed, MAX_PAGE_SIZE); + }; + + const first = size(params.query.first, "first"); + const last = size(params.query.last, "last"); if (first !== undefined && last !== undefined) { - throw new Error("Cannot specify both first and last"); - } - if (first !== undefined && first < 0) { - throw new Error("first must be positive"); - } - if (last !== undefined && last < 0) { - throw new Error("last must be positive"); + throw new HTTPException(400, { + message: 'Use either "first" or "last", not both.', + }); } - return { cursor, first, last }; + const cursor = params.query.cursor?.trim(); + + return { cursor: cursor === "" ? undefined : cursor, first, last }; } -function getOrderFn( +/** + * Which way the rows really come back. + * + * Backward pagination runs the query in reverse and flips the page afterwards, + * so the *effective* SQL direction is not the one the caller asked for - and + * the cursor predicate has to describe the effective one, or it would be reading + * a sequence the `ORDER BY` is not producing. + */ +function effectiveDirection( isForward: boolean, order: "asc" | "desc", -): typeof asc | typeof desc { - if (isForward) { - return order === "asc" ? asc : desc; - } +): "asc" | "desc" { + if (isForward) return order; - return order === "asc" ? desc : asc; + return order === "asc" ? "desc" : "asc"; } -function buildWhereWithCursor< - Primary extends ColumnBaseConfig<"number", string>, ->( - baseWhere: SQL | undefined, - cursor: number | undefined, - isForward: boolean, - order: "asc" | "desc", - table: PgTable, - primaryCursor: PgColumn, -): SQL | undefined { - if (!cursor) return baseWhere; +/** + * `and`/`or` given at least one defined condition always produce SQL. + * + * Stated as a check rather than a non-null assertion: the assertion would be a + * claim about code somewhere else, and this is a claim about the two lines above + * it - which is the kind that stays true. + */ +function required(value: SQL | undefined): SQL { + if (!value) throw new Error("Expected a pagination condition."); + + return value; +} + +/** + * "Strictly after this position, in this direction." + * + * The whole keyset, written out. `(column, id)` is the ordered tuple, so the + * predicate is the tuple comparison - not a comparison of one half of it: + * + * ```sql + * column > :value OR (column = :value AND id > :id) -- ascending + * column < :value OR (column = :value AND id < :id) -- descending + * ``` + * + * The `NULL` branches are the part that is easy to get wrong. Postgres sorts + * `NULLS LAST` for `ASC` and `NULLS FIRST` for `DESC`, and `column > NULL` is + * `NULL` rather than true - so a nullable order column needs the null block + * named explicitly, or a page boundary landing on it would end the walk early + * and silently. + */ +function buildCursorCondition({ + column, + cursor, + direction, + isPrimaryOrder, + primary, + table, +}: { + column: PgColumn; + cursor: PaginationCursor; + direction: "asc" | "desc"; + isPrimaryOrder: boolean; + primary: PgColumn; + table: PgTable; +}): SQL { + const after = direction === "asc" ? gt : lt; + + // The identifier is the whole tuple when the list is ordered by it, so there + // is no second half to compare and no null block to worry about. + if (isPrimaryOrder) return after(primary, cursor.id); - const cursorFilter = - (isForward && order === "asc") || (!isForward && order === "desc") - ? gt - : lt; + const boundary = boundaryValue({ column, cursor, primary, table }); - const cursorWhere = cursorFilter(table[primaryCursor.name], cursor); + if (direction === "asc") { + // NULLS LAST: a null cursor is inside the trailing block, and everything + // that is not null is already behind us. + if (cursor.value === null) { + return required(and(isNull(column), gt(primary, cursor.id))); + } - return baseWhere ? and(baseWhere, cursorWhere) : cursorWhere; + return required( + or( + gt(column, boundary), + and(eq(column, boundary), gt(primary, cursor.id)), + isNull(column), + ), + ); + } + + // NULLS FIRST: a null cursor is inside the *leading* block, so the rest of + // that block comes first and every non-null row follows it. + if (cursor.value === null) { + return required( + or(and(isNull(column), lt(primary, cursor.id)), isNotNull(column)), + ); + } + + return required( + or(lt(column, boundary), and(eq(column, boundary), lt(primary, cursor.id))), + ); +} + +/** + * The value to compare against, read back from the row the cursor names. + * + * A cursor cannot carry a Postgres `timestamp` faithfully: the column keeps + * microseconds and a JavaScript `Date` keeps milliseconds, so a value that has + * been out to a client and back is *strictly smaller* than the one still in the + * table. Comparing against it would exclude the whole millisecond it came from + * - and since `now()` gives every row in one statement the same microsecond + * stamp, that is not an edge case: a bulk-imported collection would end its walk + * after the first page, silently. + * + * So the comparison uses the stored value. The subquery is uncorrelated - one + * primary-key lookup, evaluated once - and the cursor's own copy is the fallback + * for the one case the row cannot supply: it has been deleted since. That copy + * is millisecond-accurate rather than exact, which is the honest best available + * when the row it described is gone. + */ +function boundaryValue({ + column, + cursor, + primary, + table, +}: { + column: PgColumn; + cursor: PaginationCursor; + primary: PgColumn; + table: PgTable; +}): SQL { + // `sql.param(value, column)` rather than a bare interpolation: the value has + // to go through the column's own encoder, or the driver is handed an ISO + // string where it declared a timestamp and refuses the whole statement. + const fallback = sql.param( + cursorValueForColumn(column, cursor.value), + column, + ); + const id = sql.param(cursor.id, primary); + + return sql`coalesce((select ${column} from ${table} where ${primary} = ${id}), ${fallback})`; } function buildSearchWhere( @@ -133,42 +280,54 @@ export async function withPagination< edges: QueryMin[]; pageInfo: { count: number; - endCursor: null | number; + /** An opaque cursor. Hand it back as `cursor`; never parse it. */ + endCursor: null | string; hasNextPage: boolean; hasPreviousPage: boolean; - startCursor: null | number; + startCursor: null | string; totalCount: number; }; }> { - const { cursor, first, last } = parsePaginationParams(params); + const { cursor: rawCursor, first, last } = parsePaginationParams(params); const isForward = last === undefined; - const orderFn = getOrderFn(isForward, orderByFromParams.order); + const direction = effectiveDirection(isForward, orderByFromParams.order); + const orderFn = direction === "asc" ? asc : desc; + const primary = table[primaryCursor.name]; - const ordered = orderFn(table[orderByFromParams.column.name]); + const orderName = orderByFromParams.column.name; + const orderColumn = table[orderName] as PgColumn; + const isPrimaryOrder = orderName === primaryCursor.name; + + // A column with no total order Postgres and JavaScript agree on cannot be + // paged at all, so it is refused rather than served for one page and then + // quietly wrong on the next. + if (!isCursorSortableColumn(orderColumn)) { + throw new HTTPException(400, { + message: `Results cannot be ordered by "${orderName}".`, + }); + } + /** - * The primary key, appended as a tiebreaker. - * - * Without it the ordering is only a partial one: every row that shares an - * `updatedAt` - a bulk import, a migration backfill, a batch publish - sits in - * an arbitrary position that Postgres is free to change between the two - * queries a page turn issues. The cursor is the identifier, so an arbitrary - * position means a row can be skipped or returned twice for no reason except - * that the page boundary fell inside a tie. + * The ordered tuple, `(requested column, identifier)`. * - * Appending the identifier makes the ordering total, in the same direction the - * cursor compares in, so the two agree. It costs nothing when the order column - * is already unique. - * - * What it does **not** fix, and what the pagination docs state plainly: the - * cursor is the identifier, so ordering by a column with no relationship to it - * - a title, say - still cannot page exactly. Ordering by the identifier, or - * by a timestamp that moves with it, is exact. + * The tiebreaker is not decoration: without it the ordering is partial, so + * every row sharing an `updatedAt` sits wherever Postgres feels like putting + * it, and a page boundary landing inside a tie skips or repeats rows. With it + * the ordering is total - and the cursor predicate below compares the *same* + * tuple, which is the invariant this whole module rests on. */ - const orderBy: SQL = - orderByFromParams.column.name === primaryCursor.name - ? ordered - : sql`${ordered}, ${orderFn(primary)}`; + const orderBy: SQL = isPrimaryOrder + ? orderFn(primary) + : sql`${orderFn(orderColumn)}, ${orderFn(primary)}`; + + const cursor = + rawCursor === undefined + ? undefined + : decodePaginationCursor(rawCursor, { + column: orderName, + primaryKey: primaryCursor.name, + }); const searchWhere = buildSearchWhere(search, params.query.search); const baseWhere = @@ -176,14 +335,20 @@ export async function withPagination< ? and(whereFromParams, searchWhere) : (whereFromParams ?? searchWhere); - const where = buildWhereWithCursor( - baseWhere, - cursor, - isForward, - orderByFromParams.order, - table, - primaryCursor, - ); + const cursorWhere = cursor + ? buildCursorCondition({ + column: orderColumn, + cursor, + direction, + isPrimaryOrder, + primary, + table, + }) + : undefined; + const where = + baseWhere && cursorWhere + ? and(baseWhere, cursorWhere) + : (baseWhere ?? cursorWhere); const totalCount = await fetchTotalCount(c, table, baseWhere); @@ -195,35 +360,137 @@ export async function withPagination< const slicedEdges = edges.slice(0, requested); const finalEdges = isForward ? slicedEdges : slicedEdges.reverse(); - const startCursor: null | number = - (finalEdges[0]?.[primaryCursor.name] as number) ?? null; - const endCursor: null | number = - (finalEdges.at(-1)?.[primaryCursor.name] as number) ?? null; + const boundaries = await cursorsFor({ + c, + edges: finalEdges, + orderColumn, + orderName, + primary, + primaryName: primaryCursor.name, + table, + }); return { pageInfo: { totalCount, count: finalEdges.length, - hasNextPage: isForward ? hasMore : !!cursor, - hasPreviousPage: isForward ? !!cursor : hasMore, - startCursor, - endCursor, + // An empty page has nothing to page from, so it never advertises a + // neighbour it cannot hand out a cursor for. + hasNextPage: + finalEdges.length === 0 ? false : isForward ? hasMore : Boolean(cursor), + hasPreviousPage: + finalEdges.length === 0 ? false : isForward ? Boolean(cursor) : hasMore, + ...boundaries, }, edges: finalEdges, }; } +/** + * The two cursors a page hands back. + * + * The order column's value normally comes straight off the row, because a list + * almost always selects the column it sorts by. When it does not - a projection + * that names a subset - the two boundary values are read in one extra query + * rather than the cursor quietly falling back to an identifier, which is exactly + * the bug this module exists to remove. + */ +async function cursorsFor({ + c, + edges, + orderColumn, + orderName, + primary, + primaryName, + table, +}: { + c: Context; + edges: readonly Record[]; + orderColumn: PgColumn; + orderName: string; + primary: PgColumn; + primaryName: string; + table: PgTable; +}): Promise<{ endCursor: null | string; startCursor: null | string }> { + const first = edges[0]; + const last = edges.at(-1); + if (!first || !last) return { endCursor: null, startCursor: null }; + + const idOf = (row: Record): number => + Number(row[primaryName]); + + const projected = orderName in first && orderName in last; + const values = new Map(); + if (projected) { + values.set(idOf(first), first[orderName]); + values.set(idOf(last), last[orderName]); + } else { + const ids = [...new Set([idOf(first), idOf(last)])]; + const rows = await c + .get("db") + .select({ id: primary, value: orderColumn }) + .from(table) + .where(inArray(primary, ids)); + + for (const row of rows) values.set(Number(row.id), row.value); + } + + const mint = (row: Record): string => + encodePaginationCursor({ + column: orderName, + id: idOf(row), + value: cursorValueOf(orderColumn, values.get(idOf(row))), + }); + + return { endCursor: mint(last), startCursor: mint(first) }; +} + +/** A positive whole number, as a query string carries it. */ +const zodPageSize = z + .string() + .regex(/^\d+$/, "Must be a whole number.") + .refine(value => Number(value) >= 1, "Must be greater than zero.") + .refine(value => Number.isSafeInteger(Number(value)), "Too large."); + export const zodPaginationPageInfo = z.object({ totalCount: z.number(), count: z.number(), hasNextPage: z.boolean(), hasPreviousPage: z.boolean(), - startCursor: z.number().nullable(), - endCursor: z.number().nullable(), + /** + * Opaque. It encodes the ordered tuple the next page continues from, so it is + * meaningless outside the ordering that produced it - hand it back unchanged. + */ + startCursor: z.string().nullable(), + endCursor: z.string().nullable(), }); -export const zodPaginationQuery = z.object({ - cursor: z.string().optional(), - first: z.string().optional(), - last: z.string().optional(), -}); +/** + * The pagination half of a list route's query, validated at the edge. + * + * Every rule that can be stated here is stated here rather than left to the + * internals, so a bad page size is a 400 from the route's own contract - and + * appears in the OpenAPI document - instead of something the handler discovers + * later. `parsePaginationParams` re-checks all of it, because a service can be + * called directly and a plugin can build a route without this schema. + * + * The cursor is only shape-checked here: it is opaque, so "looks like a cursor" + * is all a request schema can honestly say. Whether it decodes, and whether it + * belongs to *this* ordering, is decided where the ordering is known. + */ +export const zodPaginationQuery = z + .object({ + cursor: z + .string() + .min(1) + .max(512) + // base64url, or a legacy numeric cursor. Anything else cannot be one. + .regex(/^[A-Za-z0-9_-]+$/, "Invalid cursor.") + .optional(), + first: zodPageSize.optional(), + last: zodPageSize.optional(), + }) + .refine( + query => query.first === undefined || query.last === undefined, + 'Use either "first" or "last", not both.', + ); diff --git a/packages/vitnode/src/api/modules/search/routes/search.route.ts b/packages/vitnode/src/api/modules/search/routes/search.route.ts index ee4433064..0256d91ed 100644 --- a/packages/vitnode/src/api/modules/search/routes/search.route.ts +++ b/packages/vitnode/src/api/modules/search/routes/search.route.ts @@ -3,10 +3,24 @@ import { z } from "@hono/zod-openapi"; import { CONFIG_PLUGIN } from "@/config"; import { buildRoute } from "../../../lib/route"; -import { - zodPaginationPageInfo, - zodPaginationQuery, -} from "../../../lib/with-pagination"; +import { zodPaginationQuery } from "../../../lib/with-pagination"; + +/** + * The search index's own page info. + * + * Deliberately not `zodPaginationPageInfo`: that one describes a keyset walk + * over a table and hands out an opaque cursor for the ordered tuple. A search + * page is not that - a relevance-sorted one walks by offset and an ordinary one + * by row id - so it keeps the numeric cursors it has always had. + */ +const zodSearchPageInfo = z.object({ + totalCount: z.number(), + count: z.number(), + hasNextPage: z.boolean(), + hasPreviousPage: z.boolean(), + startCursor: z.number().nullable(), + endCursor: z.number().nullable(), +}); export const zodSearchHitSchema = z.object({ id: z.number(), @@ -57,7 +71,11 @@ export const searchRoute = buildRoute({ "application/json": { schema: z.object({ edges: z.array(zodSearchHitSchema), - pageInfo: zodPaginationPageInfo, + // The search index has its own pagination - a relevance-sorted + // page walks by offset, and an ordinary one by row id - so it + // keeps the numeric cursors it has always had rather than the + // opaque keyset cursor `withPagination` mints for a table. + pageInfo: zodSearchPageInfo, }), }, }, diff --git a/packages/vitnode/src/components/table/content.test.tsx b/packages/vitnode/src/components/table/content.test.tsx index 5099e6650..ce4aa3ebb 100644 --- a/packages/vitnode/src/components/table/content.test.tsx +++ b/packages/vitnode/src/components/table/content.test.tsx @@ -32,10 +32,11 @@ const edges: DemoUser[] = [ const pageInfo = { count: edges.length, - endCursor: 2, + // Opaque strings, as `withPagination` mints them - never row ids. + endCursor: "eyJjb2x1bW4iOiJpZCIsImlkIjoyLCJ2YWx1ZSI6Mn0", hasNextPage: false, hasPreviousPage: false, - startCursor: 1, + startCursor: "eyJjb2x1bW4iOiJpZCIsImlkIjoxLCJ2YWx1ZSI6MX0", totalCount: edges.length, }; diff --git a/packages/vitnode/src/components/table/pagination.tsx b/packages/vitnode/src/components/table/pagination.tsx index 8badd5ff9..461ca4e21 100644 --- a/packages/vitnode/src/components/table/pagination.tsx +++ b/packages/vitnode/src/components/table/pagination.tsx @@ -24,10 +24,10 @@ export const PaginationDataTable = ({ }: { pageInfo: { count: number; - endCursor: null | number; + endCursor: null | string; hasNextPage: boolean; hasPreviousPage: boolean; - startCursor: null | number; + startCursor: null | string; totalCount: number; }; }) => { @@ -93,7 +93,7 @@ export const PaginationDataTable = ({ const params = new URLSearchParams(searchParams.toString()); params.set("last", `${Number(pageSize)}`); if (startCursor) { - params.set("cursor", `${startCursor}`); + params.set("cursor", startCursor); } else { params.delete("cursor"); } @@ -122,7 +122,7 @@ export const PaginationDataTable = ({ const params = new URLSearchParams(searchParams.toString()); params.set("first", `${Number(pageSize)}`); if (endCursor) { - params.set("cursor", `${endCursor}`); + params.set("cursor", endCursor); } else { params.delete("cursor"); } diff --git a/packages/vitnode/src/content/server/openapi-parity.test.ts b/packages/vitnode/src/content/server/openapi-parity.test.ts index b2179e1c3..eacce45bc 100644 --- a/packages/vitnode/src/content/server/openapi-parity.test.ts +++ b/packages/vitnode/src/content/server/openapi-parity.test.ts @@ -246,10 +246,11 @@ const adminService = () => ({ edges: [{ ...row, labels: {} }], pageInfo: { count: 1, - endCursor: 7, + // Opaque, as `withPagination` mints them: the ordered tuple, base64url'd. + endCursor: "eyJjb2x1bW4iOiJ2ZXJzaW9uIiwiaWQiOjcsInZhbHVlIjo0fQ", hasNextPage: false, hasPreviousPage: false, - startCursor: 7, + startCursor: "eyJjb2x1bW4iOiJ2ZXJzaW9uIiwiaWQiOjcsInZhbHVlIjo0fQ", totalCount: 1, }, }), @@ -773,10 +774,11 @@ describe("public routes match their OpenAPI document", () => { edges: [publicRow], pageInfo: { count: 1, - endCursor: 7, + // Opaque, as `withPagination` mints them. + endCursor: "eyJjb2x1bW4iOiJwdWJsaXNoZWRBdCIsImlkIjo3fQ", hasNextPage: false, hasPreviousPage: false, - startCursor: 7, + startCursor: "eyJjb2x1bW4iOiJwdWJsaXNoZWRBdCIsImlkIjo3fQ", totalCount: 1, }, }), diff --git a/packages/vitnode/src/content/server/pagination-routes.test.ts b/packages/vitnode/src/content/server/pagination-routes.test.ts new file mode 100644 index 000000000..9909cae10 --- /dev/null +++ b/packages/vitnode/src/content/server/pagination-routes.test.ts @@ -0,0 +1,151 @@ +// @vitest-environment node +import type { MiddlewareHandler } from "hono"; + +import { OpenAPIHono } from "@hono/zod-openapi"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { + testArticleContentType, + testCategoryContentType, +} from "@/tests/content-fixtures"; + +import { createContentModel } from "./model"; +import { buildContentRoutes } from "./routes"; + +/** + * What a list route does with pagination input it cannot honour. + * + * Every case here used to be answered rather than refused: `first=0` clamped + * its way into a one-row page that reported `hasNextPage: true`, `first=abc` + * became `NaN` and fell through to the default page size, and `first` and + * `last` together threw a bare `Error` that surfaced as a 500. Each of them is + * a request nobody made, answered as if they had. + * + * The schema catches most of them at the edge and `parsePaginationParams` + * catches the rest; both answer 400, which is the only thing a client has to + * know. + */ + +vi.mock("../../api/lib/check-staff-permission", () => ({ + assertStaffPermission: async () => await Promise.resolve(), +})); + +const categories = createContentModel(testCategoryContentType); +const articles = createContentModel(testArticleContentType, { + references: { category: () => categories.table.id }, +}); +const PLUGIN_ID = "@vitnode/example"; + +const adminUser = { + avatarColor: "000000", + birthday: null, + createdAt: new Date(), + email: "test@test.com", + emailVerified: true, + id: 1, + language: "en", + name: "Test", + nameCode: "test", + newsletter: false, + roleId: 1, +}; + +const harness = () => { + const findMany = vi.fn().mockResolvedValue({ + edges: [], + pageInfo: { + count: 0, + endCursor: null, + hasNextPage: false, + hasPreviousPage: false, + startCursor: null, + totalCount: 0, + }, + }); + vi.spyOn(articles, "service").mockReturnValue({ + findMany, + relations: {}, + repeatable: {}, + } as never); + + const app = new OpenAPIHono(); + const context: MiddlewareHandler = async (c, next) => { + c.set("admin", { user: adminUser }); + await next(); + }; + app.use("*", context); + for (const { handler, route } of buildContentRoutes(articles, { + pluginId: PLUGIN_ID, + })) { + app.openapi(route, handler); + } + + return { app, findMany }; +}; + +beforeEach(() => { + vi.restoreAllMocks(); +}); + +describe("pagination input a list route refuses", () => { + it.each([ + ["first=0", "first=0"], + ["last=0", "last=0"], + ["first=-1", "first=-1"], + ["last=-1", "last=-1"], + ["first=abc", "first=abc"], + ["last=abc", "last=abc"], + ["a fractional page size", "first=1.5"], + ["both first and last", "first=5&last=5"], + ["a garbage cursor", "cursor=%21%21not-a-cursor"], + ["an empty cursor", "cursor="], + ])("answers 400 for %s", async (_why, query) => { + const { app } = harness(); + + const res = await app.request(`/?${query}`); + + expect([query, res.status]).toEqual([query, 400]); + }); + + it("never reaches the service with a page size it would have to clamp", async () => { + const { app, findMany } = harness(); + + await app.request("/?first=0"); + + expect(findMany).not.toHaveBeenCalled(); + }); + + it("still accepts a legitimate page", async () => { + const { app, findMany } = harness(); + + const res = await app.request("/?first=25"); + + expect(res.status).toBe(200); + expect(findMany).toHaveBeenCalledWith( + expect.objectContaining({ + query: expect.objectContaining({ first: "25" }), + }), + ); + }); + + it("refuses a legacy numeric cursor on an ordering that is not the identifier", async () => { + // The exact shape of the old bug, refused where the ordering is known: a + // bare number says nothing about where `title` was, so honouring it would + // skip rows. The service raises it; the route passes it through unchanged. + const { app } = harness(); + const { HTTPException } = await import("hono/http-exception"); + vi.spyOn(articles, "service").mockReturnValue({ + findMany: vi.fn().mockImplementation(() => { + throw new HTTPException(400, { + message: 'This cursor cannot be used with the "title" ordering.', + }); + }), + relations: {}, + repeatable: {}, + } as never); + + const res = await app.request("/?orderBy=title&cursor=42"); + + expect(res.status).toBe(400); + }); +}); diff --git a/packages/vitnode/src/content/server/service.test.ts b/packages/vitnode/src/content/server/service.test.ts index 8123e566a..9c285b4a9 100644 --- a/packages/vitnode/src/content/server/service.test.ts +++ b/packages/vitnode/src/content/server/service.test.ts @@ -324,10 +324,24 @@ describe("content service", () => { const page = (rows: unknown[]) => [[{ count: rows.length }], rows]; it("joins once per reference field instead of querying per row", async () => { + // `updatedAt` is on the rows because the list really selects it - the + // page is built from `ownSelection()`. That matters here: the cursor is + // the ordered tuple, so a page whose rows are missing the order column + // costs one extra round trip to mint one. const { c, calls } = createDbMock( page([ - { id: 1, label__author: "Ada", label__category: "News" }, - { id: 2, label__author: null, label__category: "News" }, + { + id: 1, + label__author: "Ada", + label__category: "News", + updatedAt: new Date("2026-01-02T00:00:00.000Z"), + }, + { + id: 2, + label__author: null, + label__category: "News", + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + }, ]), ); diff --git a/packages/vitnode/src/content/server/service.ts b/packages/vitnode/src/content/server/service.ts index 3e5dc85c7..e5d00a9bd 100644 --- a/packages/vitnode/src/content/server/service.ts +++ b/packages/vitnode/src/content/server/service.ts @@ -66,10 +66,17 @@ export type ContentListRow = ContentSelect & { export interface ContentPageInfo { count: number; - endCursor: null | number; + /** + * An opaque cursor for the last row on this page. + * + * It encodes the ordered tuple - the sort column's value *and* the row's + * identifier - so it is meaningless outside the ordering that produced it. + * Hand it back as `cursor`; never parse it. + */ + endCursor: null | string; hasNextPage: boolean; hasPreviousPage: boolean; - startCursor: null | number; + startCursor: null | string; totalCount: number; } diff --git a/plugins/example/src/database/pagination-postgres.test.ts b/plugins/example/src/database/pagination-postgres.test.ts new file mode 100644 index 000000000..360a67fd2 --- /dev/null +++ b/plugins/example/src/database/pagination-postgres.test.ts @@ -0,0 +1,625 @@ +import type { Context } from "hono"; + +import { HTTPException } from "hono/http-exception"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import { CONFIG_PLUGIN } from "@/const"; + +import type { ContentTestHarness } from "./harness"; + +import { articleContent } from "./articles"; +import { + clearContentTables, + createContentTestHarness, + DATABASE_TEST_URL, +} from "./harness"; + +/** + * Keyset pagination, against a collection built to break an id-only cursor. + * + * The bug this suite exists for: the cursor used to be the row identifier while + * the `ORDER BY` was something else entirely. Those describe two different + * sequences, so a page boundary landing anywhere except a coincidence would + * skip rows - permanently, and silently, because a short page looks exactly + * like the end of a collection. + * + * Every fixture here therefore makes the sort value **disagree** with the + * identifier on purpose. A cursor that is the ordered tuple walks them + * correctly; one that is only an identifier cannot. + * + * The oracle in every case is the same query without pagination: a full walk + * has to produce exactly the rows a single `ORDER BY` produces, in the same + * order. + */ + +let h: ContentTestHarness; +let categoryId = 0; + +const service = (on: Context = h.context) => articleContent.service(on); + +interface Seed { + code: string; + publishedAt?: Date | null; + title: string; + updatedAt: Date; +} + +/** + * Inserts rows in the order given, so identifiers ascend with the array while + * the sort values do whatever the fixture says. + */ +const seed = async (rows: readonly Seed[]): Promise => { + const ids: number[] = []; + for (const [index, row] of rows.entries()) { + const [inserted] = await h.sql<{ id: number }[]>` + INSERT INTO "example_articles" + ("title", "slug", "code", "category", "status", "publishedAt", "updatedAt") + VALUES ( + ${row.title}, + ${`slug-${row.code}`}, + ${row.code}, + ${categoryId}, + ${row.publishedAt === undefined || row.publishedAt === null ? "draft" : "published"}, + ${row.publishedAt?.toISOString() ?? null}::timestamp, + ${row.updatedAt.toISOString()}::timestamp + ) + RETURNING "id" + `; + ids.push(inserted.id); + expect(index).toBeGreaterThanOrEqual(0); + } + + return ids; +}; + +/** The order a single un-paginated query produces - the oracle. */ +const oracle = async ( + column: string, + order: "asc" | "desc", +): Promise => { + const rows = await h.sql.unsafe( + `SELECT "id" FROM "example_articles" + ORDER BY "${column}" ${order.toUpperCase()}, "id" ${order.toUpperCase()}`, + ); + + return rows.map(row => Number(row.id)); +}; + +/** Walks every page forward and returns the identifiers, in order. */ +const walkForward = async ({ + column, + order = "asc", + pageSize, +}: { + column?: string; + order?: "asc" | "desc"; + pageSize: number; +}): Promise => { + const seen: number[] = []; + let cursor: string | undefined; + + for (let page = 0; page < 200; page += 1) { + const result = await service().findMany({ + orderBy: column ? { column: column as never, order } : { order }, + query: { cursor, first: String(pageSize) }, + }); + + seen.push(...result.edges.map(row => row.id)); + if (!result.pageInfo.hasNextPage) break; + + // The invariant the reviewer asked for: a page that claims a neighbour has + // to hand out a cursor that reaches it. + expect(result.pageInfo.endCursor).not.toBeNull(); + cursor = result.pageInfo.endCursor ?? undefined; + } + + return seen; +}; + +const statusOf = (error: unknown): number => + error instanceof HTTPException ? error.status : 0; + +describe.skipIf(!DATABASE_TEST_URL)( + "cursor pagination against Postgres", + () => { + beforeAll(async () => { + h = await createContentTestHarness(); + }, 60_000); + + afterAll(async () => { + await h?.end(); + }); + + beforeEach(async () => { + await clearContentTables(h.sql); + h.reset(); + + const [category] = await h.sql<{ id: number }[]>` + INSERT INTO "example_categories" ("name") VALUES ('Pagination') + RETURNING "id" + `; + categoryId = category.id; + }); + + // ------------------------------------------------------------------------- + // The regression + // ------------------------------------------------------------------------- + + /** + * Identifiers ascending, sort values deliberately shuffled. + * + * `id=1` sorts last, `id=2` sorts first, `id=3` sits in the middle. An id-only + * cursor mints `2` after the first page and then asks for `id > 2`, which + * skips `id=1` forever. + */ + const NON_MONOTONIC: Seed[] = [ + { + code: "n1", + title: "Zulu", + updatedAt: new Date("2026-03-01T00:00:00.000Z"), + }, + { + code: "n2", + title: "Alpha", + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + }, + { + code: "n3", + title: "Mike", + updatedAt: new Date("2026-02-01T00:00:00.000Z"), + }, + ]; + + it("skips nothing when the sort value does not follow the identifier", async () => { + const ids = await seed(NON_MONOTONIC); + + const walked = await walkForward({ column: "title", pageSize: 1 }); + + expect(walked).toHaveLength(ids.length); + expect(new Set(walked).size).toBe(ids.length); + // And in the order a single query would have produced. + expect(walked).toEqual(await oracle("title", "asc")); + }); + + it("skips nothing ordering by a timestamp that does not follow the identifier", async () => { + const ids = await seed(NON_MONOTONIC); + + const walked = await walkForward({ column: "updatedAt", pageSize: 1 }); + + expect(walked).toHaveLength(ids.length); + expect(new Set(walked).size).toBe(ids.length); + expect(walked).toEqual(await oracle("updatedAt", "asc")); + }); + + it("skips nothing descending either", async () => { + const ids = await seed(NON_MONOTONIC); + + const walked = await walkForward({ + column: "updatedAt", + order: "desc", + pageSize: 1, + }); + + expect(walked).toHaveLength(ids.length); + expect(walked).toEqual(await oracle("updatedAt", "desc")); + }); + + it("holds over a larger, thoroughly shuffled collection", async () => { + // 40 rows whose sort values are a permutation of their identifiers, walked + // two at a time so every page boundary lands somewhere different. + const rows: Seed[] = Array.from({ length: 40 }, (_row, index) => ({ + code: `bulk-${index}`, + title: `Title ${String((index * 17) % 40).padStart(2, "0")}`, + updatedAt: new Date(2026, 0, 1 + ((index * 23) % 40)), + })); + await seed(rows); + + for (const column of ["title", "updatedAt"] as const) { + for (const order of ["asc", "desc"] as const) { + const walked = await walkForward({ column, order, pageSize: 3 }); + + expect([column, order, walked.length]).toEqual([column, order, 40]); + expect(new Set(walked).size).toBe(40); + expect(walked).toEqual(await oracle(column, order)); + } + } + }); + + // ------------------------------------------------------------------------- + // Ties + // ------------------------------------------------------------------------- + + it("returns every row exactly once when the sort values are all equal", async () => { + // The tie case: with no tiebreaker the rows sit wherever Postgres feels + // like putting them, and a page boundary inside the tie loses one. + const stamp = new Date("2026-05-05T00:00:00.000Z"); + const ids = await seed( + Array.from({ length: 12 }, (_row, index) => ({ + code: `tie-${index}`, + title: `Tie ${index}`, + updatedAt: stamp, + })), + ); + + const walked = await walkForward({ column: "updatedAt", pageSize: 5 }); + + expect(walked).toHaveLength(ids.length); + expect(new Set(walked).size).toBe(ids.length); + expect(walked).toEqual(await oracle("updatedAt", "asc")); + }); + + it("returns every row exactly once with ties in a published-at ordering", async () => { + const stamp = new Date("2026-05-05T00:00:00.000Z"); + const ids = await seed( + Array.from({ length: 9 }, (_row, index) => ({ + code: `pub-${index}`, + publishedAt: stamp, + title: `Published ${index}`, + updatedAt: new Date(2026, 0, 1 + index), + })), + ); + + const walked = await walkForward({ + column: "publishedAt", + order: "desc", + pageSize: 4, + }); + + expect(walked).toHaveLength(ids.length); + expect(walked).toEqual(await oracle("publishedAt", "desc")); + }); + + // ------------------------------------------------------------------------- + // Nulls + // ------------------------------------------------------------------------- + + /** + * A nullable order column is where the null block has to be named explicitly: + * Postgres sorts `NULLS LAST` ascending and `NULLS FIRST` descending, and + * `column > NULL` is `NULL` rather than true - so a page boundary landing on + * the block would otherwise end the walk early and silently. + */ + it("walks a nullable order column through its null block, ascending", async () => { + const ids = await seed([ + { + code: "u1", + publishedAt: new Date("2026-02-01"), + title: "One", + updatedAt: new Date("2026-01-01"), + }, + { + code: "u2", + publishedAt: null, + title: "Two", + updatedAt: new Date("2026-01-02"), + }, + { + code: "u3", + publishedAt: new Date("2026-01-01"), + title: "Three", + updatedAt: new Date("2026-01-03"), + }, + { + code: "u4", + publishedAt: null, + title: "Four", + updatedAt: new Date("2026-01-04"), + }, + { + code: "u5", + publishedAt: new Date("2026-03-01"), + title: "Five", + updatedAt: new Date("2026-01-05"), + }, + ]); + + const walked = await walkForward({ column: "publishedAt", pageSize: 2 }); + + expect(walked).toHaveLength(ids.length); + expect(new Set(walked).size).toBe(ids.length); + expect(walked).toEqual(await oracle("publishedAt", "asc")); + }); + + it("walks a nullable order column through its null block, descending", async () => { + const ids = await seed([ + { + code: "d1", + publishedAt: new Date("2026-02-01"), + title: "One", + updatedAt: new Date("2026-01-01"), + }, + { + code: "d2", + publishedAt: null, + title: "Two", + updatedAt: new Date("2026-01-02"), + }, + { + code: "d3", + publishedAt: new Date("2026-01-01"), + title: "Three", + updatedAt: new Date("2026-01-03"), + }, + { + code: "d4", + publishedAt: null, + title: "Four", + updatedAt: new Date("2026-01-04"), + }, + { + code: "d5", + publishedAt: new Date("2026-03-01"), + title: "Five", + updatedAt: new Date("2026-01-05"), + }, + ]); + + const walked = await walkForward({ + column: "publishedAt", + order: "desc", + pageSize: 2, + }); + + expect(walked).toHaveLength(ids.length); + expect(walked).toEqual(await oracle("publishedAt", "desc")); + }); + + // ------------------------------------------------------------------------- + // Ordering by the identifier + // ------------------------------------------------------------------------- + + it.each([["asc" as const], ["desc" as const]])( + "walks the identifier ordering (%s)", + async order => { + const ids = await seed(NON_MONOTONIC); + + const walked = await walkForward({ column: "id", order, pageSize: 2 }); + + expect(walked).toHaveLength(ids.length); + expect(walked).toEqual(await oracle("id", order)); + }, + ); + + it("still accepts a legacy numeric cursor when ordering by the identifier", async () => { + // Old bookmarks keep working exactly where an identifier really is the + // whole ordered tuple - and nowhere else. + const ids = await seed(NON_MONOTONIC); + + const page = await service().findMany({ + orderBy: { column: "id" as never, order: "asc" }, + query: { cursor: String(ids[0]), first: "10" }, + }); + + expect(page.edges.map(row => row.id)).toEqual(ids.slice(1)); + }); + + it("refuses a legacy numeric cursor on any other ordering", async () => { + await seed(NON_MONOTONIC); + + await expect( + service().findMany({ + orderBy: { column: "title" as never, order: "asc" }, + query: { cursor: "1", first: "10" }, + }), + ).rejects.toThrow(/cannot be used with the "title" ordering/); + }); + + // ------------------------------------------------------------------------- + // Backward pagination + // ------------------------------------------------------------------------- + + it("walks backward from the end and reaches the beginning", async () => { + const rows: Seed[] = Array.from({ length: 11 }, (_row, index) => ({ + code: `back-${index}`, + title: `Title ${String((index * 7) % 11).padStart(2, "0")}`, + updatedAt: new Date(2026, 0, 1 + ((index * 5) % 11)), + })); + await seed(rows); + + const expected = await oracle("title", "asc"); + + // Forward to the end, keeping the cursor of the final page's first row. + const forward = await service().findMany({ + orderBy: { column: "title" as never, order: "asc" }, + query: { first: "11" }, + }); + expect(forward.edges.map(row => row.id)).toEqual(expected); + + // Then backward from the last row, four at a time. + const seen: number[] = []; + let cursor = forward.pageInfo.endCursor ?? undefined; + for (let page = 0; page < 20; page += 1) { + const result = await service().findMany({ + orderBy: { column: "title" as never, order: "asc" }, + query: { cursor, last: "4" }, + }); + if (result.edges.length === 0) break; + + seen.unshift(...result.edges.map(row => row.id)); + if (!result.pageInfo.hasPreviousPage) break; + expect(result.pageInfo.startCursor).not.toBeNull(); + cursor = result.pageInfo.startCursor ?? undefined; + } + + // Everything before the row we started from, in the same order. + expect(seen).toEqual(expected.slice(0, expected.length - 1)); + expect(new Set(seen).size).toBe(seen.length); + }); + + it("keeps backward pagination correct with a non-monotonic ordering", async () => { + await seed(NON_MONOTONIC); + const expected = await oracle("updatedAt", "desc"); + + const forward = await service().findMany({ + orderBy: { column: "updatedAt" as never, order: "desc" }, + query: { first: "3" }, + }); + + const back = await service().findMany({ + orderBy: { column: "updatedAt" as never, order: "desc" }, + query: { cursor: forward.pageInfo.endCursor ?? undefined, last: "2" }, + }); + + expect(back.edges.map(row => row.id)).toEqual(expected.slice(0, 2)); + }); + + // ------------------------------------------------------------------------- + // Page info + // ------------------------------------------------------------------------- + + it("never claims a next page it cannot hand out a cursor for", async () => { + const rows: Seed[] = Array.from({ length: 6 }, (_row, index) => ({ + code: `info-${index}`, + title: `Info ${index}`, + updatedAt: new Date(2026, 0, 1 + index), + })); + await seed(rows); + + let cursor: string | undefined; + for (let page = 0; page < 10; page += 1) { + const result = await service().findMany({ + orderBy: { column: "title" as never, order: "asc" }, + query: { cursor, first: "2" }, + }); + + if (result.pageInfo.hasNextPage) { + expect(result.pageInfo.endCursor).toEqual(expect.any(String)); + } + if (result.pageInfo.hasPreviousPage) { + expect(result.pageInfo.startCursor).toEqual(expect.any(String)); + } + if (!result.pageInfo.hasNextPage) break; + cursor = result.pageInfo.endCursor ?? undefined; + } + }); + + it("reports an empty collection with no cursors and no neighbours", async () => { + const result = await service().findMany({ query: { first: "5" } }); + + expect(result.pageInfo).toMatchObject({ + endCursor: null, + hasNextPage: false, + hasPreviousPage: false, + startCursor: null, + totalCount: 0, + }); + }); + + it("hands out an opaque cursor rather than a row identifier", async () => { + const ids = await seed(NON_MONOTONIC); + + const page = await service().findMany({ + orderBy: { column: "title" as never, order: "asc" }, + query: { first: "1" }, + }); + + const cursor = page.pageInfo.endCursor ?? ""; + expect(cursor).not.toBe(String(ids[0])); + expect(Number.isNaN(Number(cursor))).toBe(true); + // It carries the ordered tuple, which is the whole point. + expect( + JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")), + ).toMatchObject({ column: "title" }); + }); + + // ------------------------------------------------------------------------- + // Validation + // ------------------------------------------------------------------------- + + describe("refuses a request it cannot answer", () => { + const expect400 = async (query: Record) => { + try { + await service().findMany({ query }); + } catch (error) { + expect(statusOf(error)).toBe(400); + + return; + } + + throw new Error(`Expected ${JSON.stringify(query)} to be refused.`); + }; + + it.each([ + ["first=0", { first: "0" }], + ["last=0", { last: "0" }], + ["first=-1", { first: "-1" }], + ["last=-1", { last: "-1" }], + ["first=abc", { first: "abc" }], + ["last=abc", { last: "abc" }], + ["first=1.5", { first: "1.5" }], + ["both first and last", { first: "5", last: "5" }], + ["cursor=garbage", { cursor: "!!!not-a-cursor!!!" }], + ])("answers 400 for %s", async (_why, query) => { + await expect400(query); + }); + + it("does not turn first=0 into a one-row page", async () => { + // What it used to do: clamp to a limit of one, return a row, and report + // `hasNextPage: true` for a page nobody asked for. + await seed(NON_MONOTONIC); + + await expect400({ first: "0" }); + }); + + it("caps a page at the maximum rather than trusting the caller", async () => { + const rows: Seed[] = Array.from({ length: 3 }, (_row, index) => ({ + code: `cap-${index}`, + title: `Cap ${index}`, + updatedAt: new Date(2026, 0, 1 + index), + })); + await seed(rows); + + const page = await service().findMany({ query: { first: "100000" } }); + + expect(page.edges).toHaveLength(3); + }); + }); + + // ------------------------------------------------------------------------- + // The public list, which is the anonymous half of the same machinery + // ------------------------------------------------------------------------- + + it("walks the public list with a non-monotonic publication order", async () => { + const build = articleContent.publicService; + if (!build) throw new Error("no public service"); + + await seed([ + { + code: "p1", + publishedAt: new Date("2026-03-01"), + title: "Zulu", + updatedAt: new Date("2026-01-01"), + }, + { + code: "p2", + publishedAt: new Date("2026-01-01"), + title: "Alpha", + updatedAt: new Date("2026-01-02"), + }, + { + code: "p3", + publishedAt: new Date("2026-02-01"), + title: "Mike", + updatedAt: new Date("2026-01-03"), + }, + ]); + + const seen: number[] = []; + let cursor: string | undefined; + for (let page = 0; page < 10; page += 1) { + const result = await build(h.context).findMany({ + orderBy: { column: "publishedAt" as never, order: "desc" }, + query: { cursor, first: "1" }, + }); + seen.push(...result.edges.map(row => Number(row.publishedAt))); + if (!result.pageInfo.hasNextPage) break; + cursor = result.pageInfo.endCursor ?? undefined; + } + + expect(seen).toHaveLength(3); + // Newest first, which is the ordering the route asked for. + expect([...seen].sort((a, b) => b - a)).toEqual(seen); + expect(CONFIG_PLUGIN.pluginId).toBe("@vitnode/example"); + }); + }, +); diff --git a/plugins/example/src/database/performance-postgres.test.ts b/plugins/example/src/database/performance-postgres.test.ts index c2884ed86..cf573dd0e 100644 --- a/plugins/example/src/database/performance-postgres.test.ts +++ b/plugins/example/src/database/performance-postgres.test.ts @@ -201,7 +201,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { }); seen.push(...result.edges.map(row => row.id)); if (!result.pageInfo.hasNextPage) break; - cursor = String(result.pageInfo.endCursor); + cursor = result.pageInfo.endCursor ?? undefined; } expect(seen).toHaveLength(103); @@ -221,7 +221,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { const second = await articleContent.service(h.context).findMany({ query: { - cursor: String(first.pageInfo.endCursor), + cursor: first.pageInfo.endCursor ?? undefined, first: String(PAGE), }, }); @@ -238,16 +238,17 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { .service(h.context) .findMany({ query: { first: String(PAGE) } }); - // The list is newest-first, so everything *after* the cursor has a - // smaller identifier than it. + // Everything after the first page. The cursor is opaque now, so the + // boundary is the last identifier the page actually handed back - the + // list is newest-first, so "after" means a smaller identifier. + const boundary = first.edges.at(-1)?.id ?? 0; await h.sql` - DELETE FROM "example_articles" - WHERE "id" < ${Number(first.pageInfo.endCursor)} + DELETE FROM "example_articles" WHERE "id" < ${boundary} `; const second = await articleContent.service(h.context).findMany({ query: { - cursor: String(first.pageInfo.endCursor), + cursor: first.pageInfo.endCursor ?? undefined, first: String(PAGE), }, }); From 8181a4eab7f9e4d262e28d5ed599be5b1df98883 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sat, 8 Aug 2026 21:38:07 +0200 Subject: [PATCH 11/14] feat(search): make drift diagnostics provider-aware, and page rebuilds by key Two gaps a healthy `core_search_index` was hiding. **The provider was never asked.** `SearchModel.index` writes the canonical row and *then* hands the document to the provider, so an Elasticsearch that refuses the second half leaves a canonical table that is perfectly correct and a search box that is missing results - which the diagnostic reported as `healthy: true`. The report now separates `canonicalIndexed` from `providerIndexed` and derives `healthy` from both. Providers answer through one optional, engine-neutral `count(c, { itemType, languageCode })`: - **Postgres** declares `capabilities.canonicalStorage` - its store *is* the canonical table, so it is verified without a second count of the same rows; - **Elasticsearch** implements it with `_count`, filtered by item type and language, so a per-locale gap is visible and no document is fetched to count; - **anything else** is reported `verified: false`, `healthy: null`. Absence of evidence is not a clean bill of health; - **a provider that throws** is `verified: true, healthy: false` with the reason attached and logged behind `[content-diagnostics]`. The status route still answers - a diagnostic that crashes when the thing it diagnoses is broken is one nobody can use. There is deliberately no single `indexed` number any more: with two storages, one number can only be a guess about which was meant. **The non-localized rebuild paged by OFFSET.** That re-reads and discards every earlier row, so a deep page pays for every page before it - and worse, the offset counts rows in a set that moves: a record unpublished after page one shifts everything behind it forward, and the next `OFFSET 100` steps over a row nobody ever indexed. It now seeks on `WHERE id > :last`, like the localized indexer already did, with `offset === 0` as the "fresh pass" signal the shared contract provides. The non-snapshot semantics are documented rather than implied. **Health semantics** are explicit: `searchHealthy && effectsHealthy`. A pending schedule is normal, and so is a pending one whose last attempt threw - the transition has not happened and the queue is retrying. Only `effectsError` - committed and unannounced - makes the engine unhealthy. Co-Authored-By: Claude Opus 5 (1M context) --- .../content-engine-observability.mdx | 157 +++-- .../performance-and-scaling.mdx | 23 +- packages/elasticsearch/src/index.test.ts | 77 +++ packages/elasticsearch/src/index.ts | 33 ++ .../src/api/adapters/search/postgres.ts | 10 +- .../vitnode/src/api/models/search.test.ts | 47 ++ packages/vitnode/src/api/models/search.ts | 54 ++ .../debug/routes/content-status.route.ts | 22 +- .../vitnode/src/content/server/diagnostics.ts | 245 +++++++- .../src/content/server/search-indexer.test.ts | 13 +- .../src/content/server/search-indexer.ts | 58 +- plugins/example/src/database/harness.ts | 44 ++ .../src/database/resilience-postgres.test.ts | 540 +++++++++++++++++- 13 files changed, 1211 insertions(+), 112 deletions(-) diff --git a/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx b/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx index 8fa5573f5..078652070 100644 --- a/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx +++ b/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx @@ -13,9 +13,13 @@ something looks wrong. ## Is the search index telling the truth? -The database is the source of truth. So "how many documents should there be" is a -`COUNT` over published rows, and "how many are there" is a `COUNT` over -`core_search_index`. A difference is drift. +Two storages, two questions. `SearchModel.index` writes the canonical +`core_search_index` row and *then* hands the document to the active provider, so +an Elasticsearch that refuses the second half leaves a canonical table that is +perfectly correct and a search box that is missing results. + +A diagnostic that only looked at the canonical table would call that healthy. So +this one asks both. ```http GET /api/@vitnode/core/admin/debug/content/status @@ -24,24 +28,38 @@ GET /api/@vitnode/core/admin/debug/content/status ```json { "healthy": false, + "searchHealthy": false, + "effectsHealthy": true, "contentTypes": [ { "contentTypeId": "example.localized-article", "pluginId": "@vitnode/example", - "features": { - "editorial": true, - "localization": true, - "publicApi": true, - "publication": true, - "scheduling": false, - "search": true - }, "search": { "contentTypeId": "example.localized-article", + "canonicalHealthy": true, "healthy": false, + "provider": { + "name": "elasticsearch", + "verified": true, + "healthy": false + }, "locales": [ - { "locale": "en", "expected": 421, "indexed": 421, "healthy": true }, - { "locale": "pl", "expected": 421, "indexed": 419, "healthy": false } + { + "locale": "en", + "expected": 421, + "canonicalIndexed": 421, + "canonicalHealthy": true, + "providerIndexed": 421, + "providerHealthy": true + }, + { + "locale": "pl", + "expected": 421, + "canonicalIndexed": 421, + "canonicalHealthy": true, + "providerIndexed": 419, + "providerHealthy": false + } ] }, "schedules": null @@ -50,28 +68,55 @@ GET /api/@vitnode/core/admin/debug/content/status } ``` -Two aggregates per content type, whatever its size, so this is safe to open on an -install with a large table. +### The vocabulary - -The existing search status page reports what is *in* the index, measured against -the indexer's own count. A collection can read as fully covered there and still -be missing every Polish document. This measures against the **rows**. +| Field | Means | +| ----- | ----- | +| `expected` | Published rows - or published translations - the **database** holds. | +| `canonicalIndexed` | Documents in `core_search_index`. | +| `canonicalHealthy` | The canonical table matches the database. | +| `providerIndexed` | Documents the **active provider** holds. `null` when it cannot say. | +| `providerHealthy` | The provider matches the database. `null` when nobody looked. | +| `verified` | Whether the provider was actually asked. | +| `healthy` | Canonical **and** provider both agree. | + +There is deliberately no single `indexed` number. Once there are two storages, +one number can only be a guess about which of them you meant. + + +A provider that offers no `count` reports `verified: false` and +`providerHealthy: null`, and the content type is **not** healthy. Absence of +evidence is reported as absence of evidence - turning it into a clean bill of +health is exactly how a broken search box hides behind a good canonical table. -Three things it can tell you: +### What each provider does + +| Provider | Behaviour | +| -------- | --------- | +| **Postgres** (bundled) | Its store *is* `core_search_index`, so it declares `canonicalStorage` and is verified without a second query. Canonical and provider always agree, because they are one thing. | +| **Elasticsearch** | Implements `count` with the `_count` API - a number, never fetched documents - filtered by `itemType` and, where applicable, `languageCode`. | +| **Anything else** | Reported as `verified: false` unless it implements `count`. | +| **A provider that throws** | `verified: true`, `healthy: false`, and `error` carries the reason. The status route still answers, and the failure is logged behind `[content-diagnostics]`. | -- **`indexed` below `expected`** - a live synchronisation that threw, or a - rebuild that stopped part-way. Fix the engine, then rebuild. -- **`indexed` above `expected`** - documents that outlived their records. Reported - as measured rather than clamped, because a stale document is exactly what needs - seeing. -- **One locale healthy and another not** - the shape a half-finished localized - rebuild makes, and the one a single total cannot show. +The expected side uses the **same** `publishedCondition` the indexer uses. +Re-deriving "published" here would let the diagnostic disagree with the thing it +is diagnosing, which is the one way a health check is worse than none. -The expected side uses the **same** `publishedCondition` the indexer uses. Re- -deriving "published" here would let the diagnostic disagree with the thing it is -diagnosing, which is the one way a health check is worse than none. +### Teaching your own provider to answer + +```ts +export const MySearchAdapter = (): SearchProviderApiPlugin => ({ + name: "my-engine", + // ... + count: async (c, { itemType, languageCode }) => + await myEngine.count({ itemType, languageCode }), +}); +``` + +Count, do not fetch: a diagnostic over a large collection has to cost the same +as one over an empty one. Honour `languageCode` if your store keeps one document +per translation - a single total cannot say "Polish is missing forty documents". ### Repairing drift @@ -97,24 +142,48 @@ the same status route: | Field | Means | | ----- | ----- | -| `pending` | Bookings still waiting to fire. | -| `withErrors` | Pending bookings whose last run threw. The transition has not happened. | +| `pending` | Bookings still waiting to fire. Normal. | +| `withErrors` | Pending bookings whose last run threw. The transition has **not** happened and the queue is retrying, so this is visible without being a failure. | | `failedEffects` | Transitions that **did** happen and were never announced. | -`failedEffects` is the one worth alerting on: the record *is* published, and -nobody has been told. The effects task retries on the queue's backoff, so a -non-zero value that stays non-zero is an outage rather than a blip. Per booking, -the AdminCP schedule panel shows the reason. +`failedEffects` is the one that matters: the record *is* published, and nobody +has been told, and no amount of waiting fixes that on its own. The effects task +retries on the queue's backoff, so a non-zero value that stays non-zero is an +outage rather than a blip. Per booking, the AdminCP schedule panel shows the +reason. + +## What "healthy" means + +Three flags, because one would be misleading: + +```ts +{ + healthy: false, // searchHealthy && effectsHealthy + searchHealthy: true, // every searchable content type agrees, canonical and provider + effectsHealthy: false, // nothing committed without being announced +} +``` + +`healthy: true` beside `failedEffects: 15` is worse than no answer - it tells an +operator to stop looking. Splitting the dimensions means a search outage and an +announcement backlog are separately visible, and the headline is simply their +conjunction. + +A pending schedule is **not** unhealthy, and neither is a pending one whose last +attempt threw: in both cases the transition has not happened and the queue is +still working on it. Only `effectsError` - committed, unannounced - moves the +needle. ## The logs -Two greppable prefixes in `core_logs`, visible in AdminCP → Advanced → Logs. +Four greppable prefixes in `core_logs`, visible in AdminCP → Advanced → Logs. | Prefix | Written when | | ------ | ------------ | | `[content-search]` | An index write or delete threw after a committed mutation. | | `[content-effects]` | An event was emitted and a listener did not receive it. | | `[content-revalidate]` | A web origin refused or could not be reached. | +| `[content-diagnostics]` | The search provider could not be counted for a health check. | Each is a single JSON object behind the prefix, with the content type, the item, the operation and the underlying error - enough to find the record without @@ -140,10 +209,15 @@ import { // report the same order. const report = await contentEngineDiagnostics(c); -// Or one content type at a time. +// Or one content type at a time. `healthy` is canonical *and* provider, so +// check the two separately when you want to tell an index problem from an +// engine outage. const drift = await contentSearchDrift(c, { model: articleContent }); -if (!drift.healthy) { - await c.get("log").error(`[my-plugin] ${JSON.stringify(drift.locales)}`); +if (!drift.canonicalHealthy) { + await c.get("log").error(`[my-plugin] index drift: ${JSON.stringify(drift.locales)}`); +} +if (drift.provider.verified && drift.provider.healthy === false) { + await c.get("log").error(`[my-plugin] ${drift.provider.name}: ${drift.provider.error ?? "documents missing"}`); } // Or just the schedules, in one grouped query for many content types. @@ -160,6 +234,7 @@ nothing in it a visitor should read either. a rebuild, and a rebuild is a decision an operator makes. - **It does not compare documents.** A count is not a checksum: two stale documents still count as two. It catches the failure that actually happens - a - write that threw, a rebuild that stopped - for the price of two aggregates. + write that threw, a rebuild that stopped - for the price of two aggregates and + one provider count per locale. - **It keeps no history.** Every number is computed on demand. Trending them over time is what a monitoring system is for, and this is not one. diff --git a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx index 69fd5be2e..da4982552 100644 --- a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx +++ b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx @@ -170,6 +170,24 @@ than a migration that fails at deploy time. That includes the names nobody wrote down: a junction's primary key, its position constraint, its target index and a repeatable's position constraint. +## A rebuild walks by key, not by offset + +Both generated indexers page with a cursor: `(itemId, languageId)` for the +localized one, `id` for the other. `OFFSET` was wrong twice over - it re-reads +and discards every earlier row, so a deep page pays for every page before it, +and the offset counts rows in a set that is *moving*. A record unpublished after +page one shifts everything behind it forward by one, and the next `OFFSET 100` +steps straight over a row nobody ever indexed. + + +It walks the collection as it stands. A row published **ahead** of the cursor is +picked up by the same pass; one published behind it is not, and is indexed by its +own publish instead. A row unpublished before the walk reaches it is simply never +read; one unpublished after it was read has already had its document written, and +the live unpublish is what removes it. Nothing is skipped that the walk had not +already passed. + + ## Reads stay page-bound A page is a page whatever the table holds behind it: `first: 25` materialises 25 @@ -186,8 +204,9 @@ call it, and does not. is 200 and the default is 50; model a content type for anything larger. - **A to-many relation is capped** for the same reason - it is edited as a set in a picker, not paged. -- **A rebuild is a queue task**, paged, and safe to run against a large table. - It is not free; run it when you need it rather than on a schedule. +- **A rebuild is a queue task**, paged by key rather than by `OFFSET`, so page + five hundred costs what page one costs. It is not free; run it when you need it + rather than on a schedule. - **`totalCount` is a `COUNT(*)`** on every list page. On a very large table with a filter that no index covers, that is the expensive part of the request - and the fix is an index on the filter, not a change to the pagination. diff --git a/packages/elasticsearch/src/index.test.ts b/packages/elasticsearch/src/index.test.ts index 864f74194..8cf1d3b99 100644 --- a/packages/elasticsearch/src/index.test.ts +++ b/packages/elasticsearch/src/index.test.ts @@ -6,6 +6,7 @@ const { bulk, index, deleteByQuery, + countDocs, ping, search, exists, @@ -15,6 +16,7 @@ const { const bulk = vi.fn(); const del = vi.fn(); const deleteByQuery = vi.fn(); + const countDocs = vi.fn(); const ping = vi.fn(); const search = vi.fn(); const exists = vi.fn(); @@ -25,6 +27,7 @@ const { bulk, delete: del, deleteByQuery, + count: countDocs, ping, search, indices: { exists, create }, @@ -44,6 +47,7 @@ const { bulk, index, deleteByQuery, + countDocs, ping, search, exists, @@ -448,3 +452,76 @@ describe("ElasticsearchSearchAdapter.ping", () => { expect(await ElasticsearchSearchAdapter(config).ping?.(c)).toBe(false); }); }); + +/** + * Provider-level diagnostics. + * + * The canonical `core_search_index` and this index are two storages, and only + * one of them is what a visitor actually searches. A drift diagnostic that could + * not ask this one would report a perfectly healthy canonical table while the + * search box was missing results - which is the failure this API exists to make + * visible. + */ +describe("ElasticsearchSearchAdapter.count", () => { + it("declares itself countable, so diagnostics can verify it", () => { + const adapter = ElasticsearchSearchAdapter(config); + + expect(typeof adapter.count).toBe("function"); + // And it is *not* the canonical storage: it mirrors, so it can drift. + expect(adapter.capabilities?.canonicalStorage).toBeUndefined(); + }); + + it("counts one collection without fetching a single document", async () => { + countDocs.mockResolvedValue({ count: 42 }); + + const total = await ElasticsearchSearchAdapter(config).count?.(c, { + itemType: "blog_post", + }); + + expect(total).toBe(42); + // `_count`, never `_search`: a diagnostic over a large index has to cost + // the same as one over an empty one. + expect(search).not.toHaveBeenCalled(); + expect(countDocs.mock.calls[0][0]).toMatchObject({ + index: "test", + query: { bool: { filter: [{ term: { itemType: "blog_post" } }] } }, + }); + }); + + it("narrows to one language when asked", async () => { + // Per-locale is the whole point: "Polish is missing forty documents" is not + // something a single total can say. + countDocs.mockResolvedValue({ count: 7 }); + + await ElasticsearchSearchAdapter(config).count?.(c, { + itemType: "blog_post", + languageCode: "pl", + }); + + expect(countDocs.mock.calls[0][0].query.bool.filter).toEqual([ + { term: { itemType: "blog_post" } }, + { term: { languageCode: "pl" } }, + ]); + }); + + it("reads an index that does not exist yet as empty", async () => { + // An install that has never rebuilt has no index. That is drift to report, + // not a crash in the status route. + countDocs.mockResolvedValue({}); + + await expect( + ElasticsearchSearchAdapter(config).count?.(c, { itemType: "blog_post" }), + ).resolves.toBe(0); + expect(countDocs.mock.calls[0][1]).toMatchObject({ ignore: [404] }); + }); + + it("lets a transport failure surface, so the caller can report it", async () => { + // Swallowing it here would turn "Elasticsearch is down" into "zero + // documents", which reads as drift rather than as an outage. + countDocs.mockRejectedValue(new Error("connect ECONNREFUSED")); + + await expect( + ElasticsearchSearchAdapter(config).count?.(c, { itemType: "blog_post" }), + ).rejects.toThrow("connect ECONNREFUSED"); + }); +}); diff --git a/packages/elasticsearch/src/index.ts b/packages/elasticsearch/src/index.ts index 31947ba5d..08fee597f 100644 --- a/packages/elasticsearch/src/index.ts +++ b/packages/elasticsearch/src/index.ts @@ -294,6 +294,39 @@ export const ElasticsearchSearchAdapter = ( languageScopedDelete: true, }, + /** + * How many documents this index holds for one collection. + * + * `_count` rather than a search: it returns a number without fetching a + * single document, so a diagnostic over a large index costs the same as one + * over an empty one. `languageCode` narrows it to a single translation, + * which is what makes per-locale drift visible - "Polish is missing forty + * documents" is not something a total can say. + * + * A missing index means zero rather than an error: an install that has never + * rebuilt has no index yet, and that is drift to report, not a crash. + */ + count: async (_c, { itemType, languageCode }) => { + const response = await getClient().count( + { + index, + query: { + bool: { + filter: [ + { term: { itemType } }, + ...(languageCode === undefined + ? [] + : [{ term: { languageCode } }]), + ], + }, + }, + }, + { ignore: [404] }, + ); + + return response.count ?? 0; + }, + index: async (_c, doc) => { await ensureIndex(); await getClient().index({ diff --git a/packages/vitnode/src/api/adapters/search/postgres.ts b/packages/vitnode/src/api/adapters/search/postgres.ts index 88193f36f..3a4ee863f 100644 --- a/packages/vitnode/src/api/adapters/search/postgres.ts +++ b/packages/vitnode/src/api/adapters/search/postgres.ts @@ -75,11 +75,15 @@ const buildFilters = (params: SearchQueryParams): SQL | undefined => { export const PostgresSearchAdapter = (): SearchProviderApiPlugin => ({ name: "postgres", - // `languageScopedDelete` is true by construction: this provider's store *is* - // `core_search_index`, and `SearchModel.delete` already narrows that table by - // `languageCode` before it gets here. There is no second copy to keep in step. + // Two capabilities that are both true for the same reason: this provider's + // store *is* `core_search_index`. `languageScopedDelete` because + // `SearchModel.delete` already narrows that table by `languageCode` before it + // gets here, and `canonicalStorage` because there is no second copy to drift + // from - so diagnostics can report the canonical count as the provider count + // rather than asking the same table twice. capabilities: { authorBoost: false, + canonicalStorage: true, facets: false, languageScopedDelete: true, timeDecay: false, diff --git a/packages/vitnode/src/api/models/search.test.ts b/packages/vitnode/src/api/models/search.test.ts index c38cfe079..f9f9a034c 100644 --- a/packages/vitnode/src/api/models/search.test.ts +++ b/packages/vitnode/src/api/models/search.test.ts @@ -7,6 +7,7 @@ import { core_search_index } from "@/database/search"; import type { SearchDocument, SearchProviderApiPlugin } from "./search"; +import { PostgresSearchAdapter } from "../adapters/search/postgres"; import { assertSearchProviderCapabilities, normalizeSearchIndexerPage, @@ -376,3 +377,49 @@ describe("assertSearchProviderCapabilities", () => { ).not.toThrow(); }); }); + +/** + * The provider half of a search diagnostic. + * + * `SearchModel.index` writes the canonical row and *then* hands the document to + * the provider, so the two can disagree - and a diagnostic that cannot ask the + * provider would report the canonical table's health as the whole story. + */ +describe("provider diagnostics", () => { + const modelFor = (provider: SearchProviderApiPlugin) => + new SearchModel({ + get: (key: string) => + key === "core" ? { search: { adapter: provider } } : undefined, + } as never); + + it("reports the bundled Postgres provider as canonical storage", () => { + // Its store *is* `core_search_index`, so a diagnostic can use the canonical + // count rather than paying for a second one over the same rows. + expect(modelFor(PostgresSearchAdapter()).isCanonicalStorage()).toBe(true); + }); + + it("reports a mirroring provider as not canonical", () => { + expect(modelFor(createProvider()).isCanonicalStorage()).toBe(false); + }); + + it("answers null when the provider offers no count", async () => { + // `null` is not zero and not healthy - it means nobody looked, and the + // caller has to report that as unverified. + await expect( + modelFor(createProvider()).countDocuments({ itemType: "blog_post" }), + ).resolves.toBeNull(); + }); + + it("passes the item type and language straight through", async () => { + const count = vi.fn().mockResolvedValue(12); + const model = modelFor({ ...createProvider(), count }); + + await expect( + model.countDocuments({ itemType: "blog_post", languageCode: "pl" }), + ).resolves.toBe(12); + expect(count.mock.calls[0][1]).toEqual({ + itemType: "blog_post", + languageCode: "pl", + }); + }); +}); diff --git a/packages/vitnode/src/api/models/search.ts b/packages/vitnode/src/api/models/search.ts index cdd0e98e6..4d3db855b 100644 --- a/packages/vitnode/src/api/models/search.ts +++ b/packages/vitnode/src/api/models/search.ts @@ -86,6 +86,17 @@ export interface SearchResult { export interface SearchProviderCapabilities { authorBoost: boolean; + /** + * Whether the provider's store **is** `core_search_index`. + * + * True only for the bundled Postgres provider, which queries the canonical + * table directly rather than mirroring it. Diagnostics use this to skip a + * second count of the same rows: canonical and provider are one storage, so + * asking twice would cost a query to learn something already known. + * + * A mirroring provider - anything with its own store - must leave it unset. + */ + canonicalStorage?: boolean; facets: boolean; /** * Whether {@link SearchProviderApiPlugin.delete} honours its `languageCode`. @@ -285,6 +296,22 @@ export interface SearchProviderApiPlugin { bulkIndex: (c: Context, docs: SearchDocument[]) => Promise; capabilities?: SearchProviderCapabilities; clear: (c: Context, itemType?: string) => Promise; + /** + * How many documents the provider holds for one collection. + * + * Optional, and its absence is meaningful: a provider that cannot be counted + * is reported as **unverified** rather than healthy, because "we did not look" + * and "we looked and it was fine" are different answers and only one of them + * is worth acting on. + * + * It must count rather than fetch - `_count` on Elasticsearch, `COUNT(*)` on a + * table - and honour `languageCode` where the provider stores one document per + * translation. Omitting the language means every language. + */ + count?: ( + c: Context, + args: { itemType: string; languageCode?: string }, + ) => Promise; /** * Removes one item's documents. * @@ -444,6 +471,23 @@ export class SearchModel { await this.provider().clear(this.c, itemType); } + /** + * How many documents the **provider** holds, or `null` when it cannot say. + * + * `null` is not zero and not healthy: it means the provider offers no + * diagnostics, and a caller has to report that as unverified rather than + * turning an absence of evidence into a clean bill of health. + */ + async countDocuments(args: { + itemType: string; + languageCode?: string; + }): Promise { + const provider = this.provider(); + if (!provider.count) return null; + + return await provider.count(this.c, args); + } + /** * Removes one item from the index, in one language or in all of them. * @@ -484,6 +528,16 @@ export class SearchModel { await this.provider().index(this.c, clean); } + /** + * Whether the active provider's store is the canonical table itself. + * + * Diagnostics ask this before counting twice - see + * {@link SearchProviderCapabilities.canonicalStorage}. + */ + isCanonicalStorage(): boolean { + return this.provider().capabilities?.canonicalStorage === true; + } + name(): string { return this.provider().name; } diff --git a/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts b/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts index feeaf8cb3..647323ca1 100644 --- a/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts +++ b/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts @@ -5,12 +5,16 @@ import { CONFIG_PLUGIN } from "@/config"; import { contentEngineDiagnostics } from "@/content/server/diagnostics"; const localeDriftSchema = z.object({ + /** Documents `core_search_index` holds for this locale. */ + canonicalIndexed: z.number(), + canonicalHealthy: z.boolean(), /** Published rows - or published translations - the database holds. */ expected: z.number(), - healthy: z.boolean(), - indexed: z.number(), /** `""` for a content type that is not localized. */ locale: z.string(), + /** `null` when the provider offers no diagnostics - unverified, not healthy. */ + providerHealthy: z.boolean().nullable(), + providerIndexed: z.number().nullable(), }); const contentTypeSchema = z.object({ @@ -27,9 +31,19 @@ const contentTypeSchema = z.object({ /** `null` for a content type without `search`. */ search: z .object({ + canonicalHealthy: z.boolean(), contentTypeId: z.string(), + /** Canonical **and** provider both agree. Unverified is not healthy. */ healthy: z.boolean(), locales: z.array(localeDriftSchema), + provider: z.object({ + /** Why the provider could not be counted, when that is the answer. */ + error: z.string().optional(), + healthy: z.boolean().nullable(), + name: z.string(), + /** Whether the provider was actually asked. */ + verified: z.boolean(), + }), }) .nullable(), /** `null` for a content type without scheduling. */ @@ -71,7 +85,11 @@ export const contentStatusDebugAdminRoute = buildRoute({ "application/json": { schema: z.object({ contentTypes: z.array(contentTypeSchema), + /** No scheduled transition committed without being announced. */ + effectsHealthy: z.boolean(), + /** `searchHealthy && effectsHealthy`. */ healthy: z.boolean(), + searchHealthy: z.boolean(), }), }, }, diff --git a/packages/vitnode/src/content/server/diagnostics.ts b/packages/vitnode/src/content/server/diagnostics.ts index 3667d5ce7..3e075c33e 100644 --- a/packages/vitnode/src/content/server/diagnostics.ts +++ b/packages/vitnode/src/content/server/diagnostics.ts @@ -38,21 +38,21 @@ import { * queries, computed on demand, behind an admin permission. */ -/** One locale's share of a content type's index, expected against actual. */ +/** One locale's share of a content type's index, from every storage that has one. */ export interface ContentSearchDriftLocale { - /** Published rows (or published translations) the database holds. */ - expected: number; /** - * `true` when the two counts agree. + * `true` when the canonical table matches the database. * * A count is not a checksum: two documents can be stale and still count as * two. It is the cheap check that catches the failure that actually happens - * a live sync that threw, or a rebuild that stopped halfway - and it costs two * aggregates rather than a full comparison. */ - healthy: boolean; - /** Documents actually in the index for this locale. */ - indexed: number; + canonicalHealthy: boolean; + /** Documents the canonical `core_search_index` holds for this locale. */ + canonicalIndexed: number; + /** Published rows - or published translations - the database holds. */ + expected: number; /** * `""` for a content type that is not localized. * @@ -61,28 +61,66 @@ export interface ContentSearchDriftLocale { * the row really holds. */ locale: string; + /** + * `true` when the active provider matches, `false` when it does not, and + * `null` when nobody looked. + * + * The three-way answer is the point. A provider that offers no diagnostics + * cannot be called healthy, and calling it healthy anyway is how an + * Elasticsearch outage hides behind a perfectly good canonical table. + */ + providerHealthy: boolean | null; + /** Documents the active provider holds, or `null` when it cannot say. */ + providerIndexed: null | number; +} + +/** What the active search provider could be asked, and what it answered. */ +export interface ContentSearchDriftProvider { + /** Why the provider could not be counted, when that is the answer. */ + error?: string; + /** `null` when unverified - see {@link ContentSearchDriftLocale.providerHealthy}. */ + healthy: boolean | null; + name: string; + /** + * Whether the provider was actually asked. + * + * `false` means it offers no `count`, so nothing about its contents is known. + * `true` with `healthy: false` means it was asked and it disagreed - or it + * threw, and `error` says so. + */ + verified: boolean; } export interface ContentSearchDrift { + /** `true` when `core_search_index` matches the database in every locale. */ + canonicalHealthy: boolean; contentTypeId: string; - /** `true` when every locale agrees. */ + /** + * `true` only when the canonical table **and** the active provider both agree. + * + * An unverified provider is not healthy: absence of evidence is reported as + * absence of evidence, and the operator decides what to do about it. + */ healthy: boolean; locales: ContentSearchDriftLocale[]; + provider: ContentSearchDriftProvider; } /** - * Compares what the database says should be indexed against what is. + * Compares what the database says should be indexed against what is - in the + * canonical table **and** in the active search provider. * - * Two aggregates per content type, whatever its size: `GROUP BY languageCode` - * over the index, and one `COUNT` (or one grouped `COUNT` for a localized - * content type) over the published rows. Neither reads a row body, so this stays - * cheap on a table with a hundred thousand articles. + * The two are not the same question, and conflating them was the gap this + * closes. `SearchModel.index` writes `core_search_index` and then hands the + * document to the provider; an Elasticsearch that refuses the second half leaves + * a canonical table that is perfectly correct and a search box that is missing + * results. A diagnostic that only ever looked at the canonical table would call + * that healthy. * - * The expected side is deliberately the **same predicate the indexer uses** - - * `publishedCondition`, and for a localized content type the base row's - * predicate `AND` the translation's. Re-deriving "published" here would let the - * diagnostic disagree with the thing it is diagnosing, which is the one way a - * health check is worse than none. + * Costs: two aggregates for the canonical side, whatever the collection's size, + * plus one provider count per locale. The bundled Postgres provider is skipped + * entirely - its store *is* the canonical table, so asking twice would buy + * nothing. */ export const contentSearchDrift = async ( c: Context, @@ -110,24 +148,140 @@ export const contentSearchDrift = async ( const expectedByLocale = await expectedDocuments(c, model); - const locales = [ + const keys = [ ...new Set([...expectedByLocale.keys(), ...indexedByLocale.keys()]), - ] - .sort() - .map(locale => { - const expected = expectedByLocale.get(locale) ?? 0; - const indexed = indexedByLocale.get(locale) ?? 0; + ].sort(); - return { expected, healthy: expected === indexed, indexed, locale }; - }); + const provider = await providerCounts(c, { + contentTypeId, + locales: keys, + }); + + const locales: ContentSearchDriftLocale[] = keys.map(locale => { + const expected = expectedByLocale.get(locale) ?? 0; + const canonicalIndexed = indexedByLocale.get(locale) ?? 0; + const canonicalHealthy = expected === canonicalIndexed; + // The canonical count stands in for the provider only when they are one + // storage. Everywhere else an absent count means unverified, never healthy. + const providerIndexed = provider.canonical + ? canonicalIndexed + : (provider.byLocale.get(locale) ?? null); + + return { + canonicalHealthy, + canonicalIndexed, + expected, + locale, + providerHealthy: + providerIndexed === null ? null : providerIndexed === expected, + providerIndexed, + }; + }); + + const canonicalHealthy = locales.every(entry => entry.canonicalHealthy); + const providerHealthy = provider.verified + ? provider.error === undefined && + locales.every(entry => entry.providerHealthy === true) + : null; return { + canonicalHealthy, contentTypeId, - healthy: locales.every(entry => entry.healthy), + healthy: canonicalHealthy && providerHealthy === true, locales, + provider: { + ...(provider.error === undefined ? {} : { error: provider.error }), + healthy: providerHealthy, + name: provider.name, + verified: provider.verified, + }, }; }; +/** + * Asks the active provider how many documents it holds, per locale. + * + * Three outcomes, and the middle one is the one Stage 7 exists for: + * + * - **canonical storage** - the bundled Postgres provider. Verified, and the + * canonical counts are its counts; no query is issued. + * - **no `count`** - a provider that offers no diagnostics. Unverified, which is + * reported as such rather than turned into `healthy: true`. + * - **it threw** - Elasticsearch is down. Verified and unhealthy, with the + * reason attached, and the status route still answers: a diagnostic that + * crashes when the thing it diagnoses is broken is a diagnostic nobody can + * use. + */ +const providerCounts = async ( + c: Context, + { contentTypeId, locales }: { contentTypeId: string; locales: string[] }, +): Promise<{ + byLocale: Map; + canonical: boolean; + error?: string; + name: string; + verified: boolean; +}> => { + const search = c.get("search"); + const name = search.name(); + + if (search.isCanonicalStorage()) { + return { byLocale: new Map(), canonical: true, name, verified: true }; + } + + const byLocale = new Map(); + try { + for (const locale of locales) { + const counted = await search.countDocuments({ + itemType: contentTypeId, + // The empty locale is what a language-agnostic document is stored + // under, and it is a real value to filter on rather than "any". + languageCode: locale, + }); + if (counted === null) { + return { byLocale: new Map(), canonical: false, name, verified: false }; + } + byLocale.set(locale, counted); + } + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + await logDiagnosticFailure( + c, + `${CONTENT_DIAGNOSTICS_LOG_PREFIX} ${JSON.stringify({ + contentTypeId, + error: message, + provider: name, + })}`, + ); + + return { + byLocale: new Map(), + canonical: false, + error: message, + name, + verified: true, + }; + } + + return { byLocale, canonical: false, name, verified: true }; +}; + +/** The prefix a provider diagnostic failure is logged behind. */ +export const CONTENT_DIAGNOSTICS_LOG_PREFIX = "[content-diagnostics]"; + +/** Logging is best effort: it writes to the database, so it can fail too. */ +const logDiagnosticFailure = async ( + c: Context, + message: string, +): Promise => { + try { + await c.get("log").error(message); + } catch { + // eslint-disable-next-line no-console + console.error(`[VitNode] ${message}`); + } +}; + /** * How many documents this content type's rows *should* produce, per locale. * @@ -269,8 +423,32 @@ export interface ContentTypeDiagnostic { export interface ContentEngineDiagnostics { contentTypes: ContentTypeDiagnostic[]; - /** `true` when every searchable content type's index agrees with the database. */ + /** + * Whether anything scheduled committed and was never announced. + * + * A *pending* schedule is normal - it has not fired yet - and so is a pending + * one whose last attempt threw, because the transition has not happened and + * the queue is still retrying it. `effectsError` is the one that matters: the + * record **is** published and nobody was told, and no amount of waiting fixes + * it on its own. + */ + effectsHealthy: boolean; + /** + * `searchHealthy && effectsHealthy`. + * + * Explicit dimensions rather than one number, because `healthy: true` beside + * `failedEffects: 15` is worse than no answer - it tells an operator to stop + * looking. + */ healthy: boolean; + /** + * Whether every searchable content type agrees with the database - in the + * canonical table **and** in the active provider. + * + * A provider that offers no diagnostics leaves this `false`: unverified is not + * healthy, and the per-content-type `provider.verified` says which it was. + */ + searchHealthy: boolean; } /** @@ -319,8 +497,17 @@ export const contentEngineDiagnostics = async ( }); } + const searchHealthy = contentTypes.every( + entry => entry.search === null || entry.search.healthy, + ); + const effectsHealthy = contentTypes.every( + entry => (entry.schedules?.failedEffects ?? 0) === 0, + ); + return { contentTypes, - healthy: contentTypes.every(entry => entry.search?.healthy !== false), + effectsHealthy, + healthy: searchHealthy && effectsHealthy, + searchHealthy, }; }; diff --git a/packages/vitnode/src/content/server/search-indexer.test.ts b/packages/vitnode/src/content/server/search-indexer.test.ts index d16bce27b..e32a8dcb6 100644 --- a/packages/vitnode/src/content/server/search-indexer.test.ts +++ b/packages/vitnode/src/content/server/search-indexer.test.ts @@ -143,7 +143,18 @@ describe("generated content search indexer", () => { expect(opOf(calls, "orderBy")).toBeDefined(); expect(opOf(calls, "limit")).toBe(200); - expect(opOf(calls, "offset")).toBe(400); + }); + + it("never issues a SQL OFFSET, however deep the rebuild has gone", async () => { + // `OFFSET` re-reads and discards every earlier row, and counts rows in a + // set that moves underneath it - a record unpublished after an earlier + // page shifts the rest forward and the next page steps over one. The + // walk is a keyset seek on the primary key instead. + const { c, calls } = createDbMock([[]]); + + await indexerFor(searchable).load(c, 400, 200); + + expect(opOf(calls, "offset")).toBeUndefined(); }); it("maps every row into a document, and stamps the owning plugin", async () => { diff --git a/packages/vitnode/src/content/server/search-indexer.ts b/packages/vitnode/src/content/server/search-indexer.ts index 8d4c906ef..2d117fde6 100644 --- a/packages/vitnode/src/content/server/search-indexer.ts +++ b/packages/vitnode/src/content/server/search-indexer.ts @@ -214,6 +214,16 @@ export const createContentSearchIndexer = < ]), ); + /** + * The keyset cursor, per request. + * + * A `WeakMap` keyed by the Hono context, exactly as the localized indexer + * does: the rebuild task calls `load` repeatedly within one request, and the + * entry is collected with it. A fresh request starts at the beginning, which + * is what a rebuild means. + */ + const cursors = new WeakMap(); + return { itemType: definition.id, @@ -230,23 +240,49 @@ export const createContentSearchIndexer = < return row?.value ?? 0; }, - // Offset paging, which is what the contract exposes. Ordering by the primary - // key keeps pages from overlapping within one rebuild; a row whose - // publication state changes mid-rebuild can still shift, and that is what - // the next publish - or the next rebuild - repairs. - // - // `itemsRead` is the row count, not the document count. A published row with - // no usable title projects to nothing, and reporting that as "no items" would - // end the rebuild before the valid rows after it. + /** + * Keyset paging on `id`, not `OFFSET`. + * + * `OFFSET` was wrong twice over. It re-reads and discards every earlier row, + * so page 500 of a rebuild costs five hundred pages of work - and worse, the + * offset counts rows in a set that is *moving*: a record unpublished after + * page one shifts everything behind it forward by one, and the next + * `OFFSET 100` steps straight over a row nobody ever indexed. A rebuild that + * silently misses rows is the failure a rebuild exists to fix. + * + * `WHERE id > :last` has neither problem. It seeks on the primary key, and + * it is anchored to a value rather than to a position, so rows appearing or + * disappearing behind the cursor cannot move it. + * + * The `offset` argument stays in the signature because the + * {@link SearchIndexer} contract is shared with hand-written indexers; it is + * used only as the "this is a fresh pass" signal, exactly as the localized + * indexer uses it. + * + * `itemsRead` is the row count, not the document count. A published row with + * no usable title projects to nothing, and reporting that as "no items" + * would end the rebuild before the valid rows after it. + */ load: async (c, offset, limit) => { + // The contract's only signal that this is a fresh pass rather than the + // next page of one. + if (offset === 0) cursors.delete(c); + const cursor = cursors.get(c); + const rows = await c .get("db") .select(selection) .from(table) - .where(publishedCondition(published)) + .where( + cursor === undefined + ? publishedCondition(published) + : and(publishedCondition(published), gt(primaryCursor, cursor)), + ) .orderBy(asc(primaryCursor)) - .limit(limit) - .offset(offset); + .limit(limit); + + const last = rows.at(-1); + if (last && typeof last.id === "number") cursors.set(c, last.id); // One batch for the whole page, and only the collections the search // configuration names - never one query per document. diff --git a/plugins/example/src/database/harness.ts b/plugins/example/src/database/harness.ts index e8ec8bbef..0f45e3c72 100644 --- a/plugins/example/src/database/harness.ts +++ b/plugins/example/src/database/harness.ts @@ -141,6 +141,21 @@ export interface ContentTestHarness { readonly behaviour: { /** Listeners `emit` should report as having failed. */ eventFailures: RecordedEventFailure[]; + /** When set, the provider's `count` throws it. */ + providerCountError: Error | null; + /** + * What the provider's own diagnostics answer. + * + * `"canonical"` is the bundled Postgres provider - its store *is* + * `core_search_index`, so it is verified without a second query. + * `"unsupported"` is a provider with no `count`, which has to be reported as + * unverified rather than healthy. A `Map` is a mirroring provider that can + * be counted, keyed by locale (`""` for language-agnostic content) - which + * is how a provider that is missing documents the canonical table has is + * simulated at all. + */ + providerCounts: "canonical" | "unsupported" | Map; + providerName: string; /** * Web origins the revalidation bridge should post to. * @@ -265,6 +280,9 @@ export const createContentTestHarness = const logs: string[] = []; const behaviour: ContentTestHarness["behaviour"] = { eventFailures: [], + providerCountError: null, + providerCounts: "canonical", + providerName: "postgres", revalidateOrigins: [], searchError: null, }; @@ -284,6 +302,29 @@ export const createContentTestHarness = if (key === "db") return handle; if (key === "search") { return { + countDocuments: async ({ + languageCode, + }: { + itemType: string; + languageCode?: string; + }) => { + if (behaviour.providerCountError) { + throw behaviour.providerCountError; + } + if (behaviour.providerCounts === "unsupported") { + return await Promise.resolve(null); + } + if (behaviour.providerCounts === "canonical") { + return await Promise.resolve(0); + } + + return await Promise.resolve( + behaviour.providerCounts.get(languageCode ?? "") ?? 0, + ); + }, + isCanonicalStorage: () => + behaviour.providerCounts === "canonical", + name: () => behaviour.providerName, delete: async ( itemType: string, itemId: number, @@ -418,6 +459,9 @@ export const createContentTestHarness = emitted.length = 0; logs.length = 0; behaviour.eventFailures = []; + behaviour.providerCountError = null; + behaviour.providerCounts = "canonical"; + behaviour.providerName = "postgres"; behaviour.revalidateOrigins = []; behaviour.searchError = null; }, diff --git a/plugins/example/src/database/resilience-postgres.test.ts b/plugins/example/src/database/resilience-postgres.test.ts index 6f0a3730e..ea520f90b 100644 --- a/plugins/example/src/database/resilience-postgres.test.ts +++ b/plugins/example/src/database/resilience-postgres.test.ts @@ -115,6 +115,29 @@ const published = async () => { return { id: created.id, version: outcome?.version ?? created.version }; }; +/** + * Writes the canonical index rows a healthy install would hold. + * + * Shared, because "search is fine" is the baseline several tests need before + * they can say anything about a *different* dimension of health. + */ +const indexPublished = async (): Promise => { + const rows = await h.sql<{ id: number; title: string }[]>` + SELECT "id", "title" FROM "example_articles" + WHERE "status" = 'published' AND "publishedAt" IS NOT NULL + `; + for (const row of rows) { + await h.sql` + INSERT INTO "core_search_index" + ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${articleContentType.id}, ${row.id}, + '', ${row.title}, ${row.title}, now() + ) + `; + } +}; + const DEAD_LISTENER = { error: "Service unavailable", listener: "send-notification", @@ -660,6 +683,182 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { ); }); + /** + * The rebuild walks by key, not by offset. + * + * `OFFSET` counts rows in a set that is *moving*: a record unpublished after + * page one shifts everything behind it forward by one, and the next + * `OFFSET 100` steps straight over a row nobody ever indexed. A rebuild that + * silently misses rows is the failure a rebuild exists to fix. + */ + describe("while the collection changes underneath it", () => { + /** Reads one page at a time so the fixture can mutate between them. */ + const pager = () => { + const build = indexer(); + let offset = 0; + + return async (limit: number) => { + const page = await build.load(h.context, offset, limit); + offset += page.itemsRead; + + return page; + }; + }; + + it("visits every remaining row when an already-read one is unpublished", async () => { + // The regression, exactly: page one is read, one of *its* rows goes + // away, and the walk continues. With `OFFSET` the next page would start + // one row too far in and skip an untouched record forever. + const ids: number[] = []; + for (let index = 0; index < 10; index += 1) { + const { id } = await published(); + ids.push(id); + } + + const next = pager(); + const first = await next(5); + expect(first.itemsRead).toBe(5); + + // A row from the page just read is withdrawn. + await h.sql` + UPDATE "example_articles" SET "status" = 'draft' + WHERE "id" = ${ids[2]} + `; + + const seen = first.documents.map(document => document.itemId); + for (let page = 0; page < 10; page += 1) { + const result = await next(5); + if (result.itemsRead === 0) break; + seen.push(...result.documents.map(document => document.itemId)); + } + + // Every row, exactly once. The withdrawn one is in there because page + // one had already read it - the live unpublish is what removes its + // document, and that is a different mechanism. What matters here is + // that nothing *else* moved: with `OFFSET` the shift would have stepped + // over an untouched record and lost it for the whole rebuild. + expect(seen.sort((a, b) => a - b)).toEqual( + [...ids].sort((a, b) => a - b), + ); + expect(new Set(seen).size).toBe(ids.length); + }); + + it("simply never reaches a row unpublished before it got there", async () => { + const ids: number[] = []; + for (let index = 0; index < 10; index += 1) { + const { id } = await published(); + ids.push(id); + } + + const next = pager(); + await next(4); + + // Withdrawn while it is still ahead of the cursor. + await h.sql` + UPDATE "example_articles" SET "status" = 'draft' + WHERE "id" = ${ids[8]} + `; + + const seen: number[] = []; + for (let page = 0; page < 10; page += 1) { + const result = await next(4); + if (result.itemsRead === 0) break; + seen.push(...result.documents.map(document => document.itemId)); + } + + expect(seen).not.toContain(ids[8]); + // And nothing near it was disturbed. + expect(seen).toContain(ids[9]); + expect(seen).toContain(ids[7]); + }); + + /** + * A row published mid-rebuild with a **higher** identifier is picked up by + * the same pass, because the cursor has not reached it yet. One with a + * lower identifier is not - the walk is already past that point. + * + * That is the honest consequence of a keyset walk, and it is stated here + * rather than described as a snapshot: a rebuild is not one. + */ + it("picks up a row published ahead of the cursor, and not one behind it", async () => { + const ids: number[] = []; + for (let index = 0; index < 6; index += 1) { + const { id } = await published(); + ids.push(id); + } + + const next = pager(); + const first = await next(3); + expect(first.itemsRead).toBe(3); + + // One behind the cursor, one ahead of it. + const behind = await article(); + await h.sql` + UPDATE "example_articles" + SET "status" = 'published', "publishedAt" = now(), "id" = ${ids[0] - 1} + WHERE "id" = ${behind.id} + `; + const ahead = await published(); + + const seen: number[] = []; + for (let page = 0; page < 10; page += 1) { + const result = await next(3); + if (result.itemsRead === 0) break; + seen.push(...result.documents.map(document => document.itemId)); + } + + expect(seen).toContain(ahead.id); + expect(seen).not.toContain(ids[0] - 1); + }); + + it("issues no SQL OFFSET at all", async () => { + // The property, asserted against the statements the driver really sent. + await published(); + await published(); + await published(); + + const build = createContentSearchIndexer(articleContent, { + pluginId: CONFIG_PLUGIN.pluginId, + }); + + h.counted.reset(); + let offset = 0; + for (let page = 0; page < 5; page += 1) { + const result = await build.load(h.counted.context, offset, 2); + if (result.itemsRead === 0) break; + offset += result.itemsRead; + } + + expect(h.counted.queries).not.toHaveLength(0); + expect( + h.counted.queries.filter(query => /\boffset\b/i.test(query)), + ).toEqual([]); + // And it does seek by key instead. + expect( + h.counted.queries.some(query => /"id"\s*>\s*\$/.test(query)), + ).toBe(true); + }); + + it("restarts from the beginning when a fresh rebuild begins", async () => { + // `offset === 0` is the contract's only "this is a new pass" signal. + const ids: number[] = []; + for (let index = 0; index < 4; index += 1) { + const { id } = await published(); + ids.push(id); + } + + const build = indexer(); + await build.load(h.context, 0, 2); + await build.load(h.context, 2, 2); + + const restarted = await build.load(h.context, 0, 2); + + expect(restarted.documents.map(document => document.itemId)).toEqual( + ids.slice(0, 2), + ); + }); + }); + it("counts exactly the records it would index", async () => { await published(); await published(); @@ -772,24 +971,6 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { // ------------------------------------------------------------------------- describe("index drift is diagnosable", () => { - /** Writes the canonical index rows a healthy install would hold. */ - const indexPublished = async () => { - const rows = await h.sql<{ id: number; title: string }[]>` - SELECT "id", "title" FROM "example_articles" - WHERE "status" = 'published' AND "publishedAt" IS NOT NULL - `; - for (const row of rows) { - await h.sql` - INSERT INTO "core_search_index" - ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") - VALUES ( - ${CONFIG_PLUGIN.pluginId}, ${articleContentType.id}, ${row.id}, - '', ${row.title}, ${row.title}, now() - ) - `; - } - }; - it("reports a healthy index as healthy", async () => { await published(); await published(); @@ -800,14 +981,40 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { }); expect(drift).toMatchObject({ + canonicalHealthy: true, contentTypeId: articleContentType.id, healthy: true, }); expect(drift.locales).toEqual([ - { expected: 2, healthy: true, indexed: 2, locale: "" }, + { + canonicalHealthy: true, + canonicalIndexed: 2, + expected: 2, + locale: "", + providerHealthy: true, + providerIndexed: 2, + }, ]); }); + it("reports the bundled provider as verified without counting twice", async () => { + // Its store *is* `core_search_index`, so the canonical counts are its + // counts - asking the same table again would cost a query to learn + // something already known. + await published(); + await indexPublished(); + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.provider).toEqual({ + healthy: true, + name: "postgres", + verified: true, + }); + }); + it("reports a document the index never received", async () => { await published(); await published(); @@ -822,7 +1029,11 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { }); expect(drift.healthy).toBe(false); - expect(drift.locales[0]).toMatchObject({ expected: 2, indexed: 1 }); + expect(drift.canonicalHealthy).toBe(false); + expect(drift.locales[0]).toMatchObject({ + canonicalIndexed: 1, + expected: 2, + }); }); it("reports a document that outlived its record", async () => { @@ -845,7 +1056,111 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { // reported as measured rather than clamped - a stale document is exactly // what an operator needs to see. expect(drift.healthy).toBe(false); - expect(drift.locales[0]).toMatchObject({ expected: 1, indexed: 2 }); + expect(drift.locales[0]).toMatchObject({ + canonicalIndexed: 2, + expected: 1, + }); + }); + + /** + * The regression the whole provider split exists for. + * + * `SearchModel.index` writes the canonical row and *then* hands the document + * to the provider. An Elasticsearch that refuses the second half leaves a + * canonical table that is perfectly correct and a search box that is missing + * results - and a diagnostic that only ever looked at the canonical table + * would call that healthy. + */ + it("reports the provider unhealthy when only the provider is missing a document", async () => { + await published(); + await published(); + await indexPublished(); + + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = new Map([["", 1]]); + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.canonicalHealthy).toBe(true); + expect(drift.locales[0]).toMatchObject({ + canonicalHealthy: true, + canonicalIndexed: 2, + expected: 2, + providerHealthy: false, + providerIndexed: 1, + }); + expect(drift.provider).toMatchObject({ + healthy: false, + name: "elasticsearch", + verified: true, + }); + // The part that used to be wrong: a healthy canonical table is not a + // healthy search. + expect(drift.healthy).toBe(false); + }); + + it("reports a provider that cannot be counted as unverified, not healthy", async () => { + await published(); + await indexPublished(); + + h.behaviour.providerName = "custom-search"; + h.behaviour.providerCounts = "unsupported"; + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.canonicalHealthy).toBe(true); + expect(drift.locales[0].providerHealthy).toBeNull(); + expect(drift.locales[0].providerIndexed).toBeNull(); + expect(drift.provider).toEqual({ + healthy: null, + name: "custom-search", + verified: false, + }); + // Absence of evidence is not a clean bill of health. + expect(drift.healthy).toBe(false); + }); + + it("stays usable when the provider itself is unavailable", async () => { + await published(); + await indexPublished(); + + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = new Map([["", 1]]); + h.behaviour.providerCountError = new Error("connect ECONNREFUSED"); + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + // It answers rather than throwing: a diagnostic that crashes when the + // thing it diagnoses is broken is a diagnostic nobody can use. + expect(drift.canonicalHealthy).toBe(true); + expect(drift.provider).toMatchObject({ + error: "connect ECONNREFUSED", + healthy: false, + verified: true, + }); + expect(drift.healthy).toBe(false); + expect(h.logs.some(line => line.includes("[content-diagnostics]"))).toBe( + true, + ); + }); + + it("keeps the whole status route answering when the provider is down", async () => { + await published(); + await indexPublished(); + h.behaviour.providerCounts = new Map([["", 1]]); + h.behaviour.providerCountError = new Error("elasticsearch unavailable"); + + const report = await contentEngineDiagnostics(h.context); + + expect(report.contentTypes).not.toHaveLength(0); + expect(report.searchHealthy).toBe(false); + expect(report.healthy).toBe(false); }); it("counts a localized content type per locale", async () => { @@ -891,11 +1206,82 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { expect(drift.healthy).toBe(false); expect(drift.locales).toEqual([ - { expected: 1, healthy: true, indexed: 1, locale: "en" }, - { expected: 1, healthy: false, indexed: 0, locale: "pl" }, + { + canonicalHealthy: true, + canonicalIndexed: 1, + expected: 1, + locale: "en", + providerHealthy: true, + providerIndexed: 1, + }, + { + canonicalHealthy: false, + canonicalIndexed: 0, + expected: 1, + locale: "pl", + providerHealthy: false, + providerIndexed: 0, + }, ]); }); + it("reports one locale unhealthy when only that locale is missing from the provider", async () => { + // The localized shape of the same regression: English agrees everywhere, + // Polish is in the canonical table and absent from the provider. A single + // total cannot show that; a per-locale provider count can. + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: "English body", title: "Locale Drift" }, + }, + { actor: ACTOR }, + ); + const base = localizedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + await translationEditorial(h.context).create( + row.id, + "pl", + { body: "Tresc", title: "Polski Locale" }, + { actor: ACTOR }, + ); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + row.id, + { actor: ACTOR }, + ); + for (const locale of ["en", "pl"] as const) { + await translationEditorial(h.context).publish(row.id, locale, { + actor: ACTOR, + }); + await h.sql` + INSERT INTO "core_search_index" + ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${localizedArticleContent.definition.id}, + ${row.id}, ${locale}, 'Locale Drift', 'Locale Drift', now() + ) + `; + } + + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = new Map([ + ["en", 1], + ["pl", 0], + ]); + + const drift = await contentSearchDrift(h.context, { + model: localizedArticleContent, + }); + + expect(drift.canonicalHealthy).toBe(true); + expect( + drift.locales.map(entry => [entry.locale, entry.providerHealthy]), + ).toEqual([ + ["en", true], + ["pl", false], + ]); + expect(drift.healthy).toBe(false); + }); + it("agrees with the localized rebuild about how many documents there should be", async () => { const { row } = await localizedService(h.context).create( { @@ -986,4 +1372,112 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { ).toBeNull(); }); }); + + // ------------------------------------------------------------------------- + // Overall health + // ------------------------------------------------------------------------- + + /** + * `healthy: true` beside `failedEffects: 15` is worse than no answer - it + * tells an operator to stop looking. So the report carries the two dimensions + * separately and derives the headline from them. + */ + describe("overall health", () => { + const bookSchedule = async (itemId: number) => { + const model = editorial(h.context).schedules; + if (!model) throw new Error("no scheduling"); + + return await model.schedule({ + action: "unpublish", + actorUserId: null, + itemId, + scheduledFor: new Date(Date.now() + 3_600_000), + }); + }; + + it("is healthy when search agrees and nothing is outstanding", async () => { + const report = await contentEngineDiagnostics(h.context); + + expect(report).toMatchObject({ + effectsHealthy: true, + healthy: true, + searchHealthy: true, + }); + }); + + it("treats a pending schedule as normal rather than unhealthy", async () => { + // It has not fired yet. Nothing is wrong. + const { id } = await published(); + await indexPublished(); + await bookSchedule(id); + + const report = await contentEngineDiagnostics(h.context); + + expect(report.effectsHealthy).toBe(true); + expect(report.healthy).toBe(true); + expect( + report.contentTypes.find( + item => item.contentTypeId === articleContentType.id, + )?.schedules?.pending, + ).toBe(1); + }); + + it("treats a pending schedule whose last attempt threw as still pending", async () => { + // The transition has not happened and the queue is retrying it, so this + // is visible - `withErrors` - without being a failure of the engine. + const { id } = await published(); + await indexPublished(); + const booked = await bookSchedule(id); + await h.sql` + UPDATE "core_content_schedules" + SET "lastError" = 'connection reset' + WHERE "id" = ${booked.id} + `; + + const report = await contentEngineDiagnostics(h.context); + + expect(report.effectsHealthy).toBe(true); + expect(report.healthy).toBe(true); + expect( + report.contentTypes.find( + item => item.contentTypeId === articleContentType.id, + )?.schedules?.withErrors, + ).toBe(1); + }); + + it("is unhealthy when a committed transition was never announced", async () => { + // The record *is* published and nobody was told. No amount of waiting + // fixes that on its own, so it is the one that moves the headline. + const { id } = await published(); + await indexPublished(); + const booked = await bookSchedule(id); + await h.sql` + UPDATE "core_content_schedules" + SET "effectsError" = 'search: down' + WHERE "id" = ${booked.id} + `; + + const report = await contentEngineDiagnostics(h.context); + + expect(report).toMatchObject({ + effectsHealthy: false, + healthy: false, + // Search is fine; the headline is not, and the two are separable. + searchHealthy: true, + }); + }); + + it("is unhealthy when search drifts even though nothing is outstanding", async () => { + await published(); + // Nothing indexed at all, so the canonical table disagrees. + + const report = await contentEngineDiagnostics(h.context); + + expect(report).toMatchObject({ + effectsHealthy: true, + healthy: false, + searchHealthy: false, + }); + }); + }); }); From 29ce1a460b7b590eab402a29744218a3ff8b5e35 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sun, 9 Aug 2026 13:51:27 +0200 Subject: [PATCH 12/14] fix(pagination): keep the cursor a historical position, and catch provider ghosts Two things the last round got wrong, both in the direction of trusting current state where a snapshot was needed. **The cursor re-read the boundary row.** It was introduced to solve a real problem - a Postgres timestamp keeps microseconds and a JavaScript Date keeps milliseconds, so a value that has been out to a client and back is strictly smaller than the one still in the table - but the cure broke the invariant a cursor exists for. A cursor is the position as it stood when the page was generated. Re-reading meant editing the boundary row dragged the boundary with it: move id=42 from 10:00 to 14:00 and every row at 10:01, 11:00, 12:00 and 13:00 is skipped for good, with no error and no short page to notice it by. The cursor is now self-contained. A date carries the column's own `::text` - microseconds included - captured while the row is still the boundary, as part of minting; the predicate binds it back with an explicit cast so Postgres parses it at the precision it wrote. Nothing reads the boundary row when the cursor comes back, so updating it, moving it or deleting it changes nothing. Serialization is per column kind rather than generic coercion, and the decode side validates instead of coercing - `Boolean("false")` is `true`, `Number("")` is `0`, and `BigInt("nonsense")` throws a `SyntaxError` that would have surfaced as a 500. Each of those is now a 400, and no malformed value reaches Postgres as a failed cast. **Provider diagnostics could not see a ghost.** Deletion runs canonical-first, so a failed provider delete leaves a document in a locale that no longer appears in the database *or* the canonical table - and per-locale enumeration is built from exactly those two, so nothing ever asked about it. On a content type with nothing in it, `[].every(...)` said healthy. `provider.indexedTotal` is now an unfiltered count, asked unconditionally, and `providerHealthy` requires it to equal `expectedTotal` as well as every locale agreeing. The same total closes the canonical side, and the bundled Postgres provider still answers from the canonical count without a second query. Co-Authored-By: Claude Opus 5 (1M context) --- .../content-engine-observability.mdx | 44 ++- .../performance-and-scaling.mdx | 51 ++- packages/elasticsearch/src/index.test.ts | 29 ++ .../src/api/lib/pagination-cursor.test.ts | 207 ++++++++++++- .../vitnode/src/api/lib/pagination-cursor.ts | 223 ++++++++++---- .../vitnode/src/api/lib/with-pagination.ts | 95 +++--- .../debug/routes/content-status.route.ts | 11 + .../vitnode/src/content/server/diagnostics.ts | 113 +++++-- .../src/content/server/service.test.ts | 10 +- plugins/example/src/database/harness.ts | 23 +- .../src/database/pagination-postgres.test.ts | 290 ++++++++++++++++++ .../src/database/resilience-postgres.test.ts | 226 +++++++++++++- 12 files changed, 1171 insertions(+), 151 deletions(-) diff --git a/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx b/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx index 078652070..b3cdfeeb7 100644 --- a/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx +++ b/apps/docs/content/docs/dev/content-engine/content-engine-observability.mdx @@ -75,14 +75,41 @@ GET /api/@vitnode/core/admin/debug/content/status | `expected` | Published rows - or published translations - the **database** holds. | | `canonicalIndexed` | Documents in `core_search_index`. | | `canonicalHealthy` | The canonical table matches the database. | -| `providerIndexed` | Documents the **active provider** holds. `null` when it cannot say. | -| `providerHealthy` | The provider matches the database. `null` when nobody looked. | +| `providerIndexed` | Documents the **active provider** holds for this locale. `null` when it cannot say. | +| `providerHealthy` | The provider matches the database for this locale. `null` when nobody looked. | +| `expectedTotal` | Every published row or translation, all locales. | +| `canonicalIndexedTotal` | Every canonical document, all locales. | +| `provider.indexedTotal` | Every provider document, **all locales, including ones nothing knows about**. | | `verified` | Whether the provider was actually asked. | -| `healthy` | Canonical **and** provider both agree. | +| `healthy` | Canonical **and** provider agree, per locale *and* in total. | There is deliberately no single `indexed` number. Once there are two storages, one number can only be a guess about which of them you meant. +### Why the totals matter as well as the locales + +Per-locale counts can only ask about locales somebody already knows to ask for - +and that list is built from the database and the canonical table. A document that +exists **only** in the provider is invisible to it. + +That is not hypothetical. Deletion runs canonical-first: `SearchModel.delete` +removes the canonical row and then asks the provider. If the provider's half +fails, the document survives in a locale that no longer appears in either source, +so nothing enumerates it: + +```text +database expected pl: 0 +core_search_index pl: 0 +Elasticsearch pl: 1 ← nothing asks about "pl" any more +``` + +An unfiltered total finds it, because it is not built from an enumeration: +`providerTotal > expectedTotal` is enough to say something is wrong even when +every locale anybody checked agreed. The same guard covers a locale that was +removed from the installation entirely, and a content type with **no** rows at +all - where the per-locale list is empty and `[].every(...)` would otherwise say +everything is fine. + A provider that offers no `count` reports `verified: false` and `providerHealthy: null`, and the content type is **not** healthy. Absence of @@ -95,9 +122,9 @@ health is exactly how a broken search box hides behind a good canonical table. | Provider | Behaviour | | -------- | --------- | | **Postgres** (bundled) | Its store *is* `core_search_index`, so it declares `canonicalStorage` and is verified without a second query. Canonical and provider always agree, because they are one thing. | -| **Elasticsearch** | Implements `count` with the `_count` API - a number, never fetched documents - filtered by `itemType` and, where applicable, `languageCode`. | +| **Elasticsearch** | Implements `count` with the `_count` API - a number, never fetched documents - filtered by `itemType`, and by `languageCode` when one is given. Omitting the language is the unfiltered total. It creates no index as a side effect: a diagnostic is observational. | | **Anything else** | Reported as `verified: false` unless it implements `count`. | -| **A provider that throws** | `verified: true`, `healthy: false`, and `error` carries the reason. The status route still answers, and the failure is logged behind `[content-diagnostics]`. | +| **A provider that throws** | `verified: true`, `healthy: false`, and `error` carries the reason. A failure in either the total or a per-locale count is handled the same way. The status route still answers, and the failure is logged behind `[content-diagnostics]`. | The expected side uses the **same** `publishedCondition` the indexer uses. Re-deriving "published" here would let the diagnostic disagree with the thing it @@ -116,7 +143,12 @@ export const MySearchAdapter = (): SearchProviderApiPlugin => ({ Count, do not fetch: a diagnostic over a large collection has to cost the same as one over an empty one. Honour `languageCode` if your store keeps one document -per translation - a single total cannot say "Polish is missing forty documents". +per translation - a single total cannot say "Polish is missing forty documents" - +and treat an **omitted** `languageCode` as every language, because that call is +what finds a document in a locale nobody thought to ask about. + +Make it observational. Creating an index, warming a cache or writing anything +would let the health check change the thing it is measuring. ### Repairing drift diff --git a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx index da4982552..61494fd1f 100644 --- a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx +++ b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx @@ -82,18 +82,59 @@ thing in reverse and flips the page back afterwards. The `ORDER BY` and the predicate are built from the same tuple in the same direction, which is the invariant everything else rests on. - + It is `base64url(JSON)` carrying the column, the value and the identifier - so it is meaningless outside the ordering that produced it, and it says so: a cursor minted while a list was ordered by `updatedAt` and replayed against the same list ordered by `title` is a `400`, not a page of wrong rows. Hand it back unchanged; never parse it, and never build one. + +Nothing re-reads the row it came from. A cursor is the position **as it stood +when the page was generated**, so editing or deleting the row that happened to +sit on the boundary does not move it. Re-reading would mean one edit silently +skips every row the ordering used to have between the old position and the new +one - a page of results nobody ever sees, with no error and no short page to +notice it by. +### Timestamps travel as the database wrote them + +A Postgres `timestamp` keeps microseconds; a JavaScript `Date` keeps +milliseconds. A boundary value that had been through a `Date` would be strictly +*smaller* than the one still in the table, and the next page would exclude the +whole millisecond it came from - which, since `now()` stamps every row in one +statement identically, would end a bulk-imported collection's walk after page +one. + +So a date cursor carries the column's own `::text` and the predicate binds it +back with an explicit cast. The value in the cursor is byte-identical to the +value the next comparison is parsed from. + +### A tampered cursor is refused, not coerced + +The cursor is opaque but not signed, so every field is checked against the +column it claims to describe: + +| Column | Accepted | +| ------ | -------- | +| number | a JSON number, finite | +| bigint | a decimal integer string | +| boolean | a JSON boolean | +| string | a JSON string | +| date | a canonical Postgres timestamp, `timestamptz`, date or ISO string | +| any | `null`, which is a real position | + +Coercion is the failure mode being avoided, not an inelegance: `Boolean("false")` +is `true`, `Number("")` is `0`, and `BigInt("nonsense")` throws a `SyntaxError` +that would leave the route answering `500`. Every mismatch is a `400`, and no +malformed value ever reaches Postgres as a failed cast. + **What is guaranteed:** - **any** orderable column pages exactly - a title, a nullable `publishedAt`, a custom field, whatever its relationship to the identifier; +- the boundary is **stable**: updating or deleting the row the cursor named does + not move it, and every row that was after that position is still reachable; - rows sharing a sort value are returned exactly once, because the identifier is the tiebreaker in both the ordering and the predicate; - a nullable order column walks through its null block correctly - Postgres sorts @@ -108,9 +149,11 @@ unchanged; never parse it, and never build one. **What is not:** -Rows inserted behind the cursor are not seen until the next pass. That is the -trade every keyset pagination makes, and it is the right one - the alternative is -holding a transaction open across requests. +Rows inserted behind the cursor are not seen until the next pass, and a row that +is *edited* so that it moves from behind the cursor to ahead of it is seen a +second time. That is the trade every keyset pagination makes, and it is the right +one - the alternative is holding a transaction open across requests. What never +happens is a row being skipped because something *else* moved. ### Invalid input is refused, not repaired diff --git a/packages/elasticsearch/src/index.test.ts b/packages/elasticsearch/src/index.test.ts index 8cf1d3b99..d207eb526 100644 --- a/packages/elasticsearch/src/index.test.ts +++ b/packages/elasticsearch/src/index.test.ts @@ -515,6 +515,35 @@ describe("ElasticsearchSearchAdapter.count", () => { expect(countDocs.mock.calls[0][1]).toMatchObject({ ignore: [404] }); }); + it("counts every language when none is named", async () => { + // The unfiltered total, which is what catches a document left behind in a + // locale the content type no longer has - per-locale counts can only ask + // about locales somebody already knows to ask for. + countDocs.mockResolvedValue({ count: 9 }); + + await ElasticsearchSearchAdapter(config).count?.(c, { + itemType: "blog_post", + }); + + expect(countDocs.mock.calls[0][0].query.bool.filter).toEqual([ + { term: { itemType: "blog_post" } }, + ]); + }); + + it("creates no index as a side effect", async () => { + // A diagnostic is observational. Creating the index here would make the + // health check change the thing it is measuring - and would report a + // never-rebuilt install as a healthy empty one. + countDocs.mockResolvedValue({ count: 0 }); + + await ElasticsearchSearchAdapter(config).count?.(c, { + itemType: "blog_post", + }); + + expect(create).not.toHaveBeenCalled(); + expect(exists).not.toHaveBeenCalled(); + }); + it("lets a transport failure surface, so the caller can report it", async () => { // Swallowing it here would turn "Elasticsearch is down" into "zero // documents", which reads as drift rather than as an outage. diff --git a/packages/vitnode/src/api/lib/pagination-cursor.test.ts b/packages/vitnode/src/api/lib/pagination-cursor.test.ts index 3b7967873..9f9d491cd 100644 --- a/packages/vitnode/src/api/lib/pagination-cursor.test.ts +++ b/packages/vitnode/src/api/lib/pagination-cursor.test.ts @@ -1,3 +1,4 @@ +import { bigint, pgTable, timestamp } from "drizzle-orm/pg-core"; // @vitest-environment node import { HTTPException } from "hono/http-exception"; import { describe, expect, it } from "vitest"; @@ -6,12 +7,19 @@ import { core_users } from "@/database/users"; import { cursorValueForColumn, + cursorValueIsCanonicalText, cursorValueOf, decodePaginationCursor, encodePaginationCursor, isCursorSortableColumn, } from "./pagination-cursor"; +/** A bigint order column, which core has only on a table without one. */ +const probes = pgTable("cursor_probes", { + big: bigint({ mode: "bigint" }), + moment: timestamp(), +}); + /** * The cursor is the ordered tuple, or it is nothing. * @@ -160,14 +168,21 @@ describe("legacy numeric cursors", () => { }); describe("column values", () => { - it("flattens a Date to an ISO string and back", () => { - const date = new Date("2026-08-08T12:00:00.000Z"); - const flattened = cursorValueOf(core_users.createdAt, date); + it("keeps a timestamp as text on both sides", () => { + // Deliberately *not* a `Date` round trip. A `Date` holds milliseconds and + // Postgres holds microseconds, so turning the text back into one would + // truncate the value the next comparison is parsed from - and exclude the + // whole millisecond the cursor came from. The predicate casts this text + // back to the column's type instead, and lets Postgres do the parsing. + const flattened = cursorValueOf( + core_users.createdAt, + "2026-08-08 12:00:00.123456", + ); - expect(flattened).toBe("2026-08-08T12:00:00.000Z"); - // Back as a `Date`, because that is what the column compares against - - // handing Postgres the string would compare a timestamp with text. - expect(cursorValueForColumn(core_users.createdAt, flattened)).toEqual(date); + expect(flattened).toBe("2026-08-08 12:00:00.123456"); + expect(cursorValueForColumn(core_users.createdAt, flattened)).toBe( + "2026-08-08 12:00:00.123456", + ); }); it("keeps a string a string", () => { @@ -185,7 +200,7 @@ describe("column values", () => { expect(cursorValueForColumn(core_users.createdAt, null)).toBeNull(); }); - it("refuses an unparseable date rather than comparing against Invalid Date", () => { + it("refuses an unparseable date rather than letting Postgres fail the cast", () => { expect(() => cursorValueForColumn(core_users.createdAt, "not-a-date"), ).toThrow(HTTPException); @@ -196,5 +211,181 @@ describe("column values", () => { expect(isCursorSortableColumn(core_users.name)).toBe(true); expect(isCursorSortableColumn(core_users.createdAt)).toBe(true); expect(isCursorSortableColumn(core_users.newsletter)).toBe(true); + expect(isCursorSortableColumn(probes.big)).toBe(true); + }); +}); + +/** + * A cursor is opaque, not signed. A client can edit it. + * + * So every field is hostile input, checked against the column it claims to + * describe. Coercion is the failure mode to avoid, not just an inelegance: + * `Boolean("false")` is `true`, `Number("")` is `0`, and `BigInt("nonsense")` + * throws a `SyntaxError` that would leave the route as a 500. Each of those is + * a wrong page or a wrong status code handed to somebody who asked for neither. + */ +describe("a tampered cursor value is refused, never coerced", () => { + const refuses = ( + column: Parameters[0], + value: unknown, + ) => { + expect(() => cursorValueForColumn(column, value as never)).toThrow( + HTTPException, + ); + try { + cursorValueForColumn(column, value as never); + } catch (error) { + expect(statusOf(error)).toBe(400); + } + }; + + describe("boolean", () => { + it('refuses the string "false", which coercion would read as true', () => { + refuses(core_users.newsletter, "false"); + }); + + it.each([ + ["a number", 0], + ["a string", "true"], + ["an empty string", ""], + ])("refuses %s", (_why, value) => { + refuses(core_users.newsletter, value); + }); + + it("accepts a real boolean, and null", () => { + expect(cursorValueForColumn(core_users.newsletter, false)).toBe(false); + expect(cursorValueForColumn(core_users.newsletter, true)).toBe(true); + expect(cursorValueForColumn(core_users.newsletter, null)).toBeNull(); + }); + }); + + describe("number", () => { + it.each([ + ["a numeric string", "42"], + ["an empty string", ""], + ["a boolean", true], + ["nonsense", "not-a-number"], + ])("refuses %s", (_why, value) => { + refuses(core_users.id, value); + }); + + it("accepts a finite number, and null", () => { + expect(cursorValueForColumn(core_users.id, 42)).toBe(42); + expect(cursorValueForColumn(core_users.id, null)).toBeNull(); + }); + }); + + describe("bigint", () => { + it("refuses a value that would make BigInt() throw", () => { + // The one that used to escape as a native `SyntaxError`, and therefore + // as a 500. + refuses(probes.big, "not-a-bigint"); + }); + + it.each([ + ["a fractional string", "1.5"], + ["an empty string", ""], + ["a number", 12], + ["a boolean", false], + ["whitespace", " 12 "], + ])("refuses %s", (_why, value) => { + refuses(probes.big, value); + }); + + it("accepts a decimal integer string, signed or not, and null", () => { + expect(cursorValueForColumn(probes.big, "9007199254740993")).toBe( + 9007199254740993n, + ); + expect(cursorValueForColumn(probes.big, "-4")).toBe(-4n); + expect(cursorValueForColumn(probes.big, null)).toBeNull(); + }); + }); + + describe("timestamp", () => { + it.each([ + ["nonsense", "not-a-date"], + ["a number", 1_700_000_000], + ["a boolean", true], + ["a half-written date", "2026-08"], + ["an injection attempt", "2026-08-09'; DROP TABLE users; --"], + ])("refuses %s", (_why, value) => { + // Reaching Postgres with any of these would be an invalid-cast 500 + // rather than a 400 - and the last one has no business getting near a + // cast at all. + refuses(core_users.createdAt, value); + }); + + it.each([ + ["a Postgres timestamp", "2026-08-09 10:00:00.123456"], + ["a Postgres timestamptz", "2026-08-09 10:00:00.123456+00"], + ["a plain date", "2026-08-09"], + ["an ISO string", "2026-08-09T10:00:00.123Z"], + ])("accepts %s, unchanged", (_why, value) => { + // Unchanged is the point: the predicate casts this text back to the + // column's type, so Postgres parses it at the precision it stored. + expect(cursorValueForColumn(core_users.createdAt, value)).toBe(value); + }); + + it("is the one kind bound as text plus a cast", () => { + expect(cursorValueIsCanonicalText(core_users.createdAt)).toBe(true); + expect(cursorValueIsCanonicalText(core_users.id)).toBe(false); + expect(cursorValueIsCanonicalText(core_users.name)).toBe(false); + }); + }); + + describe("string", () => { + it.each([ + ["a number", 12], + ["a boolean", true], + ])("refuses %s", (_why, value) => { + refuses(core_users.name, value); + }); + + it("accepts a string, and null", () => { + expect(cursorValueForColumn(core_users.name, "Ada")).toBe("Ada"); + expect(cursorValueForColumn(core_users.name, null)).toBeNull(); + }); + }); + + it("never lets a native parser error escape", () => { + // Whatever is thrown, it is an `HTTPException` - not a `SyntaxError`, a + // `RangeError`, or anything else that would surface as a 500. + const hostile = [ + [probes.big, "nope"], + [core_users.createdAt, "nope"], + [core_users.newsletter, "nope"], + [core_users.id, "nope"], + ] as const; + + for (const [column, value] of hostile) { + try { + cursorValueForColumn(column, value); + throw new Error(`Expected ${column.name} to refuse ${value}.`); + } catch (error) { + expect(error).toBeInstanceOf(HTTPException); + } + } + }); +}); + +describe("minting keeps the database's own representation", () => { + it("keeps a Postgres timestamp string exactly as it was read", () => { + // Microseconds and all: this is the value the next comparison is parsed + // from, so anything lost here is lost from the ordering. + expect( + cursorValueOf(core_users.createdAt, "2026-08-09 10:00:00.123456"), + ).toBe("2026-08-09 10:00:00.123456"); + }); + + it("falls back to an ISO string only when handed a Date", () => { + expect( + cursorValueOf(core_users.createdAt, new Date("2026-08-09T10:00:00.123Z")), + ).toBe("2026-08-09T10:00:00.123Z"); + }); + + it("carries a bigint as a decimal string, which JSON can hold", () => { + expect(cursorValueOf(probes.big, 9007199254740993n)).toBe( + "9007199254740993", + ); }); }); diff --git a/packages/vitnode/src/api/lib/pagination-cursor.ts b/packages/vitnode/src/api/lib/pagination-cursor.ts index 0de70e45a..0067a41b5 100644 --- a/packages/vitnode/src/api/lib/pagination-cursor.ts +++ b/packages/vitnode/src/api/lib/pagination-cursor.ts @@ -5,17 +5,27 @@ import { HTTPException } from "hono/http-exception"; /** * The opaque cursor a paginated list hands out, and takes back. * - * A cursor has to describe a **position in an ordering**, and an ordering is - * `(orderColumn, id)` - so the cursor is that pair. An identifier on its own is - * only a position when the list is ordered by the identifier; for any other - * column it names a row whose place in the sequence nobody knows, which is how a - * list ordered by `updatedAt` used to skip rows whose ids happened to fall on - * the wrong side of it. + * Two properties, and both of them are load-bearing. + * + * **It is the ordered tuple.** A cursor has to describe a position in an + * ordering, and an ordering is `(orderColumn, id)` - so the cursor is that pair. + * An identifier on its own is only a position when the list is ordered by the + * identifier; for any other column it names a row whose place in the sequence + * nobody knows. + * + * **It is self-contained.** The value it carries *is* the boundary, and nothing + * re-reads the row it came from. That is the difference between a cursor and a + * pointer: a cursor is the position as it stood when the page was generated, and + * editing or deleting the row that happened to sit on the boundary must not move + * it. Re-reading would mean an edit to one row silently skips every row the + * ordering used to have between the old position and the new one. * * The wire form is `base64url(JSON)`: opaque, so no client starts depending on * the shape, and self-describing, so a cursor minted for one order column is - * refused by a request that has since changed to another rather than silently - * producing nonsense. + * refused by a request that has since changed to another. + * + * It is **not signed**, so every field is treated as hostile input and validated + * against the column it claims to describe - see {@link cursorValueForColumn}. */ /** What an order column's value can be, once it has been through JSON. */ @@ -30,18 +40,30 @@ export interface PaginationCursor { value: PaginationCursorValue; } -/** Postgres types whose ordering a cursor can describe. */ -const SORTABLE_DATA_TYPES = new Set([ - "bigint", - "boolean", - "date", - "number", - "string", -]); +/** + * How one column's values travel in a cursor. + * + * Named per kind rather than inferred, because "how do I serialise this" and + * "what am I willing to accept back" are the same question asked twice, and + * answering it in one place is what stops the second answer being looser than + * the first. + */ +type CursorKind = "bigint" | "boolean" | "date" | "number" | "string"; + +const KIND_BY_DATA_TYPE: Record = { + bigint: "bigint", + boolean: "boolean", + date: "date", + number: "number", + string: "string", +}; const badRequest = (message: string): HTTPException => new HTTPException(400, { message }); +/** The one message a tampered or stale cursor ever produces. */ +const INVALID_CURSOR = "Invalid pagination cursor."; + /** * Whether a column can be paged through at all. * @@ -50,41 +72,114 @@ const badRequest = (message: string): HTTPException => * against. Refused rather than approximated. */ export const isCursorSortableColumn = (column: PgColumn): boolean => - SORTABLE_DATA_TYPES.has(column.dataType); + column.dataType in KIND_BY_DATA_TYPE; + +const kindOf = (column: PgColumn): CursorKind => { + const kind = KIND_BY_DATA_TYPE[column.dataType]; + if (!kind) { + throw badRequest( + `The "${column.name}" column cannot be used as a pagination cursor.`, + ); + } + + return kind; +}; /** - * One column value, flattened to something JSON gives back unchanged. + * A Postgres timestamp, date or time as `::text` renders it. + * + * Anchored and permissive in the right places: `timestamp` gives + * `2026-08-09 10:00:00.123456`, `timestamptz` appends `+00`, and a plain `date` + * gives just the day. An ISO string with a `T` and a `Z` is accepted too, + * because that is what a `Date` produces on the one path that has no database + * text to offer. * - * A `Date` becomes an ISO string and a `bigint` becomes a decimal string, - * because neither survives `JSON.stringify` as itself. - * {@link cursorValueForColumn} is the exact inverse. + * The regex is not decoration - it is what keeps a tampered cursor from reaching + * Postgres as an invalid cast, which would be a 500 rather than a 400. + */ +const CANONICAL_TIMESTAMP = + /^\d{4}-\d{2}-\d{2}([ T]\d{2}:\d{2}:\d{2}(\.\d{1,6})?((\+|-)\d{2}(:?\d{2})?|Z)?)?$/; + +const DECIMAL_INTEGER = /^-?\d+$/; + +/** + * One column value, flattened into the cursor's canonical representation. + * + * Per kind, and deliberately not a generic coercion: + * + * | kind | carried as | + * | --------- | --------------------------------------------- | + * | number | a JSON number | + * | boolean | a JSON boolean | + * | string | a JSON string | + * | bigint | a decimal string, because JSON has no bigint | + * | date | the database's own `::text`, microseconds and all | + * | null | `null` | + * + * The date row is the one worth reading twice. A Postgres `timestamp` keeps + * microseconds and a JavaScript `Date` keeps milliseconds, so a value that has + * been through a `Date` is *strictly smaller* than the one still in the table - + * and comparing against it would exclude the entire millisecond it came from. + * Since `now()` stamps every row in one statement identically, that is not an + * edge case: it would end a bulk-imported collection's walk after page one. So + * the mint path reads `column::text` and this function keeps it exactly as + * Postgres wrote it. */ export const cursorValueOf = ( column: PgColumn, value: unknown, ): PaginationCursorValue => { if (value === null || value === undefined) return null; - if (value instanceof Date) return value.toISOString(); - if (typeof value === "bigint") return value.toString(); - const type = typeof value; - if (type === "boolean" || type === "number" || type === "string") { - return value as PaginationCursorValue; + switch (kindOf(column)) { + case "bigint": { + if (typeof value === "bigint") return value.toString(); + if (typeof value === "number") return String(value); + break; + } + case "boolean": { + if (typeof value === "boolean") return value; + break; + } + case "date": { + // Already `::text` from the database on the normal path. A `Date` only + // reaches here if a caller supplied one, and its ISO form is the best + // that value can offer. + if (value instanceof Date) return value.toISOString(); + if (typeof value === "string") return value; + break; + } + case "number": { + if (typeof value === "number") return value; + break; + } + default: { + if (typeof value === "string") return value; + break; + } } - throw badRequest( - `The "${column.name}" column cannot be used as a pagination cursor.`, + // The value came off a row of the very column it is being minted for, so + // anything else is a wiring bug rather than bad input - and quietly writing + // `"[object Object]"` into a cursor would hide it until somebody turned a + // page. + throw new Error( + `Cannot build a pagination cursor from a ${typeof value} value of "${column.name}".`, ); }; /** - * The cursor value, back in the shape the **column** compares against. + * The cursor value, validated against the column it claims to describe. + * + * Validation rather than coercion, because the cursor is opaque but not signed: + * a client can edit it. `Boolean("false")` is `true`, `Number("")` is `0`, and + * `BigInt("nonsense")` throws a `SyntaxError` that would surface as a 500 - so + * every kind checks the shape it expects and refuses anything else with a 400. * - * This is the step that keeps the predicate honest: Drizzle maps a parameter - * through the column it is compared with, and a `timestamp` column expects a - * `Date`. Handing it the ISO string the cursor carries would compare a timestamp - * against text, which Postgres refuses - so the string is turned back into a - * `Date` here rather than coerced somewhere further down. + * Returns the value in the form the SQL comparison needs: a real `boolean`, + * `number` or `bigint` for those kinds, and for a date the **canonical text**, + * which the predicate binds with an explicit cast so Postgres parses it at full + * precision. */ export const cursorValueForColumn = ( column: PgColumn, @@ -92,31 +187,49 @@ export const cursorValueForColumn = ( ): unknown => { if (value === null) return null; - switch (column.dataType) { - case "bigint": - return BigInt(String(value)); - case "boolean": - return Boolean(value); + switch (kindOf(column)) { + case "bigint": { + // A decimal string, and nothing else: `BigInt("1.5")` and `BigInt("")` + // are a `SyntaxError` and a `0` respectively, and neither is an answer. + if (typeof value !== "string" || !DECIMAL_INTEGER.test(value)) { + throw badRequest(INVALID_CURSOR); + } + + return BigInt(value); + } + case "boolean": { + if (typeof value !== "boolean") throw badRequest(INVALID_CURSOR); + + return value; + } case "date": { - const date = new Date(String(value)); - if (Number.isNaN(date.getTime())) { - throw badRequest("Invalid pagination cursor."); + if (typeof value !== "string" || !CANONICAL_TIMESTAMP.test(value)) { + throw badRequest(INVALID_CURSOR); } - return date; + // Kept as text. The predicate casts it back to the column's own type, so + // Postgres does the parsing - at the precision it stored. + return value; } case "number": { - const parsed = Number(value); - if (!Number.isFinite(parsed)) - throw badRequest("Invalid pagination cursor."); + if (typeof value !== "number" || !Number.isFinite(value)) { + throw badRequest(INVALID_CURSOR); + } + + return value; + } + default: { + if (typeof value !== "string") throw badRequest(INVALID_CURSOR); - return parsed; + return value; } - default: - return String(value); } }; +/** Whether the comparison binds this column's value as text plus a cast. */ +export const cursorValueIsCanonicalText = (column: PgColumn): boolean => + kindOf(column) === "date"; + export const encodePaginationCursor = (cursor: PaginationCursor): string => Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); @@ -142,13 +255,17 @@ const isCursorValue = (value: unknown): value is PaginationCursorValue => * 3. **a legacy numeric cursor on a non-primary-key ordering** - the exact case * that used to skip rows. A bare number is still accepted when the list is * ordered by its identifier, because there it really is the whole tuple. + * + * The *value* is checked separately, against the column - see + * {@link cursorValueForColumn} - because only the caller knows which column this + * request is ordered by. */ export const decodePaginationCursor = ( raw: string, { column, primaryKey }: { column: string; primaryKey: string }, ): PaginationCursor => { const trimmed = raw.trim(); - if (trimmed === "") throw badRequest("Invalid pagination cursor."); + if (trimmed === "") throw badRequest(INVALID_CURSOR); if (LEGACY_CURSOR.test(trimmed)) { if (column !== primaryKey) { @@ -167,11 +284,11 @@ export const decodePaginationCursor = ( Buffer.from(trimmed, "base64url").toString("utf8"), ) as unknown; } catch { - throw badRequest("Invalid pagination cursor."); + throw badRequest(INVALID_CURSOR); } if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - throw badRequest("Invalid pagination cursor."); + throw badRequest(INVALID_CURSOR); } const candidate = parsed as Record; @@ -183,7 +300,7 @@ export const decodePaginationCursor = ( id <= 0 || !isCursorValue(candidate.value) ) { - throw badRequest("Invalid pagination cursor."); + throw badRequest(INVALID_CURSOR); } if (candidate.column !== column) { diff --git a/packages/vitnode/src/api/lib/with-pagination.ts b/packages/vitnode/src/api/lib/with-pagination.ts index b1a1853d9..ef5974db2 100644 --- a/packages/vitnode/src/api/lib/with-pagination.ts +++ b/packages/vitnode/src/api/lib/with-pagination.ts @@ -29,6 +29,7 @@ import type { PaginationCursor } from "./pagination-cursor"; import { cursorValueForColumn, + cursorValueIsCanonicalText, cursorValueOf, decodePaginationCursor, encodePaginationCursor, @@ -118,6 +119,13 @@ function required(value: SQL | undefined): SQL { * column < :value OR (column = :value AND id < :id) -- descending * ``` * + * `:value` comes from the **cursor** and nowhere else. That is the invariant a + * cursor exists to provide: it is the position as it stood when the page was + * generated, so editing the row that happened to sit on the boundary must not + * move it. Reading the row's current value instead would mean one edit silently + * skips every row the ordering used to have between the old position and the + * new one - and deleting it would leave no position at all. + * * The `NULL` branches are the part that is easy to get wrong. Postgres sorts * `NULLS LAST` for `ASC` and `NULLS FIRST` for `DESC`, and `column > NULL` is * `NULL` rather than true - so a nullable order column needs the null block @@ -130,14 +138,12 @@ function buildCursorCondition({ direction, isPrimaryOrder, primary, - table, }: { column: PgColumn; cursor: PaginationCursor; direction: "asc" | "desc"; isPrimaryOrder: boolean; primary: PgColumn; - table: PgTable; }): SQL { const after = direction === "asc" ? gt : lt; @@ -145,7 +151,7 @@ function buildCursorCondition({ // is no second half to compare and no null block to worry about. if (isPrimaryOrder) return after(primary, cursor.id); - const boundary = boundaryValue({ column, cursor, primary, table }); + const boundary = boundaryValue(column, cursor); if (direction === "asc") { // NULLS LAST: a null cursor is inside the trailing block, and everything @@ -177,43 +183,29 @@ function buildCursorCondition({ } /** - * The value to compare against, read back from the row the cursor names. + * The cursor's own value, bound so Postgres compares it at full precision. * - * A cursor cannot carry a Postgres `timestamp` faithfully: the column keeps - * microseconds and a JavaScript `Date` keeps milliseconds, so a value that has - * been out to a client and back is *strictly smaller* than the one still in the - * table. Comparing against it would exclude the whole millisecond it came from - * - and since `now()` gives every row in one statement the same microsecond - * stamp, that is not an edge case: a bulk-imported collection would end its walk - * after the first page, silently. + * Two shapes, because two kinds of value survive a round trip differently: * - * So the comparison uses the stored value. The subquery is uncorrelated - one - * primary-key lookup, evaluated once - and the cursor's own copy is the fallback - * for the one case the row cannot supply: it has been deleted since. That copy - * is millisecond-accurate rather than exact, which is the honest best available - * when the row it described is gone. + * - a **date** travels as the database's own `::text` and is bound back with an + * explicit cast, so Postgres parses the microseconds it wrote. Binding a + * JavaScript `Date` here would silently truncate to milliseconds and exclude + * the whole millisecond the cursor came from. + * - **everything else** - a number, a string, a boolean, a bigint - is exact in + * JavaScript already, so it goes through the column's own encoder. + * + * `getSQLType()` is derived from the schema rather than from the request, which + * is what makes `sql.raw` safe here; the value itself is always a bound + * parameter. */ -function boundaryValue({ - column, - cursor, - primary, - table, -}: { - column: PgColumn; - cursor: PaginationCursor; - primary: PgColumn; - table: PgTable; -}): SQL { - // `sql.param(value, column)` rather than a bare interpolation: the value has - // to go through the column's own encoder, or the driver is handed an ISO - // string where it declared a timestamp and refuses the whole statement. - const fallback = sql.param( - cursorValueForColumn(column, cursor.value), - column, - ); - const id = sql.param(cursor.id, primary); +function boundaryValue(column: PgColumn, cursor: PaginationCursor): SQL { + const value = cursorValueForColumn(column, cursor.value); + + if (cursorValueIsCanonicalText(column)) { + return sql`${String(value)}::${sql.raw(column.getSQLType())}`; + } - return sql`coalesce((select ${column} from ${table} where ${primary} = ${id}), ${fallback})`; + return sql`${sql.param(value, column)}`; } function buildSearchWhere( @@ -342,7 +334,6 @@ export async function withPagination< direction, isPrimaryOrder, primary, - table, }) : undefined; const where = @@ -390,10 +381,19 @@ export async function withPagination< * The two cursors a page hands back. * * The order column's value normally comes straight off the row, because a list - * almost always selects the column it sorts by. When it does not - a projection - * that names a subset - the two boundary values are read in one extra query - * rather than the cursor quietly falling back to an identifier, which is exactly - * the bug this module exists to remove. + * almost always selects the column it sorts by. Two cases need one extra + * primary-key lookup of at most two rows: + * + * 1. **a projection that does not include the order column** - a public read + * naming a subset of fields. Without the value there is no tuple to mint, and + * falling back to an identifier is the bug this module exists to remove; + * 2. **a date column** - the row carries a JavaScript `Date`, which has already + * lost the microseconds Postgres stored. The canonical `::text` is fetched so + * the cursor carries the value the next comparison will be parsed from. + * + * The lookup reads the boundary rows *now*, while they are still the boundary - + * it is part of minting the cursor, not part of using one. Nothing re-reads them + * when the cursor comes back. */ async function cursorsFor({ c, @@ -419,16 +419,23 @@ async function cursorsFor({ const idOf = (row: Record): number => Number(row[primaryName]); - const projected = orderName in first && orderName in last; + const canonical = cursorValueIsCanonicalText(orderColumn); + const onTheRow = !canonical && orderName in first && orderName in last; + const values = new Map(); - if (projected) { + if (onTheRow) { values.set(idOf(first), first[orderName]); values.set(idOf(last), last[orderName]); } else { const ids = [...new Set([idOf(first), idOf(last)])]; const rows = await c .get("db") - .select({ id: primary, value: orderColumn }) + .select({ + id: primary, + // `::text` for a date, so no precision is lost between the value the + // cursor carries and the value the next page compares against. + value: canonical ? sql`${orderColumn}::text` : orderColumn, + }) .from(table) .where(inArray(primary, ids)); diff --git a/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts b/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts index 647323ca1..fe7c2ae66 100644 --- a/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts +++ b/packages/vitnode/src/api/modules/admin/debug/routes/content-status.route.ts @@ -32,7 +32,11 @@ const contentTypeSchema = z.object({ search: z .object({ canonicalHealthy: z.boolean(), + /** Documents `core_search_index` holds, every locale. */ + canonicalIndexedTotal: z.number(), contentTypeId: z.string(), + /** Published rows - or translations - the database holds, every locale. */ + expectedTotal: z.number(), /** Canonical **and** provider both agree. Unverified is not healthy. */ healthy: z.boolean(), locales: z.array(localeDriftSchema), @@ -40,6 +44,13 @@ const contentTypeSchema = z.object({ /** Why the provider could not be counted, when that is the answer. */ error: z.string().optional(), healthy: z.boolean().nullable(), + /** + * Every document the provider holds, in any locale. + * + * The guard against a document left behind in a locale the database no + * longer knows about, which per-locale counts can never ask for. + */ + indexedTotal: z.number().nullable(), name: z.string(), /** Whether the provider was actually asked. */ verified: z.boolean(), diff --git a/packages/vitnode/src/content/server/diagnostics.ts b/packages/vitnode/src/content/server/diagnostics.ts index 3e075c33e..fdc497836 100644 --- a/packages/vitnode/src/content/server/diagnostics.ts +++ b/packages/vitnode/src/content/server/diagnostics.ts @@ -80,6 +80,18 @@ export interface ContentSearchDriftProvider { error?: string; /** `null` when unverified - see {@link ContentSearchDriftLocale.providerHealthy}. */ healthy: boolean | null; + /** + * Every document the provider holds for this content type, in **any** locale. + * + * The guard against ghosts. Per-locale counts can only ask about locales the + * database still knows about, so a document left behind in a locale that has + * since been removed - or one whose canonical row was deleted while the + * provider's delete failed - is invisible to them. A total is not: it is + * larger than `expectedTotal`, and that is enough to say something is wrong. + * + * `null` when the provider offers no diagnostics. + */ + indexedTotal: null | number; name: string; /** * Whether the provider was actually asked. @@ -92,11 +104,16 @@ export interface ContentSearchDriftProvider { } export interface ContentSearchDrift { - /** `true` when `core_search_index` matches the database in every locale. */ + /** `true` when `core_search_index` matches the database, per locale and in total. */ canonicalHealthy: boolean; + /** Documents `core_search_index` holds for this content type, all locales. */ + canonicalIndexedTotal: number; contentTypeId: string; + /** Published rows - or published translations - the database holds, all locales. */ + expectedTotal: number; /** - * `true` only when the canonical table **and** the active provider both agree. + * `true` only when the canonical table **and** the active provider both agree, + * per locale and in total. * * An unverified provider is not healthy: absence of evidence is reported as * absence of evidence, and the operator decides what to do about it. @@ -110,17 +127,24 @@ export interface ContentSearchDrift { * Compares what the database says should be indexed against what is - in the * canonical table **and** in the active search provider. * - * The two are not the same question, and conflating them was the gap this + * The two are not the same question, and conflating them was the first gap this * closes. `SearchModel.index` writes `core_search_index` and then hands the * document to the provider; an Elasticsearch that refuses the second half leaves * a canonical table that is perfectly correct and a search box that is missing - * results. A diagnostic that only ever looked at the canonical table would call - * that healthy. + * results. + * + * The second gap is the opposite direction, and it needs a different instrument. + * Deletion runs canonical-first: `SearchModel.delete` removes the row and then + * asks the provider. If the provider's half fails, the document survives in a + * locale that no longer appears in either the database or the canonical table - + * so per-locale enumeration, which is built from those two, can never ask about + * it. Hence `indexedTotal`: one unfiltered count that no amount of missing + * enumeration can hide from. * * Costs: two aggregates for the canonical side, whatever the collection's size, - * plus one provider count per locale. The bundled Postgres provider is skipped - * entirely - its store *is* the canonical table, so asking twice would buy - * nothing. + * plus one provider total and one provider count per locale. The bundled + * Postgres provider is skipped entirely - its store *is* the canonical table, so + * asking twice would buy nothing. */ export const contentSearchDrift = async ( c: Context, @@ -152,7 +176,17 @@ export const contentSearchDrift = async ( ...new Set([...expectedByLocale.keys(), ...indexedByLocale.keys()]), ].sort(); + const sum = (values: Iterable): number => { + let total = 0; + for (const value of values) total += value; + + return total; + }; + const expectedTotal = sum(expectedByLocale.values()); + const canonicalIndexedTotal = sum(indexedByLocale.values()); + const provider = await providerCounts(c, { + canonicalTotal: canonicalIndexedTotal, contentTypeId, locales: keys, }); @@ -160,7 +194,6 @@ export const contentSearchDrift = async ( const locales: ContentSearchDriftLocale[] = keys.map(locale => { const expected = expectedByLocale.get(locale) ?? 0; const canonicalIndexed = indexedByLocale.get(locale) ?? 0; - const canonicalHealthy = expected === canonicalIndexed; // The canonical count stands in for the provider only when they are one // storage. Everywhere else an absent count means unverified, never healthy. const providerIndexed = provider.canonical @@ -168,7 +201,7 @@ export const contentSearchDrift = async ( : (provider.byLocale.get(locale) ?? null); return { - canonicalHealthy, + canonicalHealthy: expected === canonicalIndexed, canonicalIndexed, expected, locale, @@ -178,20 +211,30 @@ export const contentSearchDrift = async ( }; }); - const canonicalHealthy = locales.every(entry => entry.canonicalHealthy); + // The total is part of *both* verdicts, not decoration on the provider one: + // a canonical row in a locale nothing expects is a ghost too, and the grouped + // query above already sees every locale the table holds. + const canonicalHealthy = + canonicalIndexedTotal === expectedTotal && + locales.every(entry => entry.canonicalHealthy); + const providerHealthy = provider.verified ? provider.error === undefined && + provider.total === expectedTotal && locales.every(entry => entry.providerHealthy === true) : null; return { canonicalHealthy, + canonicalIndexedTotal, contentTypeId, + expectedTotal, healthy: canonicalHealthy && providerHealthy === true, locales, provider: { ...(provider.error === undefined ? {} : { error: provider.error }), healthy: providerHealthy, + indexedTotal: provider.total, name: provider.name, verified: provider.verified, }, @@ -199,9 +242,15 @@ export const contentSearchDrift = async ( }; /** - * Asks the active provider how many documents it holds, per locale. + * Asks the active provider how many documents it holds - in total, and per + * locale. * - * Three outcomes, and the middle one is the one Stage 7 exists for: + * The total goes first and is asked **unconditionally**, including for a content + * type with no rows and no canonical documents at all. That is the case the + * per-locale loop cannot cover: with nothing to enumerate, `[].every(...)` is + * `true`, and a ghost document would sail through as healthy. + * + * Three outcomes: * * - **canonical storage** - the bundled Postgres provider. Verified, and the * canonical counts are its counts; no query is issued. @@ -210,27 +259,50 @@ export const contentSearchDrift = async ( * - **it threw** - Elasticsearch is down. Verified and unhealthy, with the * reason attached, and the status route still answers: a diagnostic that * crashes when the thing it diagnoses is broken is a diagnostic nobody can - * use. + * use. A failure in either call is handled the same way. */ const providerCounts = async ( c: Context, - { contentTypeId, locales }: { contentTypeId: string; locales: string[] }, + { + canonicalTotal, + contentTypeId, + locales, + }: { canonicalTotal: number; contentTypeId: string; locales: string[] }, ): Promise<{ byLocale: Map; canonical: boolean; error?: string; name: string; + total: null | number; verified: boolean; }> => { const search = c.get("search"); const name = search.name(); if (search.isCanonicalStorage()) { - return { byLocale: new Map(), canonical: true, name, verified: true }; + return { + byLocale: new Map(), + canonical: true, + name, + total: canonicalTotal, + verified: true, + }; } + const unverified = { + byLocale: new Map(), + canonical: false, + name, + total: null, + verified: false, + }; + const byLocale = new Map(); try { + // Every locale, including ones nothing here knows the name of. + const total = await search.countDocuments({ itemType: contentTypeId }); + if (total === null) return unverified; + for (const locale of locales) { const counted = await search.countDocuments({ itemType: contentTypeId, @@ -238,11 +310,11 @@ const providerCounts = async ( // under, and it is a real value to filter on rather than "any". languageCode: locale, }); - if (counted === null) { - return { byLocale: new Map(), canonical: false, name, verified: false }; - } + if (counted === null) return unverified; byLocale.set(locale, counted); } + + return { byLocale, canonical: false, name, total, verified: true }; } catch (error) { const message = error instanceof Error ? error.message : "unknown error"; await logDiagnosticFailure( @@ -259,11 +331,10 @@ const providerCounts = async ( canonical: false, error: message, name, + total: null, verified: true, }; } - - return { byLocale, canonical: false, name, verified: true }; }; /** The prefix a provider diagnostic failure is logged behind. */ diff --git a/packages/vitnode/src/content/server/service.test.ts b/packages/vitnode/src/content/server/service.test.ts index 9c285b4a9..1ff2fa5a1 100644 --- a/packages/vitnode/src/content/server/service.test.ts +++ b/packages/vitnode/src/content/server/service.test.ts @@ -347,9 +347,15 @@ describe("content service", () => { await articles.service(c).findMany(); - // `author` and `category` - one join each, and no extra round trips. + // `author` and `category` - one join each, and no per-row lookup. expect(opsOf(calls, "leftJoin")).toHaveLength(2); - expect(opsOf(calls, "select")).toHaveLength(2); // count + page + // Three, and all three are constant: the count, the page, and one + // primary-key read of the two boundary rows to mint the cursors. That + // last one exists because the default ordering is a timestamp, and a + // timestamp has to be carried at the database's own precision - see + // `pagination-cursor.ts`. What matters is that none of them grows with + // the page. + expect(opsOf(calls, "select")).toHaveLength(3); }); it("splits the joined labels out of the row", async () => { diff --git a/plugins/example/src/database/harness.ts b/plugins/example/src/database/harness.ts index 0f45e3c72..89cfa3e73 100644 --- a/plugins/example/src/database/harness.ts +++ b/plugins/example/src/database/harness.ts @@ -149,12 +149,18 @@ export interface ContentTestHarness { * `"canonical"` is the bundled Postgres provider - its store *is* * `core_search_index`, so it is verified without a second query. * `"unsupported"` is a provider with no `count`, which has to be reported as - * unverified rather than healthy. A `Map` is a mirroring provider that can - * be counted, keyed by locale (`""` for language-agnostic content) - which - * is how a provider that is missing documents the canonical table has is - * simulated at all. + * unverified rather than healthy. + * + * The object form is a mirroring provider that can be counted: `byLocale` + * answers a filtered count and `total` answers an unfiltered one. They are + * separate on purpose - a ghost document lives in a locale nothing + * enumerates, so the only way to simulate one is a total that exceeds the + * locales anybody thinks to ask about. */ - providerCounts: "canonical" | "unsupported" | Map; + providerCounts: + | "canonical" + | "unsupported" + | { byLocale: Map; total: number }; providerName: string; /** * Web origins the revalidation bridge should post to. @@ -318,8 +324,13 @@ export const createContentTestHarness = return await Promise.resolve(0); } + // No language means every language - which is what makes a + // ghost in an unenumerated locale visible at all. return await Promise.resolve( - behaviour.providerCounts.get(languageCode ?? "") ?? 0, + languageCode === undefined + ? behaviour.providerCounts.total + : (behaviour.providerCounts.byLocale.get(languageCode) ?? + 0), ); }, isCanonicalStorage: () => diff --git a/plugins/example/src/database/pagination-postgres.test.ts b/plugins/example/src/database/pagination-postgres.test.ts index 360a67fd2..f1b5fda74 100644 --- a/plugins/example/src/database/pagination-postgres.test.ts +++ b/plugins/example/src/database/pagination-postgres.test.ts @@ -225,6 +225,221 @@ describe.skipIf(!DATABASE_TEST_URL)( } }); + // ------------------------------------------------------------------------- + // The cursor is a historical position + // ------------------------------------------------------------------------- + + /** + * A cursor names where the page ended, not which row ended it. + * + * The distinction only shows itself when the boundary row moves. If the next + * page's comparison were built from that row's *current* value, one edit + * would drag the boundary with it and silently skip every row the ordering + * used to have in between - a page of results nobody ever sees, with no + * error and no short page to notice. + * + * The fixture makes the continuation deterministic: five rows ascending by + * `updatedAt`, identifiers ascending with them. + */ + describe("when the boundary row changes after the cursor was issued", () => { + const LADDER: Seed[] = [ + { + code: "l1", + title: "One", + updatedAt: new Date("2026-08-09T10:00:00Z"), + }, + { + code: "l2", + title: "Two", + updatedAt: new Date("2026-08-09T10:01:00Z"), + }, + { + code: "l3", + title: "Three", + updatedAt: new Date("2026-08-09T11:00:00Z"), + }, + { + code: "l4", + title: "Four", + updatedAt: new Date("2026-08-09T12:00:00Z"), + }, + { + code: "l5", + title: "Five", + updatedAt: new Date("2026-08-09T13:00:00Z"), + }, + ]; + + /** Page 1 of one row, plus the cursor it handed back. */ + const firstPage = async (order: "asc" | "desc" = "asc") => { + const page = await service().findMany({ + orderBy: { column: "updatedAt" as never, order }, + query: { first: "1" }, + }); + + return { + boundary: page.edges[0].id, + cursor: page.pageInfo.endCursor ?? undefined, + }; + }; + + const walkFrom = async ( + cursor: string | undefined, + order: "asc" | "desc" = "asc", + ) => { + const seen: number[] = []; + let next = cursor; + for (let page = 0; page < 20; page += 1) { + const result = await service().findMany({ + orderBy: { column: "updatedAt" as never, order }, + query: { cursor: next, first: "2" }, + }); + seen.push(...result.edges.map(row => row.id)); + if (!result.pageInfo.hasNextPage) break; + next = result.pageInfo.endCursor ?? undefined; + } + + return seen; + }; + + const moveTo = async (id: number, when: string) => { + await h.sql` + UPDATE "example_articles" + SET "updatedAt" = ${when}::timestamp + WHERE "id" = ${id} + `; + }; + + it("still reaches every row that was after the cursor when it was issued", async () => { + // The regression. Moving the boundary row to the *end* of the ordering + // would drag a re-read boundary with it, and 10:01, 11:00, 12:00 and + // 13:00 would be skipped for good. + const ids = await seed(LADDER); + const { boundary, cursor } = await firstPage(); + expect(boundary).toBe(ids[0]); + + await moveTo(boundary, "2026-08-09T14:00:00"); + + const seen = await walkFrom(cursor); + + for (const id of ids.slice(1)) expect(seen).toContain(id); + expect(new Set(seen).size).toBe(seen.length); + }); + + it("shows the moved row again, because it moved into unvisited ground", async () => { + // The honest consequence, stated rather than hidden: a keyset walk is + // not a snapshot, so a row that moves from behind the cursor to ahead + // of it is seen a second time. What matters is that nothing else moved. + const ids = await seed(LADDER); + const { boundary, cursor } = await firstPage(); + + await moveTo(boundary, "2026-08-09T14:00:00"); + + const seen = await walkFrom(cursor); + + expect(seen).toContain(boundary); + expect(seen.sort((a, b) => a - b)).toEqual( + [...ids].sort((a, b) => a - b), + ); + }); + + it("does not show it again when it moves further behind the cursor", async () => { + const ids = await seed(LADDER); + const { boundary, cursor } = await firstPage(); + + await moveTo(boundary, "2026-08-09T09:00:00"); + + const seen = await walkFrom(cursor); + + expect(seen).not.toContain(boundary); + expect(seen.sort((a, b) => a - b)).toEqual(ids.slice(1)); + }); + + it("keeps working when the boundary row is deleted outright", async () => { + // Nothing to re-read, and nothing that needs re-reading: the position + // is in the cursor. + const ids = await seed(LADDER); + const { boundary, cursor } = await firstPage(); + + await h.sql`DELETE FROM "example_articles" WHERE "id" = ${boundary}`; + + const seen = await walkFrom(cursor); + + expect(seen.sort((a, b) => a - b)).toEqual(ids.slice(1)); + expect(new Set(seen).size).toBe(seen.length); + }); + + it("survives the whole first page being deleted", async () => { + const ids = await seed(LADDER); + const page = await service().findMany({ + orderBy: { column: "updatedAt" as never, order: "asc" }, + query: { first: "3" }, + }); + const read = page.edges.map(row => row.id); + + for (const id of read) { + await h.sql`DELETE FROM "example_articles" WHERE "id" = ${id}`; + } + + const seen = await walkFrom(page.pageInfo.endCursor ?? undefined); + + expect(seen.sort((a, b) => a - b)).toEqual( + ids.filter(id => !read.includes(id)).sort((a, b) => a - b), + ); + }); + + it("holds the same way descending", async () => { + const ids = await seed(LADDER); + const { boundary, cursor } = await firstPage("desc"); + + await moveTo(boundary, "2026-08-09T00:01:00"); + + const seen = await walkFrom(cursor, "desc"); + + for (const id of ids.slice(0, 4)) expect(seen).toContain(id); + expect(new Set(seen).size).toBe(seen.length); + }); + + it("carries the database's own timestamp text, microseconds included", async () => { + // The reason the cursor can be self-contained at all. A JavaScript + // `Date` holds milliseconds; `now()` writes microseconds. A cursor that + // had been through a `Date` would be strictly smaller than the stored + // value and would exclude the whole millisecond it came from - and here + // every row shares one `now()`, so the walk would stop after page one. + await h.sql` + INSERT INTO "example_articles" + ("title", "slug", "code", "category", "status", "updatedAt") + SELECT 'Micro ' || i, 'micro-' || i, 'micro-' || i, ${categoryId}, + 'draft', now() + FROM generate_series(1, 6::int) AS i + `; + + const page = await service().findMany({ + orderBy: { column: "updatedAt" as never, order: "asc" }, + query: { first: "1" }, + }); + const decoded = JSON.parse( + Buffer.from(page.pageInfo.endCursor ?? "", "base64url").toString( + "utf8", + ), + ) as { value: string }; + + // Byte-identical to what the column holds, rather than "has enough + // digits": trailing zeros are dropped by `::text`, so counting them + // would be a coin flip, and equality is the property that matters. + const [stored] = await h.sql<{ text: string }[]>` + SELECT "updatedAt"::text AS text FROM "example_articles" + WHERE "id" = ${page.edges[0].id} + `; + expect(decoded.value).toBe(stored.text); + + // And the walk completes, which it cannot if the boundary was + // truncated: every row here shares one `now()`. + const seen = await walkFrom(page.pageInfo.endCursor ?? undefined); + expect(seen).toHaveLength(5); + }); + }); + // ------------------------------------------------------------------------- // Ties // ------------------------------------------------------------------------- @@ -539,6 +754,81 @@ describe.skipIf(!DATABASE_TEST_URL)( throw new Error(`Expected ${JSON.stringify(query)} to be refused.`); }; + /** + * The cursor is opaque but not signed, so every field is hostile input. + * + * These go through the real service, which is where the column is known - + * a value that looks fine in isolation is only wrong relative to the + * column it claims to describe. + */ + const tampered = (value: unknown, column = "updatedAt") => + Buffer.from(JSON.stringify({ column, id: 1, value })).toString( + "base64url", + ); + + it.each([ + ["nonsense", "not-a-date", "updatedAt"], + ["a number", 1_700_000_000, "updatedAt"], + ["a boolean", true, "updatedAt"], + [ + "an injection attempt", + "2026-08-09'; DROP TABLE example_articles; --", + "updatedAt", + ], + ["a number where a string belongs", 12, "title"], + ["a boolean where a string belongs", false, "title"], + ])( + "answers 400 for a cursor holding %s, without reaching Postgres", + async (_why, value, column) => { + try { + await service().findMany({ + orderBy: { column: column as never, order: "asc" }, + query: { cursor: tampered(value, column), first: "5" }, + }); + } catch (error) { + // An `HTTPException`, never a `SyntaxError`, a `RangeError` or a + // Postgres cast failure - each of which would surface as a 500. + expect(error).toBeInstanceOf(HTTPException); + expect(statusOf(error)).toBe(400); + + return; + } + + throw new Error(`Expected ${String(value)} to be refused.`); + }, + ); + + it("leaves the table alone when a cursor tries to inject SQL", async () => { + await seed(NON_MONOTONIC); + + await expect( + service().findMany({ + orderBy: { column: "updatedAt" as never, order: "asc" }, + query: { + cursor: tampered("2026-08-09'; DROP TABLE example_articles; --"), + first: "5", + }, + }), + ).rejects.toBeInstanceOf(HTTPException); + + await expect(service().findMany()).resolves.toMatchObject({ + pageInfo: { totalCount: 3 }, + }); + }); + + it("refuses a cursor whose identifier is not a positive integer", async () => { + const zeroId = Buffer.from( + JSON.stringify({ column: "updatedAt", id: 0, value: null }), + ).toString("base64url"); + + await expect( + service().findMany({ + orderBy: { column: "updatedAt" as never, order: "asc" }, + query: { cursor: zeroId, first: "5" }, + }), + ).rejects.toBeInstanceOf(HTTPException); + }); + it.each([ ["first=0", { first: "0" }], ["last=0", { last: "0" }], diff --git a/plugins/example/src/database/resilience-postgres.test.ts b/plugins/example/src/database/resilience-postgres.test.ts index ea520f90b..1c0a1a9ac 100644 --- a/plugins/example/src/database/resilience-postgres.test.ts +++ b/plugins/example/src/database/resilience-postgres.test.ts @@ -982,9 +982,12 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { expect(drift).toMatchObject({ canonicalHealthy: true, + canonicalIndexedTotal: 2, contentTypeId: articleContentType.id, + expectedTotal: 2, healthy: true, }); + expect(drift.provider.indexedTotal).toBe(2); expect(drift.locales).toEqual([ { canonicalHealthy: true, @@ -1010,6 +1013,8 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { expect(drift.provider).toEqual({ healthy: true, + // Reused from the canonical count rather than queried again. + indexedTotal: 1, name: "postgres", verified: true, }); @@ -1077,7 +1082,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { await indexPublished(); h.behaviour.providerName = "elasticsearch"; - h.behaviour.providerCounts = new Map([["", 1]]); + h.behaviour.providerCounts = { byLocale: new Map([["", 1]]), total: 1 }; const drift = await contentSearchDrift(h.context, { model: articleContent, @@ -1101,6 +1106,209 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { expect(drift.healthy).toBe(false); }); + /** + * The other direction, and the one per-locale counts cannot see. + * + * Deletion runs canonical-first: `SearchModel.delete` removes the row and + * then asks the provider. If the provider's half fails, the document + * survives in a locale that no longer appears in the database *or* the + * canonical table - so the locale list, which is built from those two, never + * thinks to ask about it. Only an unfiltered total can find it. + */ + describe("a document that exists only in the provider", () => { + it("is caught on a content type with nothing in it at all", async () => { + // The empty case matters on its own: a localized content type with no + // published translations enumerates *no* locales, so `[].every(...)` is + // `true` and a ghost would sail straight through on the per-locale + // checks alone. + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = { byLocale: new Map(), total: 1 }; + + const drift = await contentSearchDrift(h.context, { + model: localizedArticleContent, + }); + + expect(drift.locales).toEqual([]); + expect(drift.expectedTotal).toBe(0); + expect(drift.canonicalIndexedTotal).toBe(0); + expect(drift.canonicalHealthy).toBe(true); + expect(drift.provider).toMatchObject({ + healthy: false, + indexedTotal: 1, + verified: true, + }); + expect(drift.healthy).toBe(false); + }); + + it("is caught on a non-localized content type with no rows either", async () => { + // Here one locale *is* enumerated - the empty one - and it agrees on + // both sides. The total is still the thing that catches the ghost. + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = { byLocale: new Map(), total: 1 }; + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.locales).toEqual([ + { + canonicalHealthy: true, + canonicalIndexed: 0, + expected: 0, + locale: "", + providerHealthy: true, + providerIndexed: 0, + }, + ]); + expect(drift.provider).toMatchObject({ + healthy: false, + indexedTotal: 1, + }); + expect(drift.healthy).toBe(false); + }); + + it("is caught when every locale it does enumerate agrees", async () => { + // The proof that the total is doing the work: `""` matches on both + // sides, so per-locale parity is perfect and the total is not. + await published(); + await indexPublished(); + + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = { + byLocale: new Map([["", 1]]), + total: 2, + }; + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.locales).toEqual([ + { + canonicalHealthy: true, + canonicalIndexed: 1, + expected: 1, + locale: "", + providerHealthy: true, + providerIndexed: 1, + }, + ]); + expect(drift.canonicalHealthy).toBe(true); + expect(drift.provider).toMatchObject({ + healthy: false, + indexedTotal: 2, + }); + expect(drift.healthy).toBe(false); + }); + + it("is caught in a locale the content type no longer has", async () => { + // EN is published and agrees everywhere. PL exists only in the + // provider - no translation, no canonical row, no expectation - so it + // is never enumerated, and the total is the only thing that sees it. + const { row } = await localizedService(h.context).create( + { + shared: {}, + translation: { body: "English body", title: "Ghost Subject" }, + }, + { actor: ACTOR }, + ); + const base = localizedArticleContent.editorialService; + if (!base) throw new Error("no editorial service"); + await base(h.context, { pluginId: CONFIG_PLUGIN.pluginId }).publish( + row.id, + { actor: ACTOR }, + ); + await translationEditorial(h.context).publish(row.id, "en", { + actor: ACTOR, + }); + await h.sql` + INSERT INTO "core_search_index" + ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${localizedArticleContent.definition.id}, + ${row.id}, 'en', 'Ghost Subject', 'Ghost Subject', now() + ) + `; + + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = { + // Only `en` is ever asked for, and it agrees. + byLocale: new Map([["en", 1]]), + total: 2, + }; + + const drift = await contentSearchDrift(h.context, { + model: localizedArticleContent, + }); + + expect(drift.locales.map(entry => entry.locale)).toEqual(["en"]); + expect(drift.locales[0].providerHealthy).toBe(true); + expect(drift.canonicalHealthy).toBe(true); + expect(drift.expectedTotal).toBe(1); + expect(drift.provider.indexedTotal).toBe(2); + expect(drift.provider.healthy).toBe(false); + expect(drift.healthy).toBe(false); + }); + + it("makes the whole engine report unhealthy", async () => { + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = { byLocale: new Map(), total: 1 }; + + const report = await contentEngineDiagnostics(h.context); + expect(report.contentTypes).not.toHaveLength(0); + + expect(report.searchHealthy).toBe(false); + expect(report.healthy).toBe(false); + }); + + it("still reports healthy when the total agrees as well", async () => { + // The control: same provider, same enumeration, honest total. + await published(); + await indexPublished(); + + h.behaviour.providerName = "elasticsearch"; + h.behaviour.providerCounts = { + byLocale: new Map([["", 1]]), + total: 1, + }; + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.provider).toMatchObject({ + healthy: true, + indexedTotal: 1, + verified: true, + }); + expect(drift.healthy).toBe(true); + }); + }); + + it("reports a canonical row in a locale nothing expects", async () => { + // The canonical side has the same failure mode, and the grouped query + // already sees every locale the table holds - so the total closes it too. + await published(); + await indexPublished(); + await h.sql` + INSERT INTO "core_search_index" + ("pluginId", "itemType", "itemId", "languageCode", "title", "content", "createdAt") + VALUES ( + ${CONFIG_PLUGIN.pluginId}, ${articleContentType.id}, 424242, + 'de', 'Ghost', 'Ghost', now() + ) + `; + + const drift = await contentSearchDrift(h.context, { + model: articleContent, + }); + + expect(drift.expectedTotal).toBe(1); + expect(drift.canonicalIndexedTotal).toBe(2); + expect(drift.canonicalHealthy).toBe(false); + expect(drift.healthy).toBe(false); + }); + it("reports a provider that cannot be counted as unverified, not healthy", async () => { await published(); await indexPublished(); @@ -1117,6 +1325,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { expect(drift.locales[0].providerIndexed).toBeNull(); expect(drift.provider).toEqual({ healthy: null, + indexedTotal: null, name: "custom-search", verified: false, }); @@ -1129,7 +1338,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { await indexPublished(); h.behaviour.providerName = "elasticsearch"; - h.behaviour.providerCounts = new Map([["", 1]]); + h.behaviour.providerCounts = { byLocale: new Map([["", 1]]), total: 1 }; h.behaviour.providerCountError = new Error("connect ECONNREFUSED"); const drift = await contentSearchDrift(h.context, { @@ -1153,7 +1362,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { it("keeps the whole status route answering when the provider is down", async () => { await published(); await indexPublished(); - h.behaviour.providerCounts = new Map([["", 1]]); + h.behaviour.providerCounts = { byLocale: new Map([["", 1]]), total: 1 }; h.behaviour.providerCountError = new Error("elasticsearch unavailable"); const report = await contentEngineDiagnostics(h.context); @@ -1263,10 +1472,13 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine failure resilience", () => { } h.behaviour.providerName = "elasticsearch"; - h.behaviour.providerCounts = new Map([ - ["en", 1], - ["pl", 0], - ]); + h.behaviour.providerCounts = { + byLocale: new Map([ + ["en", 1], + ["pl", 0], + ]), + total: 1, + }; const drift = await contentSearchDrift(h.context, { model: localizedArticleContent, From b4d75aee7a9efd04c36a986c5619c2f4591ecf3e Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sun, 9 Aug 2026 16:04:48 +0200 Subject: [PATCH 13/14] fix(pagination): mint the cursor from the page query, and reject impossible timestamps The cursor value was fetched by a second SELECT of the boundary rows, after the page had already come back. That is a time-of-check / time-of-use gap: a row chosen at (10:00, 42) could be handed back as a cursor saying (14:00, 42), and the next page would skip everything in between. A DELETE in the same window was worse - the lookup found nothing, the value became null, and for a nullable ordering null is a real position inside the null block. The page query now projects the value itself, as `__cursorValue`, and strips it before any row leaves the module. One statement, one snapshot, no window - and one fewer round trip per page. Temporal cursors are also range-checked rather than only shape-checked. `2026-02-30`, `2025-02-29` and `2026-08-09 23:60:00` all match `YYYY-MM-DD HH:MM:SS` and none of them is a moment, so Postgres answered the cast with `invalid input syntax` - a 500 produced by a query string. The grammar is now per SQL type (so `date()` and `time()`, which Drizzle reports as strings, are covered too) and the components are validated, leap years included. The original text is what gets bound, so microseconds still survive digit for digit. Co-Authored-By: Claude Opus 5 (1M context) --- .../performance-and-scaling.mdx | 78 +++++- .../src/api/lib/pagination-cursor.test.ts | 121 ++++++++- .../vitnode/src/api/lib/pagination-cursor.ts | 245 +++++++++++++++--- .../vitnode/src/api/lib/with-pagination.ts | 162 +++++++----- .../admin/advanced/cron/routes/get.route.ts | 5 +- .../admin/advanced/queue/routes/get.route.ts | 6 +- .../modules/admin/debug/routes/logs.route.ts | 3 +- .../modules/admin/files/routes/list.route.ts | 3 +- .../modules/admin/roles/routes/list.route.ts | 3 +- .../admin/staff/routes/admins.route.ts | 3 +- .../admin/staff/routes/moderators.route.ts | 3 +- .../modules/admin/users/routes/list.route.ts | 3 +- .../modules/admin/users/routes/users.route.ts | 3 +- .../modules/users/files/routes/list.route.ts | 3 +- .../server/localized-public-service.ts | 22 +- .../src/content/server/openapi-parity.test.ts | 14 + .../src/content/server/public-service.ts | 7 +- .../src/content/server/service.test.ts | 40 +-- .../vitnode/src/content/server/service.ts | 16 +- .../modules/categories/routes/get.route.ts | 13 +- .../src/api/modules/posts/routes/get.route.ts | 3 +- .../src/database/pagination-postgres.test.ts | 224 +++++++++++++++- .../src/database/performance-postgres.test.ts | 11 +- 23 files changed, 844 insertions(+), 147 deletions(-) diff --git a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx index 61494fd1f..a2e5d85b9 100644 --- a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx +++ b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx @@ -97,6 +97,44 @@ one - a page of results nobody ever sees, with no error and no short page to notice it by. +### The value is captured by the query that returned the row + +**The cursor value comes from the same `SQL` statement as the row it describes. +There is no second lookup of the boundary row.** The page query projects it +alongside the ordinary columns: + +```sql +SELECT title, slug, updatedAt::text AS "__cursorValue" +FROM articles +ORDER BY updatedAt ASC, id ASC +LIMIT 26 +``` + +The reason is the same one that makes a cursor self-contained, applied half a +step earlier. A cursor has to name **the exact ordered tuple the row occupied at +issuance**, and a second `SELECT` after the page has come back is a +time-of-check / time-of-use gap another writer can walk through: + +```text +page query returns id=42, updatedAt=10:00 + ← another writer sets id=42 to 14:00 +boundary lookup id=42, updatedAt=14:00 +cursor issued (14:00, 42) ← the row was never there +``` + +The next page then starts after 14:00 and 10:01, 11:00, 12:00 and 13:00 are gone +for good. A `DELETE` in the same window was worse: the lookup found nothing, the +value became `null`, and `null` is not "no position" - for a nullable ordering it +is a *real* one inside the null block, so the walk jumped there and abandoned the +rest of the collection. + +One statement closes the window rather than locking it. `__cursorValue` is +internal: it is stripped before a row reaches a handler, so it appears in no +response, no OpenAPI schema, no search document and no revision snapshot. A +projection that names only `title` and `slug` still pages by `updatedAt` +correctly, because pagination selects what it needs without widening the +allowlist. + ### Timestamps travel as the database wrote them A Postgres `timestamp` keeps microseconds; a JavaScript `Date` keeps @@ -106,7 +144,7 @@ whole millisecond it came from - which, since `now()` stamps every row in one statement identically, would end a bulk-imported collection's walk after page one. -So a date cursor carries the column's own `::text` and the predicate binds it +So a temporal cursor carries the column's own `::text` and the predicate binds it back with an explicit cast. The value in the cursor is byte-identical to the value the next comparison is parsed from. @@ -121,20 +159,50 @@ column it claims to describe: | bigint | a decimal integer string | | boolean | a JSON boolean | | string | a JSON string | -| date | a canonical Postgres timestamp, `timestamptz`, date or ISO string | +| `date` | `2026-08-09` - a day, and nothing more | +| `time` | `10:00:00`, `10:00:00.123456`, plus an offset only if the column has a zone | +| `timestamp`, `timestamptz` | a day, optionally a time, optionally an offset - `2026-08-09 10:00:00.123456+00` | | any | `null`, which is a real position | Coercion is the failure mode being avoided, not an inelegance: `Boolean("false")` is `true`, `Number("")` is `0`, and `BigInt("nonsense")` throws a `SyntaxError` -that would leave the route answering `500`. Every mismatch is a `400`, and no -malformed value ever reaches Postgres as a failed cast. +that would leave the route answering `500`. Every mismatch is a `400`. + +**A malformed or impossible timestamp cursor is rejected as `400` before it +reaches PostgreSQL.** Shape is not enough for a temporal value, because a shape +check cannot tell a day from a date that looks like one: + +```text +2026-13-01 2026-00-01 2026-02-30 2025-02-29 +2026-01-32 2026-08-09 24:00:00 2026-08-09 23:60:00 +2026-08-09 23:59:61 2026-08-09 10:00:00+25:00 +``` + +Every one of those matches `YYYY-MM-DD HH:MM:SS` and none of them is a moment, +so Postgres answers the cast with `invalid input syntax` - a `500` produced by a +query string. So the components are range-checked as well: month 1-12, the day +count for that month *in that year* (2024-02-29 is a day, 2025-02-29 is not), +hours 0-23, minutes and seconds 0-59, and an offset within ±15:59:59. + +The grammar is derived from the column's SQL type rather than from Drizzle's +JavaScript one, because the two disagree exactly where it matters: `date()` and +`time()` hand back plain strings, so a value bound straight through would reach +Postgres as `'nonsense'::date`. Validation is deliberately **stricter** than +Postgres in one place - `24:00:00` is a valid input to Postgres and something its +own `::text` never writes, so a cursor carrying one did not come from a row. + +The original string is what gets bound; nothing is reformatted, so microseconds +survive validation digit for digit. Two forms this repository does not mint are +not accepted either: an era suffix (`… BC`) and a year past 294276. **What is guaranteed:** - **any** orderable column pages exactly - a title, a nullable `publishedAt`, a custom field, whatever its relationship to the identifier; - the boundary is **stable**: updating or deleting the row the cursor named does - not move it, and every row that was after that position is still reachable; + not move it, and every row that was after that position is still reachable - + including when the change lands *between* the page query and the cursor being + minted, because there is nothing in between; - rows sharing a sort value are returned exactly once, because the identifier is the tiebreaker in both the ordering and the predicate; - a nullable order column walks through its null block correctly - Postgres sorts diff --git a/packages/vitnode/src/api/lib/pagination-cursor.test.ts b/packages/vitnode/src/api/lib/pagination-cursor.test.ts index 9f9d491cd..8ec3404df 100644 --- a/packages/vitnode/src/api/lib/pagination-cursor.test.ts +++ b/packages/vitnode/src/api/lib/pagination-cursor.test.ts @@ -1,4 +1,4 @@ -import { bigint, pgTable, timestamp } from "drizzle-orm/pg-core"; +import { bigint, date, pgTable, time, timestamp } from "drizzle-orm/pg-core"; // @vitest-environment node import { HTTPException } from "hono/http-exception"; import { describe, expect, it } from "vitest"; @@ -14,9 +14,12 @@ import { isCursorSortableColumn, } from "./pagination-cursor"; -/** A bigint order column, which core has only on a table without one. */ +/** The column shapes core itself has no example of, but a plugin may. */ const probes = pgTable("cursor_probes", { big: bigint({ mode: "bigint" }), + clock: time(), + clockTz: time({ withTimezone: true }), + day: date(), moment: timestamp(), }); @@ -331,6 +334,106 @@ describe("a tampered cursor value is refused, never coerced", () => { expect(cursorValueIsCanonicalText(core_users.id)).toBe(false); expect(cursorValueIsCanonicalText(core_users.name)).toBe(false); }); + + /** + * The gap a pattern alone leaves open. + * + * Every one of these has the shape of a timestamp and is not a moment, so a + * shape check waves it through and Postgres answers the cast with + * `invalid input syntax` - a 500 produced by a query string, on a route + * whose whole promise is that it does not do that. + */ + describe("impossible values that still look like timestamps", () => { + it.each([ + ["month 13", "2026-13-01"], + ["month 0", "2026-00-01"], + ["a day past the end of the month", "2026-02-30"], + ["a day past the end of a short month", "2026-04-31"], + ["29 February in a common year", "2025-02-29"], + ["29 February in a century that is not a leap year", "1900-02-29"], + ["day 0", "2026-08-00"], + ["day 32", "2026-01-32"], + ["hour 24", "2026-08-09 24:00:00"], + ["hour 99", "2026-08-09 99:00:00"], + ["minute 60", "2026-08-09 23:60:00"], + ["second 61", "2026-08-09 23:59:61"], + ["year 0", "0000-01-01"], + ])("refuses %s", (_why, value) => { + refuses(core_users.createdAt, value); + }); + + it.each([ + ["an offset past the maximum", "2026-08-09 10:00:00+25:00"], + ["an offset with 99 minutes", "2026-08-09 10:00:00+12:99"], + ["an offset that is not a number", "2026-08-09 10:00:00+ab"], + ["a seven-digit fraction", "2026-08-09 10:00:00.1234567"], + ["trailing rubbish", "2026-08-09 10:00:00 OR 1=1"], + ["an era suffix, which nothing here mints", "2026-08-09 BC"], + ])("refuses %s", (_why, value) => { + refuses(core_users.createdAt, value); + }); + + it.each([ + ["29 February in a leap year", "2024-02-29"], + ["29 February in a leap century", "2000-02-29"], + ["the last second of a day", "2026-08-09 23:59:59"], + ["the first second of a day", "2026-08-09 00:00:00"], + ["31 December", "2026-12-31"], + ["microseconds", "2026-08-09 10:00:00.123456"], + ["a whole-hour offset", "2026-08-09 10:00:00+02"], + ["a half-hour offset", "2026-08-09 10:00:00+05:30"], + ["a compact offset", "2026-08-09 10:00:00-0400"], + ["a negative offset", "2026-08-09 10:00:00-04"], + ])("accepts %s", (_why, value) => { + expect(cursorValueForColumn(core_users.createdAt, value)).toBe(value); + }); + + it("preserves microseconds through validation, digit for digit", () => { + // The reason the value is kept as text at all. Anything that reformats + // it here is a truncation the next comparison inherits. + const value = "2026-08-09 10:00:00.000001"; + + expect(cursorValueForColumn(core_users.createdAt, value)).toBe(value); + }); + }); + }); + + /** + * A `date` and a `time` column look like strings to Drizzle - `dataType` says + * `"string"` - but Postgres still has to parse them. Classifying from the SQL + * type is what stops `'nonsense'::date` being a 500. + */ + describe("other temporal columns", () => { + it("holds a date column to a date, and nothing more", () => { + expect(cursorValueForColumn(probes.day, "2026-08-09")).toBe("2026-08-09"); + refuses(probes.day, "2026-08-09 10:00:00"); + refuses(probes.day, "2026-02-30"); + refuses(probes.day, "not-a-date"); + }); + + it("holds a time column to a time, and refuses a zone it has not got", () => { + expect(cursorValueForColumn(probes.clock, "10:00:00")).toBe("10:00:00"); + expect(cursorValueForColumn(probes.clock, "10:00:00.123456")).toBe( + "10:00:00.123456", + ); + refuses(probes.clock, "10:00:00+02"); + refuses(probes.clock, "24:00:00"); + refuses(probes.clock, "2026-08-09"); + }); + + it("lets a time-with-zone column carry its zone", () => { + expect(cursorValueForColumn(probes.clockTz, "10:00:00+02")).toBe( + "10:00:00+02", + ); + refuses(probes.clockTz, "10:00:00+25"); + }); + + it("binds every temporal column as text plus a cast", () => { + for (const column of [probes.day, probes.clock, probes.clockTz]) { + expect(cursorValueIsCanonicalText(column)).toBe(true); + expect(isCursorSortableColumn(column)).toBe(true); + } + }); }); describe("string", () => { @@ -377,10 +480,20 @@ describe("minting keeps the database's own representation", () => { ).toBe("2026-08-09 10:00:00.123456"); }); - it("falls back to an ISO string only when handed a Date", () => { + it("rewrites a Date into the form the column would have been read in", () => { + // A `Date` only arrives when a caller mints from a value it is holding - + // the paginated path reads `::text`. Writing it the way Postgres writes a + // `timestamp` keeps minting and validation speaking one grammar, so a + // cursor this module produced can never be one it later refuses. expect( cursorValueOf(core_users.createdAt, new Date("2026-08-09T10:00:00.123Z")), - ).toBe("2026-08-09T10:00:00.123Z"); + ).toBe("2026-08-09 10:00:00.123"); + }); + + it("refuses a Date that is not a moment, rather than minting nonsense", () => { + expect(() => + cursorValueOf(core_users.createdAt, new Date("not-a-date")), + ).toThrow(/invalid date/i); }); it("carries a bigint as a decimal string, which JSON can hold", () => { diff --git a/packages/vitnode/src/api/lib/pagination-cursor.ts b/packages/vitnode/src/api/lib/pagination-cursor.ts index 0067a41b5..289222d30 100644 --- a/packages/vitnode/src/api/lib/pagination-cursor.ts +++ b/packages/vitnode/src/api/lib/pagination-cursor.ts @@ -48,12 +48,12 @@ export interface PaginationCursor { * answering it in one place is what stops the second answer being looser than * the first. */ -type CursorKind = "bigint" | "boolean" | "date" | "number" | "string"; +type CursorKind = "bigint" | "boolean" | "number" | "string" | "temporal"; const KIND_BY_DATA_TYPE: Record = { bigint: "bigint", boolean: "boolean", - date: "date", + date: "temporal", number: "number", string: "string", }; @@ -64,6 +64,31 @@ const badRequest = (message: string): HTTPException => /** The one message a tampered or stale cursor ever produces. */ const INVALID_CURSOR = "Invalid pagination cursor."; +/** + * The three shapes a temporal value comes in, keyed by what Postgres will parse. + * + * Classified from the **SQL** type rather than the JavaScript one, because the + * two disagree in exactly the case that matters: `date()` and `time()` hand back + * plain strings, so `dataType` calls them `"string"` - and a string cursor bound + * straight into `column > $1` would reach Postgres as `'nonsense'::date` and + * come back as a 500 rather than a 400. + */ +type TemporalType = "date" | "time" | "timestamp"; + +const temporalTypeOf = (column: PgColumn): null | TemporalType => { + const sqlType = column.getSQLType().toLowerCase(); + + // Order matters: "timestamp with time zone" also starts with "time". + if (sqlType.startsWith("timestamp")) return "timestamp"; + if (sqlType.startsWith("time")) return "time"; + if (sqlType.startsWith("date")) return "date"; + + return null; +}; + +const hasTimeZone = (column: PgColumn): boolean => + column.getSQLType().toLowerCase().includes("with time zone"); + /** * Whether a column can be paged through at all. * @@ -72,9 +97,11 @@ const INVALID_CURSOR = "Invalid pagination cursor."; * against. Refused rather than approximated. */ export const isCursorSortableColumn = (column: PgColumn): boolean => - column.dataType in KIND_BY_DATA_TYPE; + temporalTypeOf(column) !== null || column.dataType in KIND_BY_DATA_TYPE; const kindOf = (column: PgColumn): CursorKind => { + if (temporalTypeOf(column)) return "temporal"; + const kind = KIND_BY_DATA_TYPE[column.dataType]; if (!kind) { throw badRequest( @@ -86,22 +113,152 @@ const kindOf = (column: PgColumn): CursorKind => { }; /** - * A Postgres timestamp, date or time as `::text` renders it. + * The grammar of a Postgres temporal value, as `::text` renders it. + * + * One pattern per SQL type, because a `date` column and a `timestamp` column do + * not accept the same strings and pretending they do is how a cursor for one + * ends up being parsed as the other: + * + * | SQL type | accepted | + * | --------------------------- | ------------------------------------------- | + * | `date` | `2026-08-09` | + * | `time` | `10:00:00`, `10:00:00.123456` | + * | `time with time zone` | the above, optionally `+02` / `Z` | + * | `timestamp` | a date, optionally a time, optionally a zone | + * | `timestamp with time zone` | the same, and that is what `::text` writes | * - * Anchored and permissive in the right places: `timestamp` gives - * `2026-08-09 10:00:00.123456`, `timestamptz` appends `+00`, and a plain `date` - * gives just the day. An ISO string with a `T` and a `Z` is accepted too, - * because that is what a `Date` produces on the one path that has no database - * text to offer. + * A `T` separator and a `Z` designator are accepted alongside the space-and- + * offset form Postgres writes, because a JavaScript `Date` is the one input this + * module takes that has no database text behind it. * - * The regex is not decoration - it is what keeps a tampered cursor from reaching - * Postgres as an invalid cast, which would be a 500 rather than a 400. + * Matching the shape is only half of it. These patterns cannot tell `2026-02-30` + * from `2026-02-28`, so every capture is range-checked afterwards - see + * {@link isRealTemporal}. */ -const CANONICAL_TIMESTAMP = - /^\d{4}-\d{2}-\d{2}([ T]\d{2}:\d{2}:\d{2}(\.\d{1,6})?((\+|-)\d{2}(:?\d{2})?|Z)?)?$/; +const TEMPORAL_GRAMMAR: Record = { + date: /^(?\d{4,6})-(?\d{2})-(?\d{2})$/, + time: /^(?\d{2}):(?\d{2}):(?\d{2})(?:\.\d{1,6})?(?.*)$/, + timestamp: + /^(?\d{4,6})-(?\d{2})-(?\d{2})(?:[ T](?\d{2}):(?\d{2}):(?\d{2})(?:\.\d{1,6})?(?.*))?$/, +}; + +/** + * Whatever the trailing group swallowed, checked rather than trusted. + * + * `(?.*)` is deliberately greedy: it catches a seventh fractional digit, + * an era suffix and `OR 1=1` alike, and hands all of them here to be refused. + */ +type TemporalParts = Partial< + Record< + "day" | "hour" | "minute" | "month" | "second" | "year" | "zone", + string + > +>; + +/** `Z`, or `±HH`, `±HH:MM`, `±HHMM`, `±HH:MM:SS` - the forms Postgres writes. */ +const ZONE = /^([+-])(\d{2})(?::?(\d{2}))?(?::?(\d{2}))?$/; + +const DAYS_IN_MONTH = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + +const isLeapYear = (year: number): boolean => + (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0; + +const isRealZone = (raw: string, allowed: boolean): boolean => { + if (raw === "") return true; + if (!allowed) return false; + if (raw === "Z") return true; + + const match = ZONE.exec(raw); + if (!match) return false; + + const [, , hours, minutes = "0", seconds = "0"] = match; + + // Postgres refuses anything past ±15:59:59, and so does this. + return Number(hours) <= 15 && Number(minutes) <= 59 && Number(seconds) <= 59; +}; + +/** + * Whether a shaped temporal string is a moment that exists. + * + * The reason a pattern is not enough: `2026-02-30`, `2025-02-29`, `2026-13-01` + * and `2026-08-09 23:60:00` all match the shape and all make Postgres raise + * `invalid input syntax`, which is a 500 arriving from a query string. Every one + * of them is refused here instead, before anything is bound. + * + * Deliberately stricter than Postgres in one place: Postgres reads `24:00:00` as + * the following midnight, but its own `::text` never writes it, so a cursor + * carrying one did not come from a row. + */ +const isRealTemporal = ( + column: PgColumn, + temporal: TemporalType, + value: string, +): boolean => { + const match = TEMPORAL_GRAMMAR[temporal].exec(value); + if (!match) return false; + + const { day, hour, minute, month, second, year, zone } = + (match.groups as TemporalParts | undefined) ?? {}; + + if (year !== undefined) { + const [y, m, d] = [Number(year), Number(month), Number(day)]; + if (y < 1 || y > 294276 || m < 1 || m > 12) return false; + + const limit = m === 2 && isLeapYear(y) ? 29 : DAYS_IN_MONTH[m - 1]; + if (d < 1 || d > limit) return false; + } + + // A bare `date`, or a `timestamp` written as one: no time to check. + if (hour === undefined) return true; + + if (Number(hour) > 23 || Number(minute) > 59 || Number(second) > 59) { + return false; + } + + // A `timestamp` takes an offset and throws it away, which is what lets a + // `Date`-minted value carry `+00`. A `time` without a zone does not. + return isRealZone( + zone ?? "", + temporal === "timestamp" || hasTimeZone(column), + ); +}; const DECIMAL_INTEGER = /^-?\d+$/; +const pad = (value: number, width = 2): string => + String(value).padStart(width, "0"); + +/** + * A `Date` written the way Postgres writes the column it belongs to. + * + * Only reachable when a caller mints a cursor from a value it already holds + * rather than from a row - the paginated path selects `::text` and never sees a + * `Date`. Even so it goes through the same grammar as everything else, so a + * minted cursor and an accepted cursor can never disagree about what a value + * looks like. + */ +const canonicalFromDate = (column: PgColumn, value: Date): string => { + const time = value.getTime(); + if (!Number.isFinite(time)) { + throw new Error( + `Cannot build a pagination cursor from an invalid date on "${column.name}".`, + ); + } + + const zone = hasTimeZone(column) ? "+00" : ""; + const date = `${pad(value.getUTCFullYear(), 4)}-${pad(value.getUTCMonth() + 1)}-${pad(value.getUTCDate())}`; + const clock = `${pad(value.getUTCHours())}:${pad(value.getUTCMinutes())}:${pad(value.getUTCSeconds())}.${pad(value.getUTCMilliseconds(), 3)}`; + + switch (temporalTypeOf(column)) { + case "date": + return date; + case "time": + return `${clock}${zone}`; + default: + return `${date} ${clock}${zone}`; + } +}; + /** * One column value, flattened into the cursor's canonical representation. * @@ -113,16 +270,16 @@ const DECIMAL_INTEGER = /^-?\d+$/; * | boolean | a JSON boolean | * | string | a JSON string | * | bigint | a decimal string, because JSON has no bigint | - * | date | the database's own `::text`, microseconds and all | + * | temporal | the database's own `::text`, microseconds and all | * | null | `null` | * - * The date row is the one worth reading twice. A Postgres `timestamp` keeps + * The temporal row is the one worth reading twice. A Postgres `timestamp` keeps * microseconds and a JavaScript `Date` keeps milliseconds, so a value that has * been through a `Date` is *strictly smaller* than the one still in the table - * and comparing against it would exclude the entire millisecond it came from. * Since `now()` stamps every row in one statement identically, that is not an * edge case: it would end a bulk-imported collection's walk after page one. So - * the mint path reads `column::text` and this function keeps it exactly as + * the page query selects `column::text` and this function keeps it exactly as * Postgres wrote it. */ export const cursorValueOf = ( @@ -141,18 +298,18 @@ export const cursorValueOf = ( if (typeof value === "boolean") return value; break; } - case "date": { - // Already `::text` from the database on the normal path. A `Date` only - // reaches here if a caller supplied one, and its ISO form is the best - // that value can offer. - if (value instanceof Date) return value.toISOString(); - if (typeof value === "string") return value; - break; - } case "number": { if (typeof value === "number") return value; break; } + case "temporal": { + // Already `::text` from the database on the paginated path. A `Date` only + // reaches here when a caller mints from a value it is holding, and it is + // rewritten into the form Postgres would have written. + if (value instanceof Date) return canonicalFromDate(column, value); + if (typeof value === "string") return value; + break; + } default: { if (typeof value === "string") return value; break; @@ -177,9 +334,14 @@ export const cursorValueOf = ( * every kind checks the shape it expects and refuses anything else with a 400. * * Returns the value in the form the SQL comparison needs: a real `boolean`, - * `number` or `bigint` for those kinds, and for a date the **canonical text**, - * which the predicate binds with an explicit cast so Postgres parses it at full - * precision. + * `number` or `bigint` for those kinds, and for a temporal column the + * **canonical text**, which the predicate binds with an explicit cast so + * Postgres parses it at full precision. + * + * A temporal value is checked for more than shape. `2026-02-30` and + * `2026-08-09 23:60:00` look like timestamps and are not moments, and Postgres + * answers a cast of either with `invalid input syntax` - a 500 produced by a + * query string. Both are refused here, so nothing impossible is ever bound. */ export const cursorValueForColumn = ( column: PgColumn, @@ -202,20 +364,25 @@ export const cursorValueForColumn = ( return value; } - case "date": { - if (typeof value !== "string" || !CANONICAL_TIMESTAMP.test(value)) { + case "number": { + if (typeof value !== "number" || !Number.isFinite(value)) { throw badRequest(INVALID_CURSOR); } - // Kept as text. The predicate casts it back to the column's own type, so - // Postgres does the parsing - at the precision it stored. return value; } - case "number": { - if (typeof value !== "number" || !Number.isFinite(value)) { + case "temporal": { + const temporal = temporalTypeOf(column); + if ( + typeof value !== "string" || + !temporal || + !isRealTemporal(column, temporal, value) + ) { throw badRequest(INVALID_CURSOR); } + // Kept as text. The predicate casts it back to the column's own type, so + // Postgres does the parsing - at the precision it stored. return value; } default: { @@ -226,9 +393,17 @@ export const cursorValueForColumn = ( } }; -/** Whether the comparison binds this column's value as text plus a cast. */ +/** + * Whether this column's value travels as canonical text plus a cast. + * + * True for every temporal type, and it decides two things at once: the page + * query selects `column::text` rather than the column, and the predicate binds + * the cursor back with an explicit cast. Both halves exist so the microseconds + * Postgres stored survive a round trip that JavaScript's millisecond `Date` + * would otherwise truncate. + */ export const cursorValueIsCanonicalText = (column: PgColumn): boolean => - kindOf(column) === "date"; + kindOf(column) === "temporal"; export const encodePaginationCursor = (cursor: PaginationCursor): string => Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); diff --git a/packages/vitnode/src/api/lib/with-pagination.ts b/packages/vitnode/src/api/lib/with-pagination.ts index ef5974db2..99a9823c3 100644 --- a/packages/vitnode/src/api/lib/with-pagination.ts +++ b/packages/vitnode/src/api/lib/with-pagination.ts @@ -16,7 +16,6 @@ import { eq, gt, ilike, - inArray, isNotNull, isNull, lt, @@ -39,6 +38,25 @@ import { /** Nobody may ask for more than this in one page, whatever they send. */ const MAX_PAGE_SIZE = 100; +/** + * The column a page query carries purely so its rows can be turned into cursors. + * + * Selected by the **same statement** that returns the rows, and removed again + * before anything leaves this module. It exists because a cursor has to describe + * the position the returned row actually occupied, and the only way to be + * certain of that is to read the two out of one snapshot. + * + * Prefixed so it cannot collide with a column name, and stripped rather than + * documented, because it is pagination's business and nobody else's. + */ +export const PAGINATION_CURSOR_FIELD = "__cursorValue"; + +/** What a page query must spread into its projection. */ +export type PaginationCursorSelection = Record< + typeof PAGINATION_CURSOR_FIELD, + PgColumn | SQL +>; + /** * Reads `first`, `last` and `cursor`, or refuses the request with a 400. * @@ -187,10 +205,10 @@ function buildCursorCondition({ * * Two shapes, because two kinds of value survive a round trip differently: * - * - a **date** travels as the database's own `::text` and is bound back with an - * explicit cast, so Postgres parses the microseconds it wrote. Binding a - * JavaScript `Date` here would silently truncate to milliseconds and exclude - * the whole millisecond the cursor came from. + * - a **temporal** value travels as the database's own `::text` and is bound + * back with an explicit cast, so Postgres parses the microseconds it wrote. + * Binding a JavaScript `Date` here would silently truncate to milliseconds and + * exclude the whole millisecond the cursor came from. * - **everything else** - a number, a string, a boolean, a bigint - is exact in * JavaScript already, so it goes through the column's own encoder. * @@ -261,6 +279,14 @@ export async function withPagination< }; primaryCursor: PgColumn; query: (args: { + /** + * Spread this into the projection: `.select({ ...fields, ...cursorSelection })`. + * + * Not optional in practice. It is how the cursor value is read out of the + * same statement as the row, and a query that omits it can only be paged by + * a column it happens to have selected itself. + */ + cursorSelection: PaginationCursorSelection; limit: number | Placeholder; orderBy: SQL; where: SQL | undefined; @@ -269,7 +295,7 @@ export async function withPagination< table: Omit, "enableRLS">; where?: SQL; }): Promise<{ - edges: QueryMin[]; + edges: Omit[]; pageInfo: { count: number; /** An opaque cursor. Hand it back as `cursor`; never parse it. */ @@ -343,22 +369,34 @@ export async function withPagination< const totalCount = await fetchTotalCount(c, table, baseWhere); + /** + * The cursor value, projected by the page query itself. + * + * A temporal column goes through `::text` so no microsecond is lost on the way + * out; everything else is exact in JavaScript already and is selected as it + * is. Either way it rides along with the row, which is the point: a cursor + * minted from a *second* read would describe wherever the boundary row had got + * to by then, not where it was when it was chosen for this page. + */ + const cursorSelection: PaginationCursorSelection = { + [PAGINATION_CURSOR_FIELD]: cursorValueIsCanonicalText(orderColumn) + ? sql`${orderColumn}::text` + : orderColumn, + }; + const limit = (first ?? last ?? 50) + 1; - const edges = await query({ limit, where, orderBy }); + const edges = await query({ cursorSelection, limit, where, orderBy }); const requested = first ?? last ?? edges.length; const hasMore = edges.length > requested; const slicedEdges = edges.slice(0, requested); const finalEdges = isForward ? slicedEdges : slicedEdges.reverse(); - const boundaries = await cursorsFor({ - c, + const boundaries = cursorsFrom({ edges: finalEdges, orderColumn, orderName, - primary, primaryName: primaryCursor.name, - table, }); return { @@ -373,80 +411,86 @@ export async function withPagination< finalEdges.length === 0 ? false : isForward ? Boolean(cursor) : hasMore, ...boundaries, }, - edges: finalEdges, + edges: finalEdges.map(withoutCursorField), }; } /** - * The two cursors a page hands back. + * The row as the caller asked for it, with pagination's own column taken back. * - * The order column's value normally comes straight off the row, because a list - * almost always selects the column it sorts by. Two cases need one extra - * primary-key lookup of at most two rows: + * The internal value is projected for one purpose and has no business in an + * admin response, a public response, an OpenAPI schema, a search document or a + * revision snapshot - all of which are built from what this returns. + */ +function withoutCursorField>( + row: QueryMin, +): Omit { + if (!(PAGINATION_CURSOR_FIELD in row)) return row; + + const { [PAGINATION_CURSOR_FIELD]: _cursorValue, ...rest } = row; + + return rest; +} + +/** + * The two cursors a page hands back, read off the page itself. * - * 1. **a projection that does not include the order column** - a public read - * naming a subset of fields. Without the value there is no tuple to mint, and - * falling back to an identifier is the bug this module exists to remove; - * 2. **a date column** - the row carries a JavaScript `Date`, which has already - * lost the microseconds Postgres stored. The canonical `::text` is fetched so - * the cursor carries the value the next comparison will be parsed from. + * No query. That is the entire design: the value and the row come out of one + * `SELECT`, so the tuple a cursor names is the tuple that actually decided where + * the row sat. * - * The lookup reads the boundary rows *now*, while they are still the boundary - - * it is part of minting the cursor, not part of using one. Nothing re-reads them - * when the cursor comes back. + * It used to be a second `SELECT` of the boundary rows by id, which looked + * harmless and was not. Between the page query and that lookup another writer + * can move the boundary row - so a row chosen at `(10:00, 42)` would be handed + * back as a cursor saying `(14:00, 42)`, and the next page would start after + * 14:00 and skip everything in between. A `DELETE` in the same window was worse: + * the lookup returned nothing, the value became `null`, and for a nullable + * ordering `null` is a *real* position inside the null block - so the walk + * jumped there and abandoned the rest of the collection. Both are gone by + * construction rather than by locking. */ -async function cursorsFor({ - c, +function cursorsFrom({ edges, orderColumn, orderName, - primary, primaryName, - table, }: { - c: Context; edges: readonly Record[]; orderColumn: PgColumn; orderName: string; - primary: PgColumn; primaryName: string; - table: PgTable; -}): Promise<{ endCursor: null | string; startCursor: null | string }> { +}): { endCursor: null | string; startCursor: null | string } { const first = edges[0]; const last = edges.at(-1); if (!first || !last) return { endCursor: null, startCursor: null }; - const idOf = (row: Record): number => - Number(row[primaryName]); - - const canonical = cursorValueIsCanonicalText(orderColumn); - const onTheRow = !canonical && orderName in first && orderName in last; - - const values = new Map(); - if (onTheRow) { - values.set(idOf(first), first[orderName]); - values.set(idOf(last), last[orderName]); - } else { - const ids = [...new Set([idOf(first), idOf(last)])]; - const rows = await c - .get("db") - .select({ - id: primary, - // `::text` for a date, so no precision is lost between the value the - // cursor carries and the value the next page compares against. - value: canonical ? sql`${orderColumn}::text` : orderColumn, - }) - .from(table) - .where(inArray(primary, ids)); + /** + * Where the boundary value comes from, in order of preference. + * + * The projected field is the answer for every query built through this module. + * A query that omits it can still be paged by a column it selected itself - + * exact for a number, a string or a boolean, and from the same statement, so + * the invariant holds. A temporal column is the one case with no safe + * fallback: the row carries a `Date` that has already dropped the microseconds + * the next comparison needs, so minting from it would hand out a cursor that + * silently re-reads part of the page it came from. + */ + const valueOf = (row: Record): unknown => { + if (PAGINATION_CURSOR_FIELD in row) return row[PAGINATION_CURSOR_FIELD]; + if (!cursorValueIsCanonicalText(orderColumn) && orderName in row) { + return row[orderName]; + } - for (const row of rows) values.set(Number(row.id), row.value); - } + throw new Error( + `The page query for "${orderName}" must spread \`cursorSelection\` into its projection, so the cursor value is read from the same statement as the row.`, + ); + }; const mint = (row: Record): string => encodePaginationCursor({ column: orderName, - id: idOf(row), - value: cursorValueOf(orderColumn, values.get(idOf(row))), + id: Number(row[primaryName]), + value: cursorValueOf(orderColumn, valueOf(row)), }); return { endCursor: mint(last), startCursor: mint(first) }; diff --git a/packages/vitnode/src/api/modules/admin/advanced/cron/routes/get.route.ts b/packages/vitnode/src/api/modules/admin/advanced/cron/routes/get.route.ts index b49a25c9c..d12a9d01a 100644 --- a/packages/vitnode/src/api/modules/admin/advanced/cron/routes/get.route.ts +++ b/packages/vitnode/src/api/modules/admin/advanced/cron/routes/get.route.ts @@ -1,3 +1,4 @@ +import { getTableColumns } from "drizzle-orm"; import z from "zod"; import { buildRoute } from "@/api/lib/route"; @@ -56,10 +57,10 @@ export const getCronsRoute = buildRoute({ }, c, primaryCursor: core_cron.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") - .select() + .select({ ...getTableColumns(core_cron), ...cursorSelection }) .from(core_cron) .where(where) .orderBy(orderBy) diff --git a/packages/vitnode/src/api/modules/admin/advanced/queue/routes/get.route.ts b/packages/vitnode/src/api/modules/admin/advanced/queue/routes/get.route.ts index 40181afa9..affd8a4ae 100644 --- a/packages/vitnode/src/api/modules/admin/advanced/queue/routes/get.route.ts +++ b/packages/vitnode/src/api/modules/admin/advanced/queue/routes/get.route.ts @@ -1,4 +1,4 @@ -import { inArray } from "drizzle-orm"; +import { getTableColumns, inArray } from "drizzle-orm"; import z from "zod"; import { buildRoute } from "@/api/lib/route"; @@ -73,10 +73,10 @@ export const getQueueTasksRoute = buildRoute({ c, primaryCursor: core_queue.id, where: statuses.length ? inArray(core_queue.status, statuses) : undefined, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") - .select() + .select({ ...getTableColumns(core_queue), ...cursorSelection }) .from(core_queue) .where(where) .orderBy(orderBy) diff --git a/packages/vitnode/src/api/modules/admin/debug/routes/logs.route.ts b/packages/vitnode/src/api/modules/admin/debug/routes/logs.route.ts index 74266f343..0ec2eb213 100644 --- a/packages/vitnode/src/api/modules/admin/debug/routes/logs.route.ts +++ b/packages/vitnode/src/api/modules/admin/debug/routes/logs.route.ts @@ -66,10 +66,11 @@ export const logsDebugAdminRoute = buildRoute({ query, }, primaryCursor: core_logs.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_logs.id, pluginId: core_logs.pluginId, type: core_logs.type, diff --git a/packages/vitnode/src/api/modules/admin/files/routes/list.route.ts b/packages/vitnode/src/api/modules/admin/files/routes/list.route.ts index 873605bf3..5ce24159c 100644 --- a/packages/vitnode/src/api/modules/admin/files/routes/list.route.ts +++ b/packages/vitnode/src/api/modules/admin/files/routes/list.route.ts @@ -90,10 +90,11 @@ export const listFilesAdminRoute = buildRoute({ c, primaryCursor: core_files.id, search: [core_files.name], - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_files.id, name: core_files.name, key: core_files.key, diff --git a/packages/vitnode/src/api/modules/admin/roles/routes/list.route.ts b/packages/vitnode/src/api/modules/admin/roles/routes/list.route.ts index efe9f5138..4985eba3b 100644 --- a/packages/vitnode/src/api/modules/admin/roles/routes/list.route.ts +++ b/packages/vitnode/src/api/modules/admin/roles/routes/list.route.ts @@ -100,10 +100,11 @@ export const listRolesAdminRoute = buildRoute({ ) : undefined, primaryCursor: core_roles.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_roles.id, color: core_roles.color, protected: core_roles.protected, diff --git a/packages/vitnode/src/api/modules/admin/staff/routes/admins.route.ts b/packages/vitnode/src/api/modules/admin/staff/routes/admins.route.ts index f1bfbd19f..1b5b4d786 100644 --- a/packages/vitnode/src/api/modules/admin/staff/routes/admins.route.ts +++ b/packages/vitnode/src/api/modules/admin/staff/routes/admins.route.ts @@ -38,10 +38,11 @@ export const listAdminsStaffAdminRoute = buildRoute({ query, }, primaryCursor: core_admin_permissions.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_admin_permissions.id, roleId: core_admin_permissions.roleId, userId: core_admin_permissions.userId, diff --git a/packages/vitnode/src/api/modules/admin/staff/routes/moderators.route.ts b/packages/vitnode/src/api/modules/admin/staff/routes/moderators.route.ts index 1ac4f1eb7..b0bcb0353 100644 --- a/packages/vitnode/src/api/modules/admin/staff/routes/moderators.route.ts +++ b/packages/vitnode/src/api/modules/admin/staff/routes/moderators.route.ts @@ -38,10 +38,11 @@ export const listModeratorsStaffAdminRoute = buildRoute({ query, }, primaryCursor: core_moderators_permissions.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_moderators_permissions.id, roleId: core_moderators_permissions.roleId, userId: core_moderators_permissions.userId, diff --git a/packages/vitnode/src/api/modules/admin/users/routes/list.route.ts b/packages/vitnode/src/api/modules/admin/users/routes/list.route.ts index bd04bd9dc..6ddc5104a 100644 --- a/packages/vitnode/src/api/modules/admin/users/routes/list.route.ts +++ b/packages/vitnode/src/api/modules/admin/users/routes/list.route.ts @@ -90,10 +90,11 @@ export const listUsersAdminRoute = buildRoute({ search: [core_users.name, core_users.email, core_users.nameCode], where: roleIds.length ? inArray(core_users.roleId, roleIds) : undefined, primaryCursor: core_users.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_users.id, name: core_users.name, email: core_users.email, diff --git a/packages/vitnode/src/api/modules/admin/users/routes/users.route.ts b/packages/vitnode/src/api/modules/admin/users/routes/users.route.ts index a2e6fe2cc..cb9f70af2 100644 --- a/packages/vitnode/src/api/modules/admin/users/routes/users.route.ts +++ b/packages/vitnode/src/api/modules/admin/users/routes/users.route.ts @@ -59,10 +59,11 @@ export const usersAdminRoute = buildRoute({ query, }, primaryCursor: core_users.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_users.id, name: core_users.name, email: core_users.email, diff --git a/packages/vitnode/src/api/modules/users/files/routes/list.route.ts b/packages/vitnode/src/api/modules/users/files/routes/list.route.ts index d9a40c7d4..dce44b888 100644 --- a/packages/vitnode/src/api/modules/users/files/routes/list.route.ts +++ b/packages/vitnode/src/api/modules/users/files/routes/list.route.ts @@ -72,10 +72,11 @@ export const listUserFilesRoute = buildRoute({ primaryCursor: core_files.id, search: [core_files.name], where: eq(core_files.userId, user.id), - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: core_files.id, name: core_files.name, key: core_files.key, diff --git a/packages/vitnode/src/content/server/localized-public-service.ts b/packages/vitnode/src/content/server/localized-public-service.ts index 67cf47cb5..c13d30d26 100644 --- a/packages/vitnode/src/content/server/localized-public-service.ts +++ b/packages/vitnode/src/content/server/localized-public-service.ts @@ -10,6 +10,7 @@ import type { Context } from "hono"; import { and, eq, exists, not, or, sql } from "drizzle-orm"; import { alias } from "drizzle-orm/pg-core"; +import type { PaginationCursorSelection } from "../../api/lib/with-pagination"; import type { AnyContentTypeDefinition, ContentPublicSelect } from "../types"; import type { ContentAdvancedStore } from "./advanced-store"; import type { ContentLanguage } from "./language-resolver"; @@ -414,11 +415,20 @@ export const createContentLocalizedPublicService = < const read = async ( scope: ResolvedLocale, where: SQL | undefined, - { limit, order }: { limit: number; order?: SQL }, + { + cursorSelection, + limit, + order, + }: { + /** Only a paginated read asks for one; a single read has nothing to mint. */ + cursorSelection?: PaginationCursorSelection; + limit: number; + order?: SQL; + }, ): Promise[]> => { const query = c .get("db") - .select(selection(scope.fallbackTo !== null)) + .select({ ...selection(scope.fallbackTo !== null), ...cursorSelection }) .from(table); if (!scope.fallbackTo) { @@ -582,8 +592,14 @@ export const createContentLocalizedPublicService = < }, table, where, - query: async ({ limit, orderBy: order, where: paged }) => + query: async ({ + cursorSelection, + limit, + orderBy: order, + where: paged, + }) => await read(resolved, paged, { + cursorSelection, limit: typeof limit === "number" ? Math.min(limit, CONTENT_PUBLIC_MAX_PAGE_SIZE + 1) diff --git a/packages/vitnode/src/content/server/openapi-parity.test.ts b/packages/vitnode/src/content/server/openapi-parity.test.ts index eacce45bc..3a0a9ec31 100644 --- a/packages/vitnode/src/content/server/openapi-parity.test.ts +++ b/packages/vitnode/src/content/server/openapi-parity.test.ts @@ -311,6 +311,16 @@ beforeEach(() => { }); describe("admin routes match their OpenAPI document", () => { + it("publishes nothing about pagination's own column", () => { + // `__cursorValue` is projected by every list query so a cursor can be + // minted from the same statement as the row. It is implementation, not + // contract: it is stripped before the handler sees a row, so it must not + // appear anywhere a generated client would find it. + expect(JSON.stringify(editorialSuite().document)).not.toContain( + "__cursorValue", + ); + }); + it("lists", async () => { await expectParity(editorialSuite(), { expected: 200, @@ -787,6 +797,10 @@ describe("public routes match their OpenAPI document", () => { return mount(buildContentPublicRoutes(posts, { pluginId: PLUGIN_ID })); }; + it("publishes nothing about pagination's own column", () => { + expect(JSON.stringify(suite().document)).not.toContain("__cursorValue"); + }); + it("lists", async () => { await expectParity(suite(), { expected: 200, diff --git a/packages/vitnode/src/content/server/public-service.ts b/packages/vitnode/src/content/server/public-service.ts index 9ae1949a1..5bd1c32ea 100644 --- a/packages/vitnode/src/content/server/public-service.ts +++ b/packages/vitnode/src/content/server/public-service.ts @@ -465,10 +465,13 @@ export const createContentPublicService = < }, table, where: conditions.length > 1 ? and(...conditions) : conditions[0], - query: async ({ limit, orderBy: order, where }) => + query: async ({ cursorSelection, limit, orderBy: order, where }) => await c .get("db") - .select(selection()) + // The cursor value is projected by this statement and stripped from + // the row before `project` ever sees it, so the public allowlist is + // unchanged: it is pagination's own column, not a field. + .select({ ...selection(), ...cursorSelection }) .from(table) .where(where) .orderBy(order) diff --git a/packages/vitnode/src/content/server/service.test.ts b/packages/vitnode/src/content/server/service.test.ts index 1ff2fa5a1..9d00c24a9 100644 --- a/packages/vitnode/src/content/server/service.test.ts +++ b/packages/vitnode/src/content/server/service.test.ts @@ -324,23 +324,19 @@ describe("content service", () => { const page = (rows: unknown[]) => [[{ count: rows.length }], rows]; it("joins once per reference field instead of querying per row", async () => { - // `updatedAt` is on the rows because the list really selects it - the - // page is built from `ownSelection()`. That matters here: the cursor is - // the ordered tuple, so a page whose rows are missing the order column - // costs one extra round trip to mint one. const { c, calls } = createDbMock( page([ { + __cursorValue: "2026-01-02 00:00:00", id: 1, label__author: "Ada", label__category: "News", - updatedAt: new Date("2026-01-02T00:00:00.000Z"), }, { + __cursorValue: "2026-01-01 00:00:00", id: 2, label__author: null, label__category: "News", - updatedAt: new Date("2026-01-01T00:00:00.000Z"), }, ]), ); @@ -349,22 +345,29 @@ describe("content service", () => { // `author` and `category` - one join each, and no per-row lookup. expect(opsOf(calls, "leftJoin")).toHaveLength(2); - // Three, and all three are constant: the count, the page, and one - // primary-key read of the two boundary rows to mint the cursors. That - // last one exists because the default ordering is a timestamp, and a - // timestamp has to be carried at the database's own precision - see - // `pagination-cursor.ts`. What matters is that none of them grows with - // the page. - expect(opsOf(calls, "select")).toHaveLength(3); + // Two, and both constant: the count and the page. There is no third + // read to mint the cursors, because the page query already selected the + // value they are made of - which is also what makes a cursor describe + // where the row was rather than where it has since moved. + expect(opsOf(calls, "select")).toHaveLength(2); }); it("splits the joined labels out of the row", async () => { const { c } = createDbMock( - page([{ id: 1, label__author: "Ada", label__category: "News" }]), + page([ + { + __cursorValue: "2026-01-02 00:00:00", + id: 1, + label__author: "Ada", + label__category: "News", + }, + ]), ); const { edges } = await articles.service(c).findMany(); + // No `__cursorValue`: pagination takes its own column back before the + // row reaches anybody. expect(edges[0]).toEqual({ id: 1, labels: { author: "Ada", category: "News" }, @@ -373,7 +376,14 @@ describe("content service", () => { it("reports a missing label as null", async () => { const { c } = createDbMock( - page([{ id: 1, label__author: null, label__category: "News" }]), + page([ + { + __cursorValue: "2026-01-02 00:00:00", + id: 1, + label__author: null, + label__category: "News", + }, + ]), ); const { edges } = await articles.service(c).findMany(); diff --git a/packages/vitnode/src/content/server/service.ts b/packages/vitnode/src/content/server/service.ts index e5d00a9bd..8feec0465 100644 --- a/packages/vitnode/src/content/server/service.ts +++ b/packages/vitnode/src/content/server/service.ts @@ -743,10 +743,17 @@ export const createContentService = < }, table, where: combined, - query: async ({ limit, orderBy: order, where: rowWhere }) => { + query: async ({ + cursorSelection, + limit, + orderBy: order, + where: rowWhere, + }) => { // One LEFT JOIN per reference field resolves every label in the same - // round trip - there is no per-row lookup anywhere. - const selection: Record = { + // round trip - there is no per-row lookup anywhere. The cursor value + // rides along in the same statement, which is what makes the cursor a + // record of where the row was rather than where it has since moved. + const selection: Record> = { ...ownSelection(), ...Object.fromEntries( Object.entries(references).map(([name, target]) => [ @@ -754,6 +761,9 @@ export const createContentService = < target.labelColumn, ]), ), + // Last, so a content field can never shadow it and leave the page + // with no way to mint a cursor. + ...cursorSelection, }; let builder = c.get("db").select(selection).from(table).$dynamic(); diff --git a/plugins/blog/src/api/modules/categories/routes/get.route.ts b/plugins/blog/src/api/modules/categories/routes/get.route.ts index f243a6a7c..189654ec6 100644 --- a/plugins/blog/src/api/modules/categories/routes/get.route.ts +++ b/plugins/blog/src/api/modules/categories/routes/get.route.ts @@ -7,7 +7,14 @@ import { } from "@vitnode/core/api/lib/with-pagination"; import { core_languages_words } from "@vitnode/core/database/languages"; import { multiLangValueSchema } from "@vitnode/core/lib/helpers/multi-lang"; -import { and, eq, ilike, inArray, type SQL } from "drizzle-orm"; +import { + and, + eq, + getTableColumns, + ilike, + inArray, + type SQL, +} from "drizzle-orm"; import { CONFIG_PLUGIN } from "@/const"; import { blog_categories } from "@/database/categories"; @@ -65,7 +72,7 @@ export const categoriesRoute = buildRoute({ query, }, primaryCursor: blog_categories.id, - query: async ({ limit, where, orderBy }) => { + query: async ({ cursorSelection, limit, where, orderBy }) => { // The title lives in `core_languages_words`, so search resolves matching // category ids from there rather than a column on `blog_categories`. const searchCondition = query.search @@ -99,7 +106,7 @@ export const categoriesRoute = buildRoute({ return await c .get("db") - .select() + .select({ ...getTableColumns(blog_categories), ...cursorSelection }) .from(blog_categories) .where(combinedWhere) .orderBy(orderBy) diff --git a/plugins/blog/src/api/modules/posts/routes/get.route.ts b/plugins/blog/src/api/modules/posts/routes/get.route.ts index 0d98310a5..a50564630 100644 --- a/plugins/blog/src/api/modules/posts/routes/get.route.ts +++ b/plugins/blog/src/api/modules/posts/routes/get.route.ts @@ -77,10 +77,11 @@ export const postsRoute = buildRoute({ query, }, primaryCursor: blog_posts.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") .select({ + ...cursorSelection, id: blog_posts.id, categoryId: blog_posts.categoryId, createdAt: blog_posts.createdAt, diff --git a/plugins/example/src/database/pagination-postgres.test.ts b/plugins/example/src/database/pagination-postgres.test.ts index f1b5fda74..68794fedb 100644 --- a/plugins/example/src/database/pagination-postgres.test.ts +++ b/plugins/example/src/database/pagination-postgres.test.ts @@ -1,5 +1,6 @@ import type { Context } from "hono"; +import { withPagination } from "@vitnode/core/api/lib/with-pagination"; import { HTTPException } from "hono/http-exception"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; @@ -7,7 +8,7 @@ import { CONFIG_PLUGIN } from "@/const"; import type { ContentTestHarness } from "./harness"; -import { articleContent } from "./articles"; +import { articleContent, example_articles } from "./articles"; import { clearContentTables, createContentTestHarness, @@ -438,6 +439,129 @@ describe.skipIf(!DATABASE_TEST_URL)( const seen = await walkFrom(page.pageInfo.endCursor ?? undefined); expect(seen).toHaveLength(5); }); + + /** + * The window between choosing the rows and describing where they were. + * + * The cursor value used to be fetched by a *second* statement, after the + * page rows had already come back - which is a time-of-check / + * time-of-use gap wide enough for another writer to walk through. The row + * was chosen at one position and handed back as a cursor pointing at + * another, so the next page started somewhere the reader had never been + * and everything in between was gone. + * + * There is no window now: the value is projected by the page query + * itself. These tests prove that by mutating the boundary row in exactly + * that gap - after the rows are in hand, before the cursor is minted - + * and showing the cursor does not notice. + */ + describe("while the page is being turned into cursors", () => { + const decode = (cursor: null | string | undefined) => + JSON.parse( + Buffer.from(cursor ?? "", "base64url").toString("utf8"), + ) as { id: number; value: null | string }; + + /** + * One page, with a mutation spliced into the mint-time gap. + * + * `withPagination` is driven directly because the gap is inside it: + * the callback is what fetched the rows, so running the mutation on the + * way out of it lands precisely between the page query and the cursor. + */ + const pageWithRace = async ( + mutate: (boundaryId: number) => Promise, + ) => + await withPagination({ + c: h.context, + orderBy: { column: example_articles.updatedAt, order: "asc" }, + params: { query: { first: "1" } }, + primaryCursor: example_articles.id, + query: async ({ cursorSelection, limit, orderBy, where }) => { + const rows = await h.db + .select({ id: example_articles.id, ...cursorSelection }) + .from(example_articles) + .where(where) + .orderBy(orderBy) + .limit(typeof limit === "number" ? limit : 2); + + const boundary = rows[0]; + if (boundary) await mutate(boundary.id); + + return rows; + }, + table: example_articles, + }); + + it("mints the position the row had, not the one it was given meanwhile", async () => { + const ids = await seed(LADDER); + + const page = await pageWithRace( + async id => await moveTo(id, "2026-08-09T14:00:00"), + ); + + expect(page.edges[0].id).toBe(ids[0]); + // The row now says 14:00. The cursor still says 10:00, because 10:00 + // is where the row was when it ended this page. + expect(decode(page.pageInfo.endCursor).value).toBe( + "2026-08-09 10:00:00", + ); + + // And the consequence that matters: 10:01, 11:00, 12:00 and 13:00 are + // all still reachable. A cursor carrying 14:00 would have skipped + // every one of them, permanently and without an error. + const seen = await walkFrom(page.pageInfo.endCursor ?? undefined); + for (const id of ids.slice(1)) expect(seen).toContain(id); + }); + + it("mints a real position when the row is deleted meanwhile", async () => { + // The worse half of the old race. A second lookup found nothing, and + // "nothing" became `null` - which for a nullable ordering is not the + // absence of a position but a real one, inside the null block. The + // walk jumped there and abandoned the rest of the collection. + const ids = await seed(LADDER); + + const page = await pageWithRace(async id => { + await h.sql`DELETE FROM "example_articles" WHERE "id" = ${id}`; + }); + + expect(decode(page.pageInfo.endCursor).value).toBe( + "2026-08-09 10:00:00", + ); + + const seen = await walkFrom(page.pageInfo.endCursor ?? undefined); + expect(seen.sort((a, b) => a - b)).toEqual(ids.slice(1)); + }); + + it("reads its cursor value out of the page query, not a second one", async () => { + // The structural regression assertion. There is nothing to race with + // if there is no second statement, so this is the property to guard + // rather than the symptom. + await seed(LADDER); + + const list = async () => + await service(h.counted.context).findMany({ + orderBy: { column: "updatedAt" as never, order: "asc" }, + query: { first: "2" }, + }); + + // Warmed first: `postgres` prepares a statement the first time it + // sees its shape, so a cold call issues messages a warm one does not. + await list(); + h.counted.reset(); + await list(); + + const selects = h.counted.queries.filter(query => + /^\s*select/i.test(query), + ); + + // Two, and only two: the total count, and the page. A third would be + // the boundary lookup coming back. + expect(selects).toHaveLength(2); + // The page carries the cursor value with it, at the database's own + // precision. + expect(selects.some(query => query.includes("::text"))).toBe(true); + }); + }); }); // ------------------------------------------------------------------------- @@ -737,6 +861,26 @@ describe.skipIf(!DATABASE_TEST_URL)( ).toMatchObject({ column: "title" }); }); + it("keeps its own projected column out of every row it returns", async () => { + // The page query selects the cursor value so it can be minted from the + // same statement. That column is pagination's business: it is taken back + // before a row reaches a handler, so it cannot reach a response, a + // schema, a search document or a revision snapshot either. + await seed(NON_MONOTONIC); + const publicService = articleContent.publicService; + if (!publicService) throw new Error("no public service"); + + const admin = await service().findMany({ query: { first: "3" } }); + const anonymous = await publicService(h.context).findMany({ + query: { first: "3" }, + }); + + expect(admin.edges.length).toBeGreaterThan(0); + for (const row of [...admin.edges, ...anonymous.edges]) { + expect(Object.keys(row)).not.toContain("__cursorValue"); + } + }); + // ------------------------------------------------------------------------- // Validation // ------------------------------------------------------------------------- @@ -798,6 +942,84 @@ describe.skipIf(!DATABASE_TEST_URL)( }, ); + /** + * The values a pattern lets through and Postgres does not. + * + * `2026-02-30` has the shape of a timestamp and is not a day, so a shape + * check passes it straight into `'2026-02-30'::timestamp` - and the + * answer to that is `invalid input syntax`, arriving at a client as a 500 + * from a route whose contract says it does not do that. Each of these is + * refused before anything is bound. + */ + it.each([ + ["month 13", "2026-13-01"], + ["month 0", "2026-00-01"], + ["30 February", "2026-02-30"], + ["29 February in a common year", "2025-02-29"], + ["31 April", "2026-04-31"], + ["day 32", "2026-01-32"], + ["hour 24", "2026-08-09 24:00:00"], + ["minute 60", "2026-08-09 23:60:00"], + ["second 61", "2026-08-09 23:59:61"], + ["an impossible offset", "2026-08-09 10:00:00+25:00"], + ["an offset with 99 minutes", "2026-08-09 10:00:00+12:99"], + ])( + "answers 400 for a cursor holding %s, which Postgres would refuse", + async (_why, value) => { + await seed(NON_MONOTONIC); + + try { + await service().findMany({ + orderBy: { column: "updatedAt" as never, order: "asc" }, + query: { cursor: tampered(value), first: "5" }, + }); + } catch (error) { + expect(error).toBeInstanceOf(HTTPException); + expect(statusOf(error)).toBe(400); + // Specifically not a Postgres error wearing a different hat. + expect((error as Error).message).not.toMatch( + /invalid input syntax/i, + ); + + return; + } + + throw new Error(`Expected ${value} to be refused.`); + }, + ); + + it("still answers a leap day, which is a real one", async () => { + // The other half of the check: refusing impossible values must not + // refuse possible ones. 2024 is a leap year and 2024-02-29 exists. + const ids = await seed([ + { + code: "leap", + title: "Leap", + updatedAt: new Date("2024-02-29T10:00:00Z"), + }, + { + code: "after", + title: "After", + updatedAt: new Date("2024-03-01T10:00:00Z"), + }, + ]); + + const cursor = Buffer.from( + JSON.stringify({ + column: "updatedAt", + id: ids[0], + value: "2024-02-29 10:00:00", + }), + ).toString("base64url"); + + const page = await service().findMany({ + orderBy: { column: "updatedAt" as never, order: "asc" }, + query: { cursor, first: "5" }, + }); + + expect(page.edges.map(row => row.id)).toEqual([ids[1]]); + }); + it("leaves the table alone when a cursor tries to inject SQL", async () => { await seed(NON_MONOTONIC); diff --git a/plugins/example/src/database/performance-postgres.test.ts b/plugins/example/src/database/performance-postgres.test.ts index cf573dd0e..39acf3afc 100644 --- a/plugins/example/src/database/performance-postgres.test.ts +++ b/plugins/example/src/database/performance-postgres.test.ts @@ -284,6 +284,11 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { * move the exact count by one without anything being wrong. What must never * move is the *shape*: a page of 25 and a page of 100 cost the same number * of round trips, and that is what an N+1 would break. + * + * The bounds came down by one when the cursor value moved into the page + * query's own projection. There is no boundary-row lookup left to pay for - + * and that saving is the same change that closed the window another writer + * could edit the boundary through. */ const boundedAcrossPageSizes = async ( run: (size: number) => Promise, @@ -307,7 +312,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { await articleContent .service(h.counted.context) .findMany({ query: { first: String(size) } }), - 4, + 3, ); }); @@ -321,7 +326,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { await service(h.counted.context).findMany({ query: { first: String(size) }, }), - 4, + 3, ); }); @@ -363,7 +368,7 @@ describe.skipIf(!DATABASE_TEST_URL)("Content Engine at scale", () => { locale: "pl", query: { first: String(size) }, }), - 6, + 5, ); }); From 1682f8b0e20b43a8e659e0c9cacac22ae53631d7 Mon Sep 17 00:00:00 2001 From: aXenDeveloper Date: Sun, 9 Aug 2026 16:41:38 +0200 Subject: [PATCH 14/14] docs(pagination): state the temporal cursor domain as a limit, not an inability "Two forms this repository does not mint" was wrong. Cursor values are read straight from `orderColumn::text`, so a row genuinely holding a BC-era or out-of-range timestamp would mint one the next request then refuses. The restriction is a deliberate limit on the accepted grammar - AD years 1 through 294276 - and it now reads that way. Co-Authored-By: Claude Opus 5 (1M context) --- .../content-engine/performance-and-scaling.mdx | 15 +++++++++++++-- .../vitnode/src/api/lib/pagination-cursor.test.ts | 5 ++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx index a2e5d85b9..5006f6a04 100644 --- a/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx +++ b/apps/docs/content/docs/dev/content-engine/performance-and-scaling.mdx @@ -192,8 +192,19 @@ Postgres in one place - `24:00:00` is a valid input to Postgres and something it own `::text` never writes, so a cursor carrying one did not come from a row. The original string is what gets bound; nothing is reformatted, so microseconds -survive validation digit for digit. Two forms this repository does not mint are -not accepted either: an era suffix (`… BC`) and a year past 294276. +survive validation digit for digit. + + +Temporal cursor pagination intentionally supports **AD years 1 through 294276** +and does not support PostgreSQL BC-era representations. Values outside that +domain are rejected with `400`. + +This is a deliberate limit on the accepted grammar, not an inability to produce +one: cursor values are read straight from `orderColumn::text`, so a row genuinely +holding a BC-era or out-of-range timestamp would mint a cursor the next request +then refuses. Narrowing the domain keeps validation something that can be read +and reasoned about; widening it is a decision, not a fix. + **What is guaranteed:** diff --git a/packages/vitnode/src/api/lib/pagination-cursor.test.ts b/packages/vitnode/src/api/lib/pagination-cursor.test.ts index 8ec3404df..b475575d5 100644 --- a/packages/vitnode/src/api/lib/pagination-cursor.test.ts +++ b/packages/vitnode/src/api/lib/pagination-cursor.test.ts @@ -368,7 +368,10 @@ describe("a tampered cursor value is refused, never coerced", () => { ["an offset that is not a number", "2026-08-09 10:00:00+ab"], ["a seven-digit fraction", "2026-08-09 10:00:00.1234567"], ["trailing rubbish", "2026-08-09 10:00:00 OR 1=1"], - ["an era suffix, which nothing here mints", "2026-08-09 BC"], + [ + "an era suffix, which is outside the supported domain", + "2026-08-09 BC", + ], ])("refuses %s", (_why, value) => { refuses(core_users.createdAt, value); });