From 18013d23fefec6961300edc23e2f9dc7ad3c5cb7 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:22:51 -0400 Subject: [PATCH 01/55] feat(release-tracks): add STIX bundle output format for snapshot exports Implement format=bundle on the snapshot retrieval endpoints (GET /api/release-tracks/:id and .../snapshots/:modified): - include (staged/candidates) hydrates additional tiers alongside members; state (work-in-progress/awaiting-review) narrows them, with reviewed entries always included - stixVersion (2.0|2.1, default 2.1) conforms objects via the shared lib/stix-conformance.js helpers (extracted from stix-bundles-service); the bundle envelope carries spec_version only for STIX 2.0 per spec - includeToc (default true) prepends an x-mitre-collection table of contents derived from the release-track metadata - bundles are self-contained (referenced identities and marking definitions included), LinkById tags are converted to citations, and notes are never emitted Rework the ephemeral endpoint (GET /api/release-tracks/ephemeral/:domain) to delegate bundle generation to stix-bundles-service, preserving the legacy object-selection logic (secondary objects, referential integrity, STIX conformance) with a simplified parameter surface: includeToc, includeObjectsWithMissingAttackId, includeDeprecated, includeRevoked, stixVersion. The legacy GET /api/stix-bundles endpoint is marked deprecated in the OpenAPI spec. Add regression tests (running with ADM validation enabled), update the OpenAPI spec, and document the behavior in the user docs and in docs/developer/release-tracks/bundle-export.md. --- .../paths/release-tracks-paths.yml | 166 +++++++- .../definitions/paths/stix-bundles-paths.yml | 7 +- app/controllers/release-tracks-controller.js | 86 +++- app/lib/release-tracks/export-schemas.js | 111 ++++- .../release-tracks/release-track-schemas.js | 39 ++ app/lib/stix-conformance.js | 59 +++ .../release-tracks/ephemeral-service.js | 156 ++++--- app/services/release-tracks/export-service.js | 161 ++++++- .../release-tracks/release-tracks-service.js | 6 +- app/services/stix/stix-bundles-service.js | 32 +- .../release-tracks/ephemeral-bundle.spec.js | 293 +++++++++++++ .../release-tracks-bundle.spec.js | 394 ++++++++++++++++++ .../api/release-tracks/release-tracks.spec.js | 6 +- .../developer/release-tracks/bundle-export.md | 152 +++++++ docs/user/release-tracks/api-reference.md | 66 ++- docs/user/release-tracks/output-formats.md | 55 ++- 16 files changed, 1647 insertions(+), 142 deletions(-) create mode 100644 app/lib/stix-conformance.js create mode 100644 app/tests/api/release-tracks/ephemeral-bundle.spec.js create mode 100644 app/tests/api/release-tracks/release-tracks-bundle.spec.js create mode 100644 docs/developer/release-tracks/bundle-export.md diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 88974eb7..0999f816 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -8,17 +8,22 @@ paths: operationId: 'release-tracks-ephemeral-get' description: | Generate a stateless bundle containing all objects from a given ATT&CK domain. - This endpoint queries all STIX repositories by domain without persisting a release track. + This endpoint queries the database by domain without persisting a release track. + It supplants the deprecated GET /api/stix-bundles endpoint. tags: - 'Release Tracks' parameters: - name: domain in: path required: true - description: 'ATT&CK domain (e.g., enterprise-attack, mobile-attack, ics-attack)' + description: 'ATT&CK domain' schema: type: string - example: 'enterprise-attack' + enum: + - enterprise + - ics + - mobile + example: 'enterprise' - name: format in: query description: 'Output format. filesystemstore is not yet implemented.' @@ -29,6 +34,48 @@ paths: - workbench - filesystemstore default: bundle + - name: stixVersion + in: query + description: | + STIX version that the exported bundle should conform to (bundle format only). + schema: + type: string + enum: + - '2.0' + - '2.1' + default: '2.1' + - name: includeToc + in: query + description: | + Whether to include a table-of-contents object (of type `x-mitre-collection`) + in the bundle (bundle format only). + schema: + type: boolean + default: true + - name: includeObjectsWithMissingAttackId + in: query + description: | + Whether to include objects that should have an ATT&CK ID set but do not + (bundle format only). + schema: + type: boolean + default: false + - name: includeDeprecated + in: query + description: | + Whether to include objects that have the `x_mitre_deprecated` property set to true + (bundle format only). + schema: + type: boolean + default: false + - name: includeRevoked + in: query + description: | + Whether to include objects that have the `revoked` property set to true + (bundle format only). + schema: + type: boolean + default: false responses: '200': description: 'Ephemeral bundle generated successfully' @@ -156,6 +203,7 @@ paths: Retrieve the most recent snapshot for a release track. By default returns the Workbench snapshot shape with all tiers present. Use the include query parameter to narrow tier arrays when desired. + Use format=bundle to export the snapshot as a STIX bundle. tags: - 'Release Tracks' parameters: @@ -168,16 +216,20 @@ paths: example: 'release-track--a1b2c3d4-e5f6-7890-abcd-ef1234567890' - name: include in: query - description: 'Which tiers to include in response' - schema: - type: string - enum: - - members - - staged - - candidates - - quarantine - - all - default: all + description: | + Format-sensitive tier selector. + For format=workbench (default): a single value controlling which tier arrays + are returned — members | staged | candidates | quarantine | all (default: all). + For format=bundle: a list of additional tiers (staged and/or candidates, + comma-separated or repeated) to include alongside members. If omitted, only + members are included in the bundle. + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string - name: format in: query description: 'Output format. filesystemstore is not yet implemented.' @@ -188,6 +240,38 @@ paths: - workbench - filesystemstore default: workbench + - name: state + in: query + description: | + Workflow-status filter for the staged/candidate tiers selected via include + (bundle format only). Accepts work-in-progress and/or awaiting-review + (comma-separated or repeated). Entries marked reviewed are always included, + irrespective of this parameter. Members are unaffected. + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: stixVersion + in: query + description: | + STIX version that the exported bundle should conform to (bundle format only). + schema: + type: string + enum: + - '2.0' + - '2.1' + default: '2.1' + - name: includeToc + in: query + description: | + Whether to include a table-of-contents object (of type `x-mitre-collection`) + derived from the release-track metadata (bundle format only). + schema: + type: boolean + default: true responses: '200': description: 'Latest snapshot retrieved successfully' @@ -711,6 +795,7 @@ paths: operationId: 'release-tracks-snapshot-get' description: | Retrieve a historical snapshot identified by its modified timestamp. + Use format=bundle to export the snapshot as a STIX bundle. tags: - 'Release Tracks' parameters: @@ -727,15 +812,20 @@ paths: type: string - name: include in: query - schema: - type: string - enum: - - members - - staged - - candidates - - quarantine - - all - default: all + description: | + Format-sensitive tier selector. + For format=workbench (default): a single value controlling which tier arrays + are returned — members | staged | candidates | quarantine | all (default: all). + For format=bundle: a list of additional tiers (staged and/or candidates, + comma-separated or repeated) to include alongside members. If omitted, only + members are included in the bundle. + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string - name: format in: query description: 'Output format. filesystemstore is not yet implemented.' @@ -746,6 +836,38 @@ paths: - filesystemstore - workbench default: workbench + - name: state + in: query + description: | + Workflow-status filter for the staged/candidate tiers selected via include + (bundle format only). Accepts work-in-progress and/or awaiting-review + (comma-separated or repeated). Entries marked reviewed are always included, + irrespective of this parameter. Members are unaffected. + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: stixVersion + in: query + description: | + STIX version that the exported bundle should conform to (bundle format only). + schema: + type: string + enum: + - '2.0' + - '2.1' + default: '2.1' + - name: includeToc + in: query + description: | + Whether to include a table-of-contents object (of type `x-mitre-collection`) + derived from the release-track metadata (bundle format only). + schema: + type: boolean + default: true responses: '200': description: 'Snapshot retrieved successfully' diff --git a/app/api/definitions/paths/stix-bundles-paths.yml b/app/api/definitions/paths/stix-bundles-paths.yml index a95edecd..e22b6e07 100644 --- a/app/api/definitions/paths/stix-bundles-paths.yml +++ b/app/api/definitions/paths/stix-bundles-paths.yml @@ -1,9 +1,14 @@ paths: /api/stix-bundles: get: - summary: 'Export a stix bundle' + summary: 'Export a stix bundle (deprecated)' operationId: 'stix-bundle-export' + deprecated: true description: | + **Deprecated.** Use `GET /api/release-tracks/ephemeral/{domain}` for domain-scoped + bundles, or `GET /api/release-tracks/{id}?format=bundle` for release-track snapshot + bundles. This endpoint will be removed in a future release. + This endpoint exports a STIX bundle and returns the bundle. This endpoint is distinguished from exporting a collection bundle by being based on a selected domain, instead of a collection object. Also, the returned STIX bundle will not contain a collection object. diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 9f172ca7..547b99da 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -23,6 +23,10 @@ const { domainParamSchema, formatQuerySchema, includeQuerySchema, + bundleIncludeQuerySchema, + bundleStateQuerySchema, + stixVersionQuerySchema, + booleanQuerySchema, trackTypeQuerySchema, workflowStatusSchema, createTrackBodySchema, @@ -77,17 +81,57 @@ function rejectFilesystemStoreFormat(format, methodName) { /** * Parse common query parameters shared across GET snapshot endpoints. + * + * The `include` parameter is format-sensitive: + * - format=workbench: single tier name ('members' | 'staged' | 'candidates' + * | 'quarantine' | 'all') controlling which tier arrays are returned + * - format=bundle: list of additional tiers ('staged' and/or 'candidates') + * to hydrate into the bundle alongside members. Omitted → members only. + * + * The `state`, `stixVersion`, and `includeToc` parameters only apply to + * format=bundle. */ function parseSnapshotQueryParams(query) { - return { - format: parseOptionalQueryStrict(query.format, formatQuerySchema, 'workbench', 'format'), - include: parseOptionalQuery(query.include, includeQuerySchema, undefined), + const format = parseOptionalQueryStrict(query.format, formatQuerySchema, 'workbench', 'format'); + + const common = { + format, releases: query.releases === 'only' ? 'only' : undefined, version: parseOptionalQuery(query.version, xMitreVersionSchema, undefined), versions: query.versions === 'all' ? 'all' : undefined, limit: query.limit ? parseInt(query.limit, 10) : undefined, offset: query.offset ? parseInt(query.offset, 10) : undefined, }; + + if (format === 'bundle') { + return { + ...common, + include: parseOptionalQueryStrict( + query.include, + bundleIncludeQuerySchema, + undefined, + 'include', + ), + state: parseOptionalQueryStrict(query.state, bundleStateQuerySchema, undefined, 'state'), + stixVersion: parseOptionalQueryStrict( + query.stixVersion, + stixVersionQuerySchema, + '2.1', + 'stixVersion', + ), + includeToc: parseOptionalQueryStrict( + query.includeToc, + booleanQuerySchema, + true, + 'includeToc', + ), + }; + } + + return { + ...common, + include: parseOptionalQueryStrict(query.include, includeQuerySchema, undefined, 'include'), + }; } // ============================================================================= @@ -118,7 +162,41 @@ exports.retrieveEphemeralByDomain = async function retrieveEphemeralByDomain(req return next(formatError); } - const result = await releaseTracksService.getEphemeralBundle(domainResult.data, format); + const options = { + format, + stixVersion: parseOptionalQueryStrict( + req.query.stixVersion, + stixVersionQuerySchema, + '2.1', + 'stixVersion', + ), + includeToc: parseOptionalQueryStrict( + req.query.includeToc, + booleanQuerySchema, + true, + 'includeToc', + ), + includeObjectsWithMissingAttackId: parseOptionalQueryStrict( + req.query.includeObjectsWithMissingAttackId, + booleanQuerySchema, + false, + 'includeObjectsWithMissingAttackId', + ), + includeDeprecated: parseOptionalQueryStrict( + req.query.includeDeprecated, + booleanQuerySchema, + false, + 'includeDeprecated', + ), + includeRevoked: parseOptionalQueryStrict( + req.query.includeRevoked, + booleanQuerySchema, + false, + 'includeRevoked', + ), + }; + + const result = await releaseTracksService.getEphemeralBundle(domainResult.data, options); logger.debug(`Success: Retrieved ephemeral ${domainResult.data} bundle`); return res.status(200).send(result); } catch (err) { diff --git a/app/lib/release-tracks/export-schemas.js b/app/lib/release-tracks/export-schemas.js index f67be824..ae1a6eab 100644 --- a/app/lib/release-tracks/export-schemas.js +++ b/app/lib/release-tracks/export-schemas.js @@ -16,6 +16,7 @@ const { z } = require('zod'); const uuid = require('uuid'); +const { conformToStixVersion } = require('../stix-conformance'); // ----------------------------------------------------------------------------- // Shared sub-schemas @@ -33,6 +34,10 @@ const snapshotSchema = z.looseObject({ id: z.string(), version: z.string().nullable().optional(), name: z.string(), + description: z.string().optional(), + created: z.date().or(z.string()).optional(), + created_by_ref: z.string().optional(), + object_marking_refs: z.array(z.string()).optional(), modified: z.date().or(z.string()), members: z.array(tierEntrySchema).default([]), staged: z.array(tierEntrySchema).optional(), @@ -45,8 +50,12 @@ const hydratedObjectSchema = z.looseObject({ }); const exportOptionsSchema = z - .object({ - include: z.enum(['staged', 'candidates', 'all']).optional(), + .looseObject({ + include: z.array(z.enum(['staged', 'candidates'])).optional(), + state: z.array(z.enum(['work-in-progress', 'awaiting-review'])).optional(), + stixVersion: z.enum(['2.0', '2.1']).default('2.1'), + includeToc: z.boolean().default(true), + attackSpecVersion: z.string().optional(), }) .optional() .default({}); @@ -79,18 +88,99 @@ function buildTierLookup(snapshot) { return lookup; } +// ----------------------------------------------------------------------------- +// Helper: Build the x-mitre-collection table-of-contents (TOC) object +// +// The x-mitre-collection object is effectively a table of contents for the +// bundle. For release-track exports it is derived from the track/snapshot +// metadata rather than from user-supplied query parameters: +// - id: stable per track (reuses the track UUID) +// - x_mitre_version: the snapshot's tagged version, or '0.1' for drafts +// - modified: the snapshot's modified timestamp +// - x_mitre_contents: every bundle object except marking definitions, +// which are recorded in object_marking_refs instead +// ----------------------------------------------------------------------------- + +function buildTocObject(snapshot, bundleObjects, options) { + const trackUuid = snapshot.id.split('--')[1]; + + const tocObject = { + type: 'x-mitre-collection', + id: `x-mitre-collection--${trackUuid}`, + x_mitre_attack_spec_version: options.attackSpecVersion, + name: snapshot.name, + x_mitre_version: snapshot.version || '0.1', + description: snapshot.description, + created_by_ref: snapshot.created_by_ref || '', + created: snapshot.created || snapshot.modified, + modified: snapshot.modified, + x_mitre_contents: [], + object_marking_refs: [], + }; + + for (const bundleObject of bundleObjects) { + if (bundleObject.type === 'marking-definition') { + tocObject.object_marking_refs.push(bundleObject.id); + } else { + tocObject.x_mitre_contents.push({ + object_ref: bundleObject.id, + object_modified: bundleObject.modified, + }); + } + } + + if (options.stixVersion === '2.1') { + tocObject.spec_version = '2.1'; + } + + // Sort x_mitre_contents by id for deterministic output + tocObject.x_mitre_contents.sort((x, y) => x.object_ref.localeCompare(y.object_ref)); + + return tocObject; +} + // ----------------------------------------------------------------------------- // Bundle Transform Schema // -// Standard STIX 2.1 bundle format. Only includes `stix` properties - no -// workspace data or workflow metadata. Suitable for external publication. +// Standard STIX bundle format. Only includes `stix` properties - no workspace +// data or workflow metadata. Suitable for external publication. +// +// Options: +// - stixVersion ('2.0' | '2.1', default '2.1'): each object is conformed to +// the requested STIX version. The bundle envelope carries spec_version +// only for STIX 2.0 — the STIX 2.1 specification removed spec_version +// from the bundle object (objects declare their own spec_version). +// - includeToc (default true): prepend an x-mitre-collection object derived +// from the snapshot metadata +// - attackSpecVersion: x_mitre_attack_spec_version for the TOC object +// +// Notes are Workbench-native objects, not STIX objects, so they are never +// included in emitted bundles. // ----------------------------------------------------------------------------- -const bundleTransformSchema = exportInputSchema.transform((input) => ({ - type: 'bundle', - id: `bundle--${uuid.v4()}`, - objects: input.hydratedObjects.map((doc) => doc.stix), -})); +const bundleTransformSchema = exportInputSchema.transform((input) => { + const { stixVersion, includeToc, attackSpecVersion } = input.options; + + const objects = input.hydratedObjects + .map((doc) => doc.stix) + .filter((stixObject) => stixObject.type !== 'note'); + + for (const stixObject of objects) { + conformToStixVersion(stixObject, stixVersion); + } + + if (includeToc) { + objects.unshift(buildTocObject(input.snapshot, objects, { stixVersion, attackSpecVersion })); + } + + return { + type: 'bundle', + id: `bundle--${uuid.v4()}`, + // STIX 2.0 bundles must declare spec_version; STIX 2.1 bundles must not + ...(stixVersion === '2.0' ? { spec_version: '2.0' } : {}), + objects, + }; +}); // ----------------------------------------------------------------------------- // Workbench Transform Schema @@ -174,6 +264,7 @@ module.exports = { workbenchTransformSchema, filesystemStoreTransformSchema, - // Helper (exported for testing) + // Helpers (exported for testing) buildTierLookup, + buildTocObject, }; diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index c4c52f3d..fd070454 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -154,6 +154,41 @@ const formatQuerySchema = z.enum(['bundle', 'filesystemstore', 'workbench']); const includeQuerySchema = z.enum(['members', 'staged', 'candidates', 'quarantine', 'all']); +/** + * Normalize a query-string value that represents a list. Accepts a repeated + * parameter (array), a comma-separated string, or a single value, and returns + * an array of trimmed strings. + */ +function normalizeQueryArray(value) { + const rawValues = Array.isArray(value) ? value : [value]; + return rawValues + .flatMap((entry) => String(entry).split(',')) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); +} + +// `include` for format=bundle: which non-member tiers to add to the bundle. +// Accepts singular or plural tier names; normalized to the plural tier names. +const bundleIncludeQuerySchema = z.preprocess( + (value) => + normalizeQueryArray(value).map((entry) => (entry === 'candidate' ? 'candidates' : entry)), + z.array(z.enum(['candidates', 'staged'])).min(1), +); + +// `state` for format=bundle: workflow-status filter applied to the tiers +// selected via `include`. 'reviewed' is intentionally not a valid filter +// value — reviewed objects are always included. +const bundleStateQuerySchema = z.preprocess( + (value) => normalizeQueryArray(value), + z.array(z.enum(['work-in-progress', 'awaiting-review'])).min(1), +); + +const stixVersionQuerySchema = z.enum(['2.0', '2.1']); + +// Boolean query parameters arrive as strings ('true'/'false') unless the +// OpenAPI validator has already coerced them to booleans. +const booleanQuerySchema = z.union([z.boolean(), z.stringbool()]); + const trackTypeQuerySchema = z.enum(['standard', 'virtual']); const bumpTypeSchema = z.enum(['major', 'minor']); @@ -375,6 +410,10 @@ module.exports = { domainParamSchema, formatQuerySchema, includeQuerySchema, + bundleIncludeQuerySchema, + bundleStateQuerySchema, + stixVersionQuerySchema, + booleanQuerySchema, trackTypeQuerySchema, bumpTypeSchema, workflowStatusSchema, diff --git a/app/lib/stix-conformance.js b/app/lib/stix-conformance.js new file mode 100644 index 00000000..21370e9c --- /dev/null +++ b/app/lib/stix-conformance.js @@ -0,0 +1,59 @@ +'use strict'; + +// ============================================================================= +// STIX version conformance helpers. +// +// Shared by the legacy stix-bundles-service and the release-tracks export +// pipeline so that every emitted bundle applies identical version rules: +// - STIX 2.0: objects must not have spec_version; malware/tool need labels +// - STIX 2.1: objects must have spec_version; labels are dropped except on +// course-of-action objects +// ============================================================================= + +/** + * Removes empty array properties from a STIX object. + * @param {Object} stixObject - The STIX object to clean + */ +function removeEmptyArrays(stixObject) { + for (const propertyName of Object.keys(stixObject)) { + if (Array.isArray(stixObject[propertyName]) && stixObject[propertyName].length === 0) { + delete stixObject[propertyName]; + } + } +} + +/** + * Modifies a STIX object in place to conform to the specified STIX version + * ('2.0' or '2.1'). Handles version-specific requirements for various object + * types. + * @param {Object} stixObject - The STIX object to modify + * @param {string} stixVersion - Target STIX version ('2.0' or '2.1') + */ +function conformToStixVersion(stixObject, stixVersion) { + if (stixVersion === '2.0') { + // Remove STIX 2.1 specific properties + delete stixObject.spec_version; + + // Handle malware and tool specific requirements + if (stixObject.type === 'malware') { + delete stixObject.is_family; + stixObject.labels = ['malware']; + } + + if (stixObject.type === 'tool') { + stixObject.labels = ['tool']; + } + } else if (stixVersion === '2.1') { + stixObject.spec_version = '2.1'; + if (stixObject.type != 'course-of-action') { + delete stixObject.labels; + } + } + + removeEmptyArrays(stixObject); +} + +module.exports = { + removeEmptyArrays, + conformToStixVersion, +}; diff --git a/app/services/release-tracks/ephemeral-service.js b/app/services/release-tracks/ephemeral-service.js index f551cdfb..d3e7afa5 100644 --- a/app/services/release-tracks/ephemeral-service.js +++ b/app/services/release-tracks/ephemeral-service.js @@ -5,18 +5,26 @@ // // Generates stateless, non-persisted STIX bundles for a given ATT&CK domain. // Unlike regular release tracks (which store snapshots with object refs), -// ephemeral bundles are computed on-the-fly by querying all STIX repositories -// for objects belonging to the requested domain. +// ephemeral bundles are computed on-the-fly by querying the database for +// objects belonging to the requested domain. // // This service performs cross-service READS (permitted by the event-driven // architecture — see docs/CROSS_SERVICE_READS_PATTERN.md) by querying STIX // repositories directly. It does NOT write to any repository. // -// The domain query pattern mirrors stix-bundles-service.exportBundle, but -// operates independently of the legacy collection-bundles infrastructure. +// The default 'bundle' format supplants the legacy GET /api/stix-bundles +// endpoint. Bundle generation delegates to stix-bundles-service.exportBundle +// so that all of its object-selection logic is preserved: secondary objects +// (groups, campaigns, detection strategies), relationship referential +// integrity, LinkById citation conversion, STIX version conformance, and +// x-mitre-collection (TOC) generation. See +// docs/developer/release-tracks/bundle-export.md for the parameter mapping. +// +// The 'workbench' format retains the simpler domain-query pipeline below, +// which returns full Workbench documents (stix + workspace). // ============================================================================= -const uuid = require('uuid'); +const config = require('../../config/config'); const logger = require('../../lib/logger'); // --------------------------------------------------------------------------- @@ -124,16 +132,47 @@ async function fetchSupportingObjects(identityIds, markingIds) { /** * Generate an ephemeral STIX bundle for a domain. * - * Queries all domain-aware repositories in parallel for the latest version - * of each object in the given domain, then discovers and includes - * relationships that connect those objects, along with referenced identities - * and marking definitions. + * For the default 'bundle' format, delegates to + * stix-bundles-service.exportBundle with the following parameter mapping + * (this endpoint supplants the deprecated GET /api/stix-bundles endpoint): + * + * - stixVersion: preserved (default '2.1') + * - includeRevoked/includeDeprecated: preserved (default false) + * - includeObjectsWithMissingAttackId: renamed from includeMissingAttackId + * - includeToc: renamed from includeCollectionObject + * (default true) + * - collectionObjectVersion: fixed at '0.1' — signifies that the + * TOC was generated ephemerally and is + * not connected to a release track + * - collectionObjectModified: fixed at the current timestamp + * - collectionAttackSpecVersion: fixed at config.app.attackSpecVersion + * - includeNotes: removed — notes are Workbench-native + * objects, not STIX objects + * - includeDataSources: removed — data sources are deprecated + * or revoked as of ATT&CK v18, so their + * inclusion is governed entirely by + * includeDeprecated/includeRevoked + * - useLegacyMethod: removed + * - state: removed — workflow status is scoped + * to release tracks, and this endpoint + * is domain-scoped + * + * For the 'workbench' format, queries all domain-aware repositories in + * parallel for the latest version of each object in the given domain, then + * discovers and includes relationships that connect those objects, along + * with referenced identities and marking definitions. * * @param {string} domain - One of: 'enterprise', 'ics', 'mobile' - * @param {string} [format='bundle'] - Output format (currently only 'bundle') + * @param {Object} [options] - Output options + * @param {string} [options.format='bundle'] - Output format + * @param {string} [options.stixVersion='2.1'] - STIX version ('2.0' or '2.1') + * @param {boolean} [options.includeToc=true] - Include the x-mitre-collection TOC object + * @param {boolean} [options.includeObjectsWithMissingAttackId=false] - Include objects without ATT&CK IDs + * @param {boolean} [options.includeDeprecated=false] - Include deprecated objects + * @param {boolean} [options.includeRevoked=false] - Include revoked objects * @returns {Promise} A STIX bundle (or formatted output) */ -exports.getEphemeralBundle = async function getEphemeralBundle(domain, format) { +exports.getEphemeralBundle = async function getEphemeralBundle(domain, options = {}) { const attackDomain = DOMAIN_MAP[domain]; if (!attackDomain) { const { BadRequestError } = require('../../exceptions'); @@ -143,6 +182,34 @@ exports.getEphemeralBundle = async function getEphemeralBundle(domain, format) { }); } + const format = options.format || 'bundle'; + + if (format === 'bundle') { + const stixVersion = options.stixVersion || '2.1'; + + // Lazy-load to avoid circular dependency issues at startup + const stixBundlesService = require('../stix/stix-bundles-service'); + const bundle = await stixBundlesService.exportBundle({ + domain: attackDomain, + stixVersion, + includeRevoked: options.includeRevoked === true, + includeDeprecated: options.includeDeprecated === true, + includeMissingAttackId: options.includeObjectsWithMissingAttackId === true, + // Notes are Workbench-native objects, not STIX objects + includeNotes: false, + // Data sources are all deprecated/revoked as of ATT&CK v18; let the + // includeDeprecated/includeRevoked flags govern their inclusion + includeDataSources: true, + includeCollectionObject: options.includeToc !== false, + collectionObjectVersion: '0.1', + collectionObjectModified: new Date().toISOString(), + collectionAttackSpecVersion: config.app.attackSpecVersion, + }); + + logger.verbose(`EphemeralService: Built ephemeral ${stixVersion} bundle for "${attackDomain}"`); + return bundle; + } + const repos = getRepositories(); const queryOptions = { includeRevoked: false, @@ -218,64 +285,41 @@ exports.getEphemeralBundle = async function getEphemeralBundle(domain, format) { const supportingObjects = await fetchSupportingObjects(identityIds, markingIds); // ------------------------------------------------------------------ - // Step 5: Assemble the STIX bundle + // Step 5: Format via export-service with a synthetic snapshot envelope // ------------------------------------------------------------------ - // Deduplicate by stix.id (in case of overlapping supporting objects) + // Deduplicate by stix.id + stix.modified (in case of overlapping + // supporting objects) const seen = new Set(); - const bundleObjects = []; - + const deduped = []; for (const doc of [...primaryObjects, ...relevantRelationships, ...supportingObjects]) { const key = `${doc.stix.id}::${doc.stix.modified}`; if (seen.has(key)) continue; seen.add(key); - bundleObjects.push(doc.stix); + deduped.push(doc); } - const bundle = { - type: 'bundle', - id: `bundle--${uuid.v4()}`, - objects: bundleObjects, - }; - logger.verbose( - `EphemeralService: Built ephemeral bundle for "${attackDomain}" ` + + `EphemeralService: Built ephemeral ${format} snapshot for "${attackDomain}" ` + `(${primaryObjects.length} primary, ${relevantRelationships.length} relationships, ` + - `${supportingObjects.length} supporting → ${bundleObjects.length} total objects)`, + `${supportingObjects.length} supporting → ${deduped.length} total objects)`, ); - // Format conversion (if not plain bundle) - if (format === 'workbench' || format === 'filesystemstore') { - // Re-use export-service formatters with a synthetic snapshot envelope - const exportService = require('./export-service'); - const syntheticDocs = [...primaryObjects, ...relevantRelationships, ...supportingObjects]; - const deduped = []; - const dedupSeen = new Set(); - for (const doc of syntheticDocs) { - const key = `${doc.stix.id}::${doc.stix.modified}`; - if (dedupSeen.has(key)) continue; - dedupSeen.add(key); - deduped.push(doc); - } + const exportService = require('./export-service'); + const syntheticSnapshot = { + id: `ephemeral-${domain}`, + version: null, + name: `${domain} (ephemeral)`, + modified: new Date(), + members: deduped.map((doc) => ({ + object_ref: doc.stix.id, + // Marking definitions have no modified timestamp; fall back to created + object_modified: doc.stix.modified || doc.stix.created, + })), + }; - const syntheticSnapshot = { - id: `ephemeral-${domain}`, - version: null, - name: `${domain} (ephemeral)`, - modified: new Date(), - members: deduped.map((doc) => ({ - object_ref: doc.stix.id, - object_modified: doc.stix.modified, - })), - }; - - if (format === 'workbench') { - return exportService.formatAsWorkbench(syntheticSnapshot, deduped); - } - if (format === 'filesystemstore') { - return exportService.formatAsFilesystemStore(syntheticSnapshot, deduped); - } + if (format === 'filesystemstore') { + return exportService.formatAsFilesystemStore(syntheticSnapshot, deduped); } - - return bundle; + return exportService.formatAsWorkbench(syntheticSnapshot, deduped); }; diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index 3547ce51..f5a2cef4 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -17,8 +17,10 @@ // app/lib/release-tracks/export-schemas.js for schema definitions. // ============================================================================= +const config = require('../../config/config'); const types = require('../../lib/types'); const logger = require('../../lib/logger'); +const linkById = require('../../lib/linkById'); const { bundleTransformSchema, workbenchTransformSchema, @@ -111,18 +113,140 @@ exports.hydrateMembers = async function hydrateMembers(entries) { return hydrated; }; +// ============================================================================= +// Bundle assembly helpers +// ============================================================================= + +/** + * Select the tier entries that belong in a bundle export. + * + * Members are always included. Staged and candidate entries are included only + * when named in `include`. When `state` is provided it further narrows the + * staged/candidate entries to those whose workflow status matches — except + * entries marked 'reviewed', which are always included irrespective of + * `state` (reviewed content is release-ready by definition, mirroring how all + * members are inherently reviewed). + * + * @param {Object} snapshot - The raw snapshot document + * @param {Object} options - { include?: Array<'staged'|'candidates'>, state?: Array } + * @returns {Array<{object_ref: string, object_modified: string|Date}>} Deduplicated entries + */ +function collectBundleEntries(snapshot, options) { + const include = options.include || []; + const state = options.state; + + const filterByState = (entries) => { + if (!state) return entries; + return entries.filter( + (entry) => entry.object_status === 'reviewed' || state.includes(entry.object_status), + ); + }; + + const entries = [...(snapshot.members || [])]; + if (include.includes('staged')) { + entries.push(...filterByState(snapshot.staged || [])); + } + if (include.includes('candidates')) { + entries.push(...filterByState(snapshot.candidates || [])); + } + + // Deduplicate by object_ref + object_modified + const seen = new Set(); + const deduped = []; + for (const entry of entries) { + const key = `${entry.object_ref}::${new Date(entry.object_modified).getTime()}`; + if (seen.has(key)) continue; + seen.add(key); + deduped.push(entry); + } + + return deduped; +} + +/** + * Fetch identities and marking definitions referenced by the hydrated + * documents (via created_by_ref / object_marking_refs) that are not already + * part of the export. Emitted bundles must be self-contained, so referenced + * supporting objects are appended even though they are not tier entries. + * + * @param {Array} documents - Hydrated lean documents ({ stix, ... }) + * @returns {Promise>} Supporting lean documents + */ +async function fetchSupportingObjects(documents) { + const repoMap = getRepositoryMap(); + const existingIds = new Set(documents.map((doc) => doc.stix.id)); + + const identityIds = new Set(); + const markingIds = new Set(); + for (const doc of documents) { + if (doc.stix.created_by_ref && !existingIds.has(doc.stix.created_by_ref)) { + identityIds.add(doc.stix.created_by_ref); + } + for (const ref of doc.stix.object_marking_refs || []) { + if (!existingIds.has(ref)) markingIds.add(ref); + } + } + + const supportingObjects = []; + const fetchLatest = async (repo, stixId, description) => { + try { + const doc = await repo.retrieveLatestByStixIdLean(stixId); + if (doc) supportingObjects.push(doc); + else logger.warn(`ExportService: Referenced ${description} not found: ${stixId}`); + } catch (err) { + logger.warn(`ExportService: Could not fetch ${description} "${stixId}": ${err.message}`); + } + }; + + await Promise.all([ + ...[...identityIds].map((id) => fetchLatest(repoMap[types.Identity], id, 'identity')), + ...[...markingIds].map((id) => + fetchLatest(repoMap[types.MarkingDefinition], id, 'marking definition'), + ), + ]); + + return supportingObjects; +} + +/** + * Convert LinkById tags (e.g. "(LinkById: T1234)") in descriptions to + * markdown citations, preferring objects already in the export before + * falling back to a database lookup. Mirrors the legacy stix-bundles-service + * behavior so bundles emitted from release tracks match published output. + * + * @param {Array} documents - Hydrated lean documents ({ stix, ... }) + */ +async function convertLinkByIdTags(documents) { + const byAttackId = new Map(); + for (const doc of documents) { + const attackId = linkById.getAttackId(doc.stix); + if (attackId) byAttackId.set(attackId, doc); + } + + const getAttackObject = async (attackId) => + byAttackId.get(attackId) || (await linkById.getAttackObjectFromDatabase(attackId)); + + for (const doc of documents) { + await linkById.convertLinkByIdTags(doc.stix, getAttackObject); + } +} + // ============================================================================= // Format helpers (delegating to Zod transform schemas) // ============================================================================= /** - * Format as a standard STIX 2.1 bundle. + * Format as a standard STIX bundle. * * Only includes `stix` properties — no workspace data or workflow metadata. * Transformation logic is defined in export-schemas.js. + * + * @param {Object} snapshot - The raw snapshot document + * @param {Array} hydratedObjects - Hydrated lean documents + * @param {Object} [options] - { stixVersion?, includeToc?, attackSpecVersion? } */ -exports.formatAsBundle = function formatAsBundle(snapshot, hydratedObjects) { - return bundleTransformSchema.parse({ snapshot, hydratedObjects }); +exports.formatAsBundle = function formatAsBundle(snapshot, hydratedObjects, options) { + return bundleTransformSchema.parse({ snapshot, hydratedObjects, options }); }; /** @@ -156,22 +280,41 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd * Workbench snapshot retrieval is handled by release-tracks-service because it * returns the release-track snapshot shape with UI-friendly tier entry details. * + * Bundle exports (see docs/developer/release-tracks/bundle-export.md): + * 1. Select tier entries — members always; staged/candidates via + * options.include, narrowed by options.state + * 2. Hydrate entries into full documents + * 3. Append referenced identities and marking definitions + * 4. Convert LinkById tags to markdown citations + * 5. Assemble the bundle (STIX version conformance + optional TOC) via the + * Zod transform schema + * * @param {Object} snapshot - The raw snapshot document from the dynamic repo * @param {string} format - One of: 'bundle', 'filesystemstore' * @param {Object} [options] - Additional options + * @param {Array} [options.include] - Extra tiers to include in bundles ('staged', 'candidates') + * @param {Array} [options.state] - Workflow status filter for included staged/candidates + * @param {string} [options.stixVersion] - '2.0' or '2.1' (default '2.1') + * @param {boolean} [options.includeToc] - Include the x-mitre-collection TOC object (default true) * @returns {Promise} The formatted export */ -// eslint-disable-next-line no-unused-vars exports.exportSnapshot = async function exportSnapshot(snapshot, format, options = {}) { - const members = snapshot.members || []; - if (format === 'bundle') { - const hydratedMembers = await exports.hydrateMembers(members); - return exports.formatAsBundle(snapshot, hydratedMembers); + const entries = collectBundleEntries(snapshot, options); + const hydratedObjects = await exports.hydrateMembers(entries); + const supportingObjects = await fetchSupportingObjects(hydratedObjects); + const allObjects = [...hydratedObjects, ...supportingObjects]; + await convertLinkByIdTags(allObjects); + + return exports.formatAsBundle(snapshot, allObjects, { + stixVersion: options.stixVersion, + includeToc: options.includeToc, + attackSpecVersion: config.app.attackSpecVersion, + }); } if (format === 'filesystemstore') { - const hydratedMembers = await exports.hydrateMembers(members); + const hydratedMembers = await exports.hydrateMembers(snapshot.members || []); return exports.formatAsFilesystemStore(snapshot, hydratedMembers); } diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index d8b9ab74..502c2b8a 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -250,9 +250,9 @@ exports.deleteSnapshot = function deleteSnapshot(trackId, modified) { // Ephemeral (Phase 6 → ephemeral-service) // ----------------------------------------------------------------------------- -exports.getEphemeralBundle = function getEphemeralBundle(domain, format) { - rejectFilesystemStoreFormat(format, 'getEphemeralBundle'); - return ephemeralService.getEphemeralBundle(domain, format); +exports.getEphemeralBundle = function getEphemeralBundle(domain, options) { + rejectFilesystemStoreFormat(options?.format, 'getEphemeralBundle'); + return ephemeralService.getEphemeralBundle(domain, options); }; // ----------------------------------------------------------------------------- diff --git a/app/services/stix/stix-bundles-service.js b/app/services/stix/stix-bundles-service.js index 50908869..04433cc3 100644 --- a/app/services/stix/stix-bundles-service.js +++ b/app/services/stix/stix-bundles-service.js @@ -6,6 +6,7 @@ const { BaseService } = require('../meta-classes'); const linkById = require('../../lib/linkById'); const logger = require('../../lib/logger'); const { requiresAttackId } = require('../../lib/attack-id-generator'); +const stixConformance = require('../../lib/stix-conformance'); // Import repositories const analyticsRepository = require('../../repository/analytics-repository'); @@ -220,43 +221,20 @@ class StixBundlesService extends BaseService { /** * Removes empty array properties from a STIX object. + * Delegates to the shared lib/stix-conformance helpers. * @param {Object} stixObject - The STIX object to clean */ static removeEmptyArrays(stixObject) { - for (const propertyName of Object.keys(stixObject)) { - if (Array.isArray(stixObject[propertyName]) && stixObject[propertyName].length === 0) { - delete stixObject[propertyName]; - } - } + stixConformance.removeEmptyArrays(stixObject); } /** * Modifies a STIX object to conform to the specified STIX version (2.0 or 2.1). - * Handles version-specific requirements for various object types. + * Delegates to the shared lib/stix-conformance helpers. * @param {Object} stixObject - The STIX object to modify */ static conformToStixVersion(stixObject, stixVersion) { - if (stixVersion === '2.0') { - // Remove STIX 2.1 specific properties - delete stixObject.spec_version; - - // Handle malware and tool specific requirements - if (stixObject.type === 'malware') { - delete stixObject.is_family; - stixObject.labels = ['malware']; - } - - if (stixObject.type === 'tool') { - stixObject.labels = ['tool']; - } - } else if (stixVersion === '2.1') { - stixObject.spec_version = '2.1'; - if (stixObject.type != 'course-of-action') { - delete stixObject.labels; - } - } - - this.removeEmptyArrays(stixObject); + stixConformance.conformToStixVersion(stixObject, stixVersion); } // ============================ diff --git a/app/tests/api/release-tracks/ephemeral-bundle.spec.js b/app/tests/api/release-tracks/ephemeral-bundle.spec.js new file mode 100644 index 00000000..d411a18e --- /dev/null +++ b/app/tests/api/release-tracks/ephemeral-bundle.spec.js @@ -0,0 +1,293 @@ +/** + * Ephemeral Bundle Export Tests + * ============================== + * + * Regression tests for GET /api/release-tracks/ephemeral/:domain, which + * supplants the deprecated GET /api/stix-bundles endpoint. + * + * Covered behavior: + * - Bundle generation preserves the legacy stix-bundles object-selection + * logic (secondary objects such as groups are pulled in via + * relationships, referenced identities/markings are included) + * - A table-of-contents (x-mitre-collection) object is included by default + * with the ephemeral defaults: x_mitre_version '0.1' and the global + * default ATT&CK spec version + * - includeToc=false omits the TOC + * - includeObjectsWithMissingAttackId (renamed from includeMissingAttackId) + * - includeDeprecated / includeRevoked (also govern deprecated data + * sources, replacing the removed includeDataSources parameter) + * - stixVersion ('2.0' | '2.1', default '2.1') + * - format=workbench still returns the Workbench document shape + */ + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const AttackObject = require('../../../models/attack-object-model'); + +const logger = require('../../../lib/logger'); +logger.level = 'debug'; + +// Seeded by databaseConfiguration.checkSystemConfiguration() +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +const enterpriseDomain = 'enterprise-attack'; +const icsDomain = 'ics-attack'; + +describe('Ephemeral Bundle API', function () { + let app; + let passportCookie; + + let enterpriseTechnique; + let noAttackIdTechnique; + let deprecatedTechnique; + let revokedTechnique; + let icsTechnique; + let group; + let relationship; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function postObject(path, body) { + const res = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201) + .expect('Content-Type', /json/); + return res.body; + } + + async function getEphemeral(query = '', expectedStatus = 200) { + const res = await request(app) + .get(`/api/release-tracks/ephemeral/enterprise${query}`) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return res.body; + } + + function buildTechnique(name, domains, overrides = {}) { + const timestamp = new Date().toISOString(); + return { + workspace: { + workflow: { + state: 'work-in-progress', + }, + }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `Description for ${name}`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + x_mitre_domains: domains, + ...overrides, + }, + }; + } + + function bundleObjectIds(bundle) { + return bundle.objects.map((o) => o.id); + } + + before('set up domain objects', async function () { + enterpriseTechnique = await postObject( + '/api/techniques', + buildTechnique('Enterprise Technique', [enterpriseDomain]), + ); + + icsTechnique = await postObject( + '/api/techniques', + buildTechnique('ICS Technique', [icsDomain]), + ); + + deprecatedTechnique = await postObject( + '/api/techniques', + buildTechnique('Deprecated Technique', [enterpriseDomain], { x_mitre_deprecated: true }), + ); + + // 'revoked' is server-controlled on create, so set it directly + revokedTechnique = await postObject( + '/api/techniques', + buildTechnique('Revoked Technique', [enterpriseDomain]), + ); + await AttackObject.updateOne( + { 'stix.id': revokedTechnique.stix.id, 'stix.modified': revokedTechnique.stix.modified }, + { $set: { 'stix.revoked': true } }, + ); + + // The server auto-generates ATT&CK IDs for techniques, so strip the + // generated external reference to simulate an object with a missing + // ATT&CK ID + noAttackIdTechnique = await postObject( + '/api/techniques', + buildTechnique('No AttackId Technique', [enterpriseDomain]), + ); + await AttackObject.updateOne( + { + 'stix.id': noAttackIdTechnique.stix.id, + 'stix.modified': noAttackIdTechnique.stix.modified, + }, + { $set: { 'stix.external_references': [] }, $unset: { 'workspace.attack_id': '' } }, + ); + + // Group (secondary object): pulled into the bundle via its relationship + // to the enterprise technique + group = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Ephemeral Test Group', + spec_version: '2.1', + type: 'intrusion-set', + description: 'Group used to verify secondary-object inclusion.', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + + relationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: enterpriseTechnique.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + }); + + it('GET /api/release-tracks/ephemeral/:domain returns a STIX 2.1 bundle with legacy-parity contents', async function () { + const bundle = await getEphemeral(); + + expect(bundle.type).toBe('bundle'); + expect(bundle.id).toMatch(/^bundle--/); + // STIX 2.1 removed spec_version from the bundle object + expect(bundle.spec_version).toBeUndefined(); + + const ids = bundleObjectIds(bundle); + + // Primary object from the requested domain + expect(ids).toContain(enterpriseTechnique.stix.id); + + // Secondary object (group) discovered through its 'uses' relationship + expect(ids).toContain(group.stix.id); + expect(ids).toContain(relationship.stix.id); + + // The group's domains are inferred from the technique it uses + const bundleGroup = bundle.objects.find((o) => o.id === group.stix.id); + expect(bundleGroup.x_mitre_domains).toEqual([enterpriseDomain]); + + // Referenced supporting objects + expect(ids).toContain(enterpriseTechnique.stix.created_by_ref); + expect(ids).toContain(staticMarkingDefinitionId); + + // Excluded by default: wrong domain, deprecated, revoked, missing ATT&CK ID + expect(ids).not.toContain(icsTechnique.stix.id); + expect(ids).not.toContain(deprecatedTechnique.stix.id); + expect(ids).not.toContain(revokedTechnique.stix.id); + expect(ids).not.toContain(noAttackIdTechnique.stix.id); + }); + + it('includes a TOC object with ephemeral defaults', async function () { + const bundle = await getEphemeral(); + + const toc = bundle.objects[0]; + expect(toc.type).toBe('x-mitre-collection'); + expect(toc.name).toBe('Enterprise ATT&CK'); + // '0.1' signifies an ephemerally generated collection that is not + // connected to a release track + expect(toc.x_mitre_version).toBe('0.1'); + expect(toc.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion); + expect(toc.spec_version).toBe('2.1'); + expect(typeof toc.modified).toBe('string'); + + const contentRefs = toc.x_mitre_contents.map((entry) => entry.object_ref); + expect(contentRefs).toContain(enterpriseTechnique.stix.id); + expect(toc.object_marking_refs).toContain(staticMarkingDefinitionId); + }); + + it('includeToc=false omits the TOC object', async function () { + const bundle = await getEphemeral('?includeToc=false'); + const tocObjects = bundle.objects.filter((o) => o.type === 'x-mitre-collection'); + expect(tocObjects.length).toBe(0); + }); + + it('includeObjectsWithMissingAttackId=true includes objects without ATT&CK IDs', async function () { + const bundle = await getEphemeral('?includeObjectsWithMissingAttackId=true'); + expect(bundleObjectIds(bundle)).toContain(noAttackIdTechnique.stix.id); + }); + + it('includeDeprecated=true includes deprecated objects', async function () { + const bundle = await getEphemeral('?includeDeprecated=true'); + expect(bundleObjectIds(bundle)).toContain(deprecatedTechnique.stix.id); + }); + + it('includeRevoked=true includes revoked objects', async function () { + const bundle = await getEphemeral('?includeRevoked=true'); + expect(bundleObjectIds(bundle)).toContain(revokedTechnique.stix.id); + }); + + it('stixVersion=2.0 conforms the bundle to STIX 2.0', async function () { + const bundle = await getEphemeral('?stixVersion=2.0'); + + expect(bundle.spec_version).toBe('2.0'); + const technique = bundle.objects.find((o) => o.id === enterpriseTechnique.stix.id); + expect(technique.spec_version).toBeUndefined(); + }); + + it('rejects invalid query parameter values', async function () { + await getEphemeral('?stixVersion=1.0', 400); + await getEphemeral('?includeToc=maybe', 400); + }); + + it('format=workbench returns the Workbench document shape', async function () { + const result = await getEphemeral('?format=workbench'); + + expect(result.collection).toBeDefined(); + expect(Array.isArray(result.objects)).toBe(true); + const technique = result.objects.find((o) => o.stix.id === enterpriseTechnique.stix.id); + expect(technique).toBeDefined(); + expect(technique.workspace).toBeDefined(); + }); + + it('format=filesystemstore returns 501', async function () { + await getEphemeral('?format=filesystemstore', 501); + }); + + it('rejects an unknown domain', async function () { + await request(app) + .get('/api/release-tracks/ephemeral/unknown-domain') + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(400); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js new file mode 100644 index 00000000..57d084d7 --- /dev/null +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -0,0 +1,394 @@ +/** + * Release Track Snapshot Bundle Export Tests + * =========================================== + * + * Regression tests for the `format=bundle` output format on the snapshot + * retrieval endpoints: + * + * - GET /api/release-tracks/:id + * - GET /api/release-tracks/:id/snapshots/:modified + * + * Covered behavior: + * - Default bundle contains members only, plus referenced identities and + * marking definitions (self-contained bundle) + * - `include` adds staged and/or candidate tiers (comma-separated or + * repeated, singular or plural tier names) + * - `state` narrows the included staged/candidate entries by workflow + * status; entries marked 'reviewed' are always included + * - `stixVersion` controls bundle/object STIX version conformance + * - `includeToc` controls the x-mitre-collection table-of-contents object, + * which is derived from the release-track metadata + * - LinkById tags are converted to markdown citations + * - Invalid `include`/`state` values are rejected with 400 + */ + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +const logger = require('../../../lib/logger'); +logger.level = 'debug'; + +// Seeded by databaseConfiguration.checkSystemConfiguration() +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Release Tracks Bundle Export API', function () { + let app; + let passportCookie; + + // The organization identity stamped onto created objects by the server + let organizationIdentityId; + let trackId; + let trackUuid; + let snapshotModified; + + let memberObject; + let linkedMemberObject; + let linkedAttackId; + let linkedAttackUrl; + let candidateWip; + let candidateAwaitingReview; + let candidateReviewed; + let stagedObject; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function postObject(path, body) { + const res = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201) + .expect('Content-Type', /json/); + return res.body; + } + + async function postAction(path, body, expectedStatus = 200) { + const res = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return res.body; + } + + async function getBundle(path, expectedStatus = 200) { + const res = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return res.body; + } + + function buildTechnique(name, overrides = {}) { + const timestamp = new Date().toISOString(); + return { + workspace: { + workflow: { + state: 'work-in-progress', + }, + }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `Description for ${name}`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + ...overrides, + }, + }; + } + + function bundleObjectIds(bundle) { + return bundle.objects.map((o) => o.id); + } + + before('set up release track with tiered contents', async function () { + // The server auto-generates an ATT&CK ID (and matching external reference) + // for techniques; LinkById tags resolve against that generated ID + linkedMemberObject = await postObject('/api/techniques', buildTechnique('Linked Technique')); + linkedAttackId = linkedMemberObject.workspace.attack_id; + const linkedAttackRef = (linkedMemberObject.stix.external_references || []).find( + (ref) => ref.external_id === linkedAttackId, + ); + linkedAttackUrl = linkedAttackRef?.url || ''; + + // The server stamps created_by_ref with the organization identity + organizationIdentityId = linkedMemberObject.stix.created_by_ref; + + // Member whose description references the linked technique + memberObject = await postObject( + '/api/techniques', + buildTechnique('Member Technique', { + description: `See (LinkById: ${linkedAttackId}) for details.`, + }), + ); + + candidateWip = await postObject('/api/techniques', buildTechnique('Candidate WIP')); + candidateAwaitingReview = await postObject( + '/api/techniques', + buildTechnique('Candidate Awaiting Review'), + ); + candidateReviewed = await postObject('/api/techniques', buildTechnique('Candidate Reviewed')); + stagedObject = await postObject('/api/techniques', buildTechnique('Staged Technique')); + + const track = await postAction( + '/api/release-tracks/new', + { + name: 'Bundle Test Track', + description: 'Release track bundle export test', + type: 'standard', + }, + 201, + ); + trackId = track.id; + trackUuid = trackId.split('--')[1]; + + // Disable auto-promotion so reviewed candidates stay in the candidates tier + await request(app) + .put(`/api/release-tracks/${trackId}/config`) + .send({ auto_promote: false }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + // Members + await postAction(`/api/release-tracks/${trackId}/contents`, { + x_mitre_contents: [ + { obj_ref: memberObject.stix.id, obj_modified: memberObject.stix.modified }, + { obj_ref: linkedMemberObject.stix.id, obj_modified: linkedMemberObject.stix.modified }, + ], + }); + + // Candidates (all start as work-in-progress) + await postAction(`/api/release-tracks/${trackId}/candidates`, { + object_refs: [ + { id: candidateWip.stix.id, modified: candidateWip.stix.modified }, + { id: candidateAwaitingReview.stix.id, modified: candidateAwaitingReview.stix.modified }, + { id: candidateReviewed.stix.id, modified: candidateReviewed.stix.modified }, + { id: stagedObject.stix.id, modified: stagedObject.stix.modified }, + ], + }); + + // Transition candidate statuses + await postAction(`/api/release-tracks/${trackId}/candidates/review`, { + from: 'work-in-progress', + to: 'awaiting-review', + object_refs: [candidateAwaitingReview.stix.id], + }); + await postAction(`/api/release-tracks/${trackId}/candidates/review`, { + from: 'work-in-progress', + to: 'reviewed', + object_refs: [candidateReviewed.stix.id], + }); + + // Promote one candidate to staged (retains work-in-progress status) + const promoteRes = await postAction(`/api/release-tracks/${trackId}/candidates/promote`, { + object_refs: [stagedObject.stix.id], + }); + snapshotModified = promoteRes.modified; + }); + + it('GET /api/release-tracks/:id?format=bundle returns a members-only STIX 2.1 bundle', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle`); + + expect(bundle.type).toBe('bundle'); + expect(bundle.id).toMatch(/^bundle--/); + // STIX 2.1 removed spec_version from the bundle object + expect(bundle.spec_version).toBeUndefined(); + + const ids = bundleObjectIds(bundle); + expect(ids).toContain(memberObject.stix.id); + expect(ids).toContain(linkedMemberObject.stix.id); + + // Tier entries not selected via include are excluded + expect(ids).not.toContain(candidateWip.stix.id); + expect(ids).not.toContain(candidateAwaitingReview.stix.id); + expect(ids).not.toContain(candidateReviewed.stix.id); + expect(ids).not.toContain(stagedObject.stix.id); + + // Referenced supporting objects are included so the bundle is self-contained + expect(ids).toContain(organizationIdentityId); + expect(ids).toContain(staticMarkingDefinitionId); + + // Objects conform to STIX 2.1 + const member = bundle.objects.find((o) => o.id === memberObject.stix.id); + expect(member.spec_version).toBe('2.1'); + + // Bundle objects contain STIX properties only (no workspace/workflow data) + expect(member.workspace).toBeUndefined(); + }); + + it('GET /api/release-tracks/:id?format=bundle includes a TOC derived from the track metadata', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle`); + + const toc = bundle.objects[0]; + expect(toc.type).toBe('x-mitre-collection'); + expect(toc.id).toBe(`x-mitre-collection--${trackUuid}`); + expect(toc.name).toBe('Bundle Test Track'); + // Draft snapshots (version: null) fall back to '0.1' + expect(toc.x_mitre_version).toBe('0.1'); + expect(toc.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion); + expect(toc.spec_version).toBe('2.1'); + + // Marking definitions are tracked in object_marking_refs, everything else + // in x_mitre_contents + expect(toc.object_marking_refs).toContain(staticMarkingDefinitionId); + const contentRefs = toc.x_mitre_contents.map((entry) => entry.object_ref); + expect(contentRefs).toContain(memberObject.stix.id); + expect(contentRefs).toContain(organizationIdentityId); + expect(contentRefs).not.toContain(staticMarkingDefinitionId); + expect(contentRefs).not.toContain(toc.id); + }); + + it('GET /api/release-tracks/:id?format=bundle&includeToc=false omits the TOC', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle&includeToc=false`); + const tocObjects = bundle.objects.filter((o) => o.type === 'x-mitre-collection'); + expect(tocObjects.length).toBe(0); + }); + + it('GET /api/release-tracks/:id?format=bundle converts LinkById tags to markdown citations', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle`); + const member = bundle.objects.find((o) => o.id === memberObject.stix.id); + expect(member.description).toBe(`See [Linked Technique](${linkedAttackUrl}) for details.`); + }); + + it('GET /api/release-tracks/:id?format=bundle&include=candidates adds the candidates tier', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}?format=bundle&include=candidates`, + ); + + const ids = bundleObjectIds(bundle); + expect(ids).toContain(memberObject.stix.id); + expect(ids).toContain(candidateWip.stix.id); + expect(ids).toContain(candidateAwaitingReview.stix.id); + expect(ids).toContain(candidateReviewed.stix.id); + expect(ids).not.toContain(stagedObject.stix.id); + }); + + it('GET /api/release-tracks/:id?format=bundle&include=staged adds the staged tier', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle&include=staged`); + + const ids = bundleObjectIds(bundle); + expect(ids).toContain(memberObject.stix.id); + expect(ids).toContain(stagedObject.stix.id); + expect(ids).not.toContain(candidateWip.stix.id); + }); + + it('GET /api/release-tracks/:id?format=bundle&include=candidates,staged adds both tiers', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}?format=bundle&include=candidates,staged`, + ); + + const ids = bundleObjectIds(bundle); + expect(ids).toContain(memberObject.stix.id); + expect(ids).toContain(candidateWip.stix.id); + expect(ids).toContain(candidateAwaitingReview.stix.id); + expect(ids).toContain(candidateReviewed.stix.id); + expect(ids).toContain(stagedObject.stix.id); + }); + + it('GET /api/release-tracks/:id?format=bundle accepts singular tier names and repeated params', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}?format=bundle&include=candidate&include=staged`, + ); + + const ids = bundleObjectIds(bundle); + expect(ids).toContain(candidateWip.stix.id); + expect(ids).toContain(stagedObject.stix.id); + }); + + it('state narrows included candidates but reviewed entries are always included', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}?format=bundle&include=candidates&state=work-in-progress`, + ); + + const ids = bundleObjectIds(bundle); + // Members are unaffected by state + expect(ids).toContain(memberObject.stix.id); + // Matching workflow status + expect(ids).toContain(candidateWip.stix.id); + // Reviewed entries are always included, irrespective of state + expect(ids).toContain(candidateReviewed.stix.id); + // Non-matching, non-reviewed status is excluded + expect(ids).not.toContain(candidateAwaitingReview.stix.id); + }); + + it('state applies to the staged tier as well', async function () { + // The staged object retained its work-in-progress status through promotion + const withMatchingState = await getBundle( + `/api/release-tracks/${trackId}?format=bundle&include=staged&state=work-in-progress`, + ); + expect(bundleObjectIds(withMatchingState)).toContain(stagedObject.stix.id); + + const withoutMatchingState = await getBundle( + `/api/release-tracks/${trackId}?format=bundle&include=staged&state=awaiting-review`, + ); + expect(bundleObjectIds(withoutMatchingState)).not.toContain(stagedObject.stix.id); + }); + + it('GET /api/release-tracks/:id?format=bundle&stixVersion=2.0 conforms the bundle to STIX 2.0', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle&stixVersion=2.0`); + + expect(bundle.spec_version).toBe('2.0'); + const member = bundle.objects.find((o) => o.id === memberObject.stix.id); + expect(member.spec_version).toBeUndefined(); + }); + + it('rejects invalid include, state, and stixVersion values for bundle exports', async function () { + await getBundle(`/api/release-tracks/${trackId}?format=bundle&include=quarantine`, 400); + await getBundle( + `/api/release-tracks/${trackId}?format=bundle&include=candidates&state=reviewed`, + 400, + ); + await getBundle(`/api/release-tracks/${trackId}?format=bundle&stixVersion=3.0`, 400); + }); + + it('GET /api/release-tracks/:id/snapshots/:modified?format=bundle exports a historical snapshot', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/${snapshotModified}?format=bundle&include=candidates,staged`, + ); + + expect(bundle.type).toBe('bundle'); + expect(bundle.spec_version).toBeUndefined(); + expect(bundle.objects[0].type).toBe('x-mitre-collection'); + + const ids = bundleObjectIds(bundle); + expect(ids).toContain(memberObject.stix.id); + expect(ids).toContain(candidateWip.stix.id); + expect(ids).toContain(stagedObject.stix.id); + }); + + it('GET /api/release-tracks/:id (workbench default) is unaffected by bundle parameters', async function () { + const snapshot = await getBundle(`/api/release-tracks/${trackId}`); + expect(snapshot.members).toBeDefined(); + expect(snapshot.candidates).toBeDefined(); + expect(snapshot.staged).toBeDefined(); + expect(snapshot.type).toBe('standard'); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 7cced1a4..9b9a4e78 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -28,9 +28,9 @@ function buildTechnique(name, description) { type: 'attack-pattern', object_marking_refs: ['marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'], created_by_ref: 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5', - kill_chain_phases: [{ kill_chain_name: 'kill-chain-name-1', phase_name: 'phase-1' }], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], x_mitre_is_subtechnique: false, - x_mitre_platforms: ['platform-1'], + x_mitre_platforms: ['Windows'], }, }; } @@ -43,7 +43,7 @@ describe('Release Tracks API', function () { await database.initializeConnection(); await databaseConfiguration.checkSystemConfiguration(); - config.validateRequests.withAttackDataModel = false; + config.validateRequests.withAttackDataModel = true; config.validateRequests.withOpenApi = true; app = await require('../../../index').initializeApp(); diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md new file mode 100644 index 00000000..af688cdf --- /dev/null +++ b/docs/developer/release-tracks/bundle-export.md @@ -0,0 +1,152 @@ +# Bundle Export + +This document explains how STIX bundle emission works after the introduction +of release tracks: what the legacy behavior was, why it changed, and how the +new endpoints are implemented. + +## What was: `GET /api/stix-bundles` + +Before release tracks, Workbench emitted STIX bundles exclusively through the +domain-scoped `GET /api/stix-bundles` endpoint +([stix-bundles-routes.js](../../../app/routes/stix-bundles-routes.js)). Its +service module ([stix-bundles-service.js](../../../app/services/stix/stix-bundles-service.js)) +implements the ATT&CK bundle-composition rules: + +1. **Primary objects** are retrieved by domain (`x_mitre_domains`): + techniques, tactics, mitigations, software, matrices, analytics, data + components, data sources. +2. **Secondary objects** (groups, campaigns, detection strategies) cannot be + assigned domains by users; they are discovered through relationships to + primary objects and their `x_mitre_domains` is inferred at export time. +3. **Relationship referential integrity**: a relationship is only emitted if + both its `source_ref` and `target_ref` are present in the bundle. +4. **Supporting objects**: identities (`created_by_ref`) and marking + definitions (`object_marking_refs`) referenced by bundle objects are + fetched and appended so the bundle is self-contained. +5. **LinkById conversion**: `(LinkById: T1234)` tags in descriptions are + converted to markdown citations. +6. **STIX version conformance**: objects are rewritten to STIX 2.0 or 2.1 + rules (see [lib/stix-conformance.js](../../../app/lib/stix-conformance.js), + extracted from the legacy service so both pipelines share it). +7. **Collection object**: optionally, an `x-mitre-collection` object + describing the bundle contents is prepended. + +Bundle composition was configured entirely through query parameters +(`state`, `includeNotes`, `includeDataSources`, `useLegacyMethod`, +`includeCollectionObject`, `collectionObjectVersion`, ...) because there was +no persistent, curated representation of "a release" — every export was +ad hoc. + +## What is: release-track exports and ephemeral bundles + +Release tracks give Workbench a persistent, versioned model of a release +(members / staged / candidates tiers with per-track workflow status). That +changes what bundle emission needs to be: + +- **Curated exports** come from a release-track snapshot. The snapshot + already records exactly which object revisions belong to the release, so + the export no longer needs domain queries, workflow-state heuristics, or + attack-id filtering — it hydrates the pinned revisions and formats them. +- **Ad hoc domain exports** remain useful ("give me everything in enterprise + right now"), which is what the ephemeral endpoint provides. + +`GET /api/stix-bundles` is therefore **deprecated** (marked in the OpenAPI +spec) and will be removed in a future release. Its replacements: + +| Legacy usage | Replacement | +|--------------|-------------| +| Domain-scoped ad hoc bundle | `GET /api/release-tracks/ephemeral/:domain` | +| Release/publication bundle | `GET /api/release-tracks/:id?format=bundle` (or `/snapshots/:modified?format=bundle`) | + +### Ephemeral endpoint parameter mapping + +`GET /api/release-tracks/ephemeral/:domain` (default `format=bundle`) +delegates to `stix-bundles-service.exportBundle` so all of the legacy +object-selection logic above is preserved verbatim. The query-parameter +surface was simplified +(see [ephemeral-service.js](../../../app/services/release-tracks/ephemeral-service.js)): + +| Legacy parameter | Disposition | +|------------------|-------------| +| `stixVersion` | **Preserved** (default changed to `2.1`) | +| `includeRevoked` / `includeDeprecated` | **Preserved** (default `false`) | +| `includeMissingAttackId` | **Renamed** to `includeObjectsWithMissingAttackId` (default `false`) | +| `includeCollectionObject` | **Renamed** to `includeToc` (default `true`). "TOC" (table of contents) describes what the `x-mitre-collection` object actually is, and avoids overloading the term "collection". | +| `collectionObjectVersion` | **Removed** — fixed at `0.1`, signifying an ephemerally generated collection not connected to a release track | +| `collectionObjectModified` | **Removed** — fixed at the current timestamp | +| `collectionAttackSpecVersion` | **Removed** — fixed at the global default (`config.app.attackSpecVersion`) | +| `includeNotes` | **Removed** — notes are Workbench-native objects, not STIX objects, and are never emitted in bundles | +| `includeDataSources` | **Removed** — data sources are deprecated (ATT&CK Spec v3.3.0) and were marked deprecated/revoked in ATT&CK v18, so their inclusion is governed entirely by `includeDeprecated`/`includeRevoked`. Internally the delegation passes `includeDataSources: true` and lets those flags filter. | +| `useLegacyMethod` | **Removed** — the pre-v17 code path (`stix-bundles-service-old.js`) is not supported by the new endpoints | +| `state` | **Removed** — workflow status is now scoped to release tracks; a domain-scoped endpoint has no workflow-status concept | + +Note on the bundle envelope: STIX 2.0 requires `spec_version` on the bundle +object, while STIX 2.1 removed it (objects declare their own `spec_version` +instead). Both the ephemeral endpoint and the legacy endpoint therefore stamp +`spec_version: "2.0"` on the envelope only when `stixVersion=2.0`. + +### Release-track snapshot exports (`format=bundle`) + +Implemented in +[export-service.js](../../../app/services/release-tracks/export-service.js) +(`exportSnapshot`) with the DTO transformation in +[export-schemas.js](../../../app/lib/release-tracks/export-schemas.js) +(`bundleTransformSchema`). The pipeline: + +1. **Tier selection** — members are always exported. `include` (values + `staged` and/or `candidates`; singular forms accepted) adds tiers. + `state` (values `work-in-progress` and/or `awaiting-review`) narrows the + added tiers; entries whose `object_status` is `reviewed` always pass the + filter, mirroring the fact that members are inherently reviewed. `state` + never affects members. `reviewed` is intentionally not a valid `state` + value for this reason. +2. **Hydration** — the selected `{object_ref, object_modified}` pins are + batch-fetched per STIX type via each repository's + `findManyByIdAndModified`. +3. **Supporting objects** — referenced identities and marking definitions + that are not themselves tier entries are fetched and appended. +4. **LinkById conversion** — same behavior as the legacy exporter, preferring + objects already in the export before falling back to a database lookup. +5. **Assembly** (Zod transform) — notes are dropped, objects are conformed to + `stixVersion` via the shared `lib/stix-conformance.js` helpers, and the + bundle envelope is emitted (with `spec_version: "2.0"` only when + `stixVersion=2.0` — STIX 2.1 removed `spec_version` from the bundle + object). +6. **TOC** — unless `includeToc=false`, an `x-mitre-collection` object is + prepended. Unlike the legacy exporter (which hardcoded per-domain + metadata) and the ephemeral endpoint (which uses ephemeral defaults), the + TOC is derived from the release track itself: + - `id`: `x-mitre-collection--` — stable across exports of the + same track + - `name`/`description`/`created_by_ref`/`object_marking_refs`: from the + snapshot metadata + - `x_mitre_version`: the snapshot's tagged version, or `0.1` for drafts + - `modified`: the snapshot's `modified` timestamp + - `x_mitre_contents`: every bundle object except marking definitions + (which are recorded in `object_marking_refs`), sorted by `object_ref` + +Because snapshot contents are explicitly curated, the export intentionally +does **not** apply the legacy attack-id / deprecated / revoked filters — if a +revision is in the snapshot, it is exported. + +### Where validation happens + +Query parameters are validated in the controller with Zod +([release-track-schemas.js](../../../app/lib/release-tracks/release-track-schemas.js)). +The OpenAPI spec declares the parameters loosely (`oneOf` string/array with +`allowReserved` for the list-valued `include`/`state`) so that both +comma-separated and repeated-parameter forms reach the Zod layer, which +normalizes and enforces the enums. Invalid values produce a 400 +`InvalidQueryStringParameterError`. + +### Regression tests + +- [release-tracks-bundle.spec.js](../../../app/tests/api/release-tracks/release-tracks-bundle.spec.js) + — snapshot bundle exports (tier selection, state filtering, STIX version + conformance, TOC, LinkById, supporting objects, validation errors) +- [ephemeral-bundle.spec.js](../../../app/tests/api/release-tracks/ephemeral-bundle.spec.js) + — ephemeral bundles (legacy-parity object selection, parameter mapping, + TOC defaults, workbench format) +- [stix-bundles.spec.js](../../../app/tests/api/stix-bundles/stix-bundles.spec.js) + — legacy endpoint behavior (still authoritative for + `stix-bundles-service.exportBundle`, which the ephemeral endpoint reuses) diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 9e330d26..b0c903ce 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -117,11 +117,31 @@ These refer to all objects delineated by ATT&CK domain membership as reflected b GET /api/release-tracks/ephemeral/:domain ``` +This endpoint supplants the deprecated `GET /api/stix-bundles` endpoint. The +generated bundle preserves the legacy object-selection behavior: primary +objects are retrieved by domain, secondary objects (groups, campaigns, +detection strategies) are discovered through relationships, and referenced +identities and marking definitions are included so the bundle is +self-contained. + **Path Parameters:** - `:domain` - `enterprise` | `ics` | `mobile` **Query Parameters:** -- `format` - `bundle` | `filesystemstore` | `workbench` (default: `bundle`; `filesystemstore` is not yet implemented) + +| Parameter | Values | Default | Description | +|-----------|--------|---------|-------------| +| `format` | `bundle` \| `workbench` \| `filesystemstore` | `bundle` | Output format (`filesystemstore` is not yet implemented) | +| `stixVersion` | `2.0` \| `2.1` | `2.1` | STIX version the emitted bundle conforms to (bundle format only) | +| `includeToc` | `true` \| `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) in the bundle. The TOC is generated with `x_mitre_version: "0.1"` (signifying an ephemeral, non-release-track collection), a `modified` of the current timestamp, and the deployment's default ATT&CK spec version. | +| `includeObjectsWithMissingAttackId` | `true` \| `false` | `false` | Include objects that should have an ATT&CK ID set but do not | +| `includeDeprecated` | `true` \| `false` | `false` | Include objects with `x_mitre_deprecated: true` (this also governs deprecated Data Sources) | +| `includeRevoked` | `true` \| `false` | `false` | Include objects with `revoked: true` | + +> [!Note] +> The ephemeral endpoint does not support the `include` or `state` tier +> filters because it does not read from a persisted release-track snapshot — +> it includes all objects in the domain. --- @@ -280,6 +300,17 @@ Workbench responses return the release-track snapshot shape. Entries in the `mem | `version` | `X.Y` | Return specific version (e.g., `14.1`) | | `versions` | `all` | List all snapshots with metadata | +**Additional query parameters for `format=bundle`:** + +| Parameter | Values | Description | +|-----------|--------|-------------| +| `include` | `staged` and/or `candidates` (comma-separated or repeated) | Additional tiers to include in the bundle alongside members. If omitted, only members are included. (Note the different semantics from `workbench` responses.) | +| `state` | `work-in-progress` and/or `awaiting-review` (comma-separated or repeated) | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included. Members are unaffected. | +| `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`) | +| `includeToc` | `true` \| `false` | Include a table-of-contents object (of type `x-mitre-collection`) derived from the release-track metadata (default: `true`) | + +See [Output Formats](output-formats.md) for details on the bundle structure. + **Examples:** ```bash @@ -289,6 +320,12 @@ GET /api/release-tracks/:id # Get latest snapshot as STIX bundle (members only) GET /api/release-tracks/:id?format=bundle +# Get latest snapshot as STIX bundle with staged and candidate objects +GET /api/release-tracks/:id?format=bundle&include=candidates,staged + +# Get latest snapshot as STIX bundle with candidates awaiting review +GET /api/release-tracks/:id?format=bundle&include=candidates&state=awaiting-review + # Get latest snapshot with members and quarantine only GET /api/release-tracks/:id?include=quarantine @@ -404,6 +441,10 @@ GET /api/release-tracks/:id/snapshots/:modified - `format` - `workbench` | `bundle` | `filesystemstore` (default: `workbench`; `filesystemstore` is not yet implemented) - `include` - `members` | `staged` | `candidates` | `quarantine` | `all` (default: all tiers) +For `format=bundle`, the same additional parameters as +[Get Latest Snapshot](#get-latest-snapshot) apply: `include` (bundle +semantics), `state`, `stixVersion`, and `includeToc`. + **Example:** ```bash # Get snapshot from January 15, 2024 for the Workbench UI @@ -411,6 +452,9 @@ GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z # Get snapshot from January 15, 2024 as STIX bundle GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle + +# Historical snapshot as a bundle including staged objects +GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle&include=staged ``` ### Update Metadata (Specific Snapshot) @@ -967,7 +1011,7 @@ The following release-track snapshot retrieval endpoints support `include` and The ephemeral bundle endpoint supports `format`, but not tier `include`, because it does not read from a persisted release-track snapshot. -**Include Parameter** (controls which tiers are returned): +**Include Parameter** (workbench format — controls which tiers are returned): ``` GET /api/release-tracks/:id # Default: all tiers GET /api/release-tracks/:id?include=members # Members tier only @@ -977,10 +1021,26 @@ GET /api/release-tracks/:id?include=quarantine # Members and quarantine GET /api/release-tracks/:id?include=all # All tiers ``` +**Include Parameter** (bundle format — controls which tiers are hydrated into +the bundle; members are always included): +``` +GET /api/release-tracks/:id?format=bundle # Members only +GET /api/release-tracks/:id?format=bundle&include=staged # Members + staged +GET /api/release-tracks/:id?format=bundle&include=candidates # Members + candidates +GET /api/release-tracks/:id?format=bundle&include=candidates,staged # Members + both +``` + +**State Parameter** (bundle format only — narrows the tiers selected via +`include` by workflow status; `reviewed` entries are always included): +``` +GET /api/release-tracks/:id?format=bundle&include=candidates&state=work-in-progress +GET /api/release-tracks/:id?format=bundle&include=candidates,staged&state=work-in-progress,awaiting-review +``` + **Format Parameter** (controls output format): ``` GET /api/release-tracks/:id?format=workbench # Workbench snapshot with metadata (default) -GET /api/release-tracks/:id?format=bundle # Standard STIX 2.1 bundle +GET /api/release-tracks/:id?format=bundle # Standard STIX bundle GET /api/release-tracks/:id?format=filesystemstore # Not implemented; returns 501 ``` diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index 8a9f9a5a..62378761 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -57,7 +57,7 @@ GET /api/release-tracks/:id?include=all ### Format: `bundle` -Standard STIX 2.1 bundle format: +Standard STIX bundle format: ```json { @@ -67,9 +67,12 @@ Standard STIX 2.1 bundle format: { "type": "x-mitre-collection", "id": "x-mitre-collection--123", + "name": "ATT&CK Enterprise", "x_mitre_version": "1.1", - "x_mitre_contents": ["attack-pattern--aaa", "malware--bbb"], - "name": "ATT&CK Enterprise" + "x_mitre_contents": [ + { "object_ref": "attack-pattern--aaa", "object_modified": "2024-01-10T10:00:00.000Z" } + ], + "object_marking_refs": ["marking-definition--..."] }, { "type": "attack-pattern", @@ -82,11 +85,55 @@ Standard STIX 2.1 bundle format: ``` **Characteristics:** -- STIX 2.1 compliant +- STIX compliant (2.1 by default; 2.0 via `stixVersion=2.0`). Per the STIX + specifications, the bundle object carries `spec_version` only for STIX 2.0; + STIX 2.1 bundles omit it and each object declares its own `spec_version`. - Only includes `stix.*` properties - No workflow states, no workspace data +- Self-contained: identities and marking definitions referenced by the + exported objects are included automatically +- `LinkById` tags in descriptions are converted to markdown citations +- Notes are never included (notes are Workbench-native objects, not STIX objects) - Suitable for external publication +**Bundle query parameters** (apply only when `format=bundle`): + +| Parameter | Values | Default | Description | +|-----------|--------|---------|-------------| +| `include` | `staged`, `candidates` (comma-separated or repeated) | _(members only)_ | Additional tiers to include in the bundle alongside members | +| `state` | `work-in-progress`, `awaiting-review` (comma-separated or repeated) | _(no filter)_ | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included, irrespective of this parameter. Members are unaffected. | +| `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to | +| `includeToc` | `true`, `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) as the first object in the bundle | + +Examples: + +```bash +# Members only (default) +GET /api/release-tracks/:id?format=bundle + +# Members + staged objects +GET /api/release-tracks/:id?format=bundle&include=staged + +# Members + candidates and staged objects that are work-in-progress or reviewed +GET /api/release-tracks/:id?format=bundle&include=candidates,staged&state=work-in-progress + +# STIX 2.0 bundle without a table of contents +GET /api/release-tracks/:id?format=bundle&stixVersion=2.0&includeToc=false +``` + +**The table of contents (TOC) object** + +By default, bundles begin with an `x-mitre-collection` object that acts as a +table of contents. It is derived from the release-track metadata: + +- `id` — stable per track (reuses the track UUID) +- `name` / `description` — from the release track +- `x_mitre_version` — the snapshot's tagged version, or `0.1` for draft snapshots +- `modified` — the snapshot's modified timestamp +- `x_mitre_attack_spec_version` — the deployment's default ATT&CK spec version +- `x_mitre_contents` — every object in the bundle (marking definitions are + recorded in `object_marking_refs` instead) + ### Format: `filesystemstore` (Not Implemented) STIX FileSystemStore export is planned, but is not implemented yet. Requests From 0db34071eee9dfcf72edc286aeafd7b09b32e891 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:23:40 -0400 Subject: [PATCH 02/55] fix(validation-bypasses): make startup rule seeding idempotent The identity, namespace, and static bypass-rule seeding paths inserted rules and relied on the unique (fieldPath, errorCode, stixType) index to reject duplicates via DuplicateIdError. On a fresh database the index builds in the background, so a duplicate insert could succeed before the index existed; a later explicit index build (e.g. Model.init()) then failed with E11000. Seed rules with an upsert keyed on the compound index fields instead, which is idempotent regardless of index state. --- .../validation-bypasses-repository.js | 31 ++++++++++++++++ .../system/validation-bypasses-service.js | 37 ++++++------------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/app/repository/validation-bypasses-repository.js b/app/repository/validation-bypasses-repository.js index 60416bbd..603e3f30 100644 --- a/app/repository/validation-bypasses-repository.js +++ b/app/repository/validation-bypasses-repository.js @@ -51,6 +51,37 @@ class ValidationBypassesRepository { } } + /** + * Insert a rule if no rule with the same (fieldPath, errorCode, stixType) + * key exists. Unlike save(), this does not rely on the unique index to + * reject duplicates — on a fresh database the index may still be building + * in the background, which would let a duplicate insert through. Used by + * the startup seeding paths (identity/namespace/static rules), which must + * be idempotent. + * + * @param {Object} data - The rule to insert + * @returns {Promise<{created: boolean}>} created is false if the rule already existed + */ + async upsertRule(data) { + try { + const existing = await this.model + .findOneAndUpdate( + { fieldPath: data.fieldPath, errorCode: data.errorCode, stixType: data.stixType }, + { $setOnInsert: data }, + { upsert: true, new: false, runValidators: true }, + ) + .lean() + .exec(); + return { created: existing === null }; + } catch (err) { + if (err.name === 'MongoServerError' && err.code === 11000) { + // Concurrent upsert with the same key — the rule exists + return { created: false }; + } + throw new DatabaseError(err); + } + } + async retrieveById(id) { if (!mongoose.Types.ObjectId.isValid(id)) { return null; diff --git a/app/services/system/validation-bypasses-service.js b/app/services/system/validation-bypasses-service.js index 028338cf..d5dd49e7 100644 --- a/app/services/system/validation-bypasses-service.js +++ b/app/services/system/validation-bypasses-service.js @@ -169,17 +169,13 @@ class ValidationBypassesService { triggerEvent: Events.SYSTEM_CONFIGURATION_NAMESPACE_CHANGED, })); + let created = 0; for (const rule of rules) { - try { - await this.repository.save(rule); - } catch (err) { - // Skip duplicates — rule may already exist - if (err.name === 'DuplicateIdError') continue; - throw err; - } + const result = await this.repository.upsertRule(rule); + if (result.created) created++; } - logger.info(`Created ${rules.length} namespace validation bypass rules`); + logger.info(`Created ${created} of ${rules.length} namespace validation bypass rules`); } /** @@ -199,17 +195,13 @@ class ValidationBypassesService { triggerEvent, })); + let created = 0; for (const rule of rules) { - try { - await this.repository.save(rule); - } catch (err) { - // Skip duplicates — rule may already exist - if (err.name === 'DuplicateIdError') continue; - throw err; - } + const result = await this.repository.upsertRule(rule); + if (result.created) created++; } - logger.info(`Created ${rules.length} identity validation bypass rules`); + logger.info(`Created ${created} of ${rules.length} identity validation bypass rules`); } /** @@ -267,16 +259,9 @@ class ValidationBypassesService { autoCreatedReason: BypassRuleReasons.STATIC, }; - try { - await this.repository.save(bypassRule); - created++; - } catch (err) { - if (err.name === 'DuplicateIdError') { - skipped++; - continue; - } - throw err; - } + const result = await this.repository.upsertRule(bypassRule); + if (result.created) created++; + else skipped++; } logger.info( From 1b8d3dc872ebe7974ae5ef133e4055ada739819e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:07:46 -0400 Subject: [PATCH 03/55] docs: add committable agent guides with local environment split Add AGENTS.md as the canonical, machine-independent agent guide: workspace conventions, architecture map, validation layers, task workflow (docs-first, TODO.md scratchpad, strict test verification, definition of done), regression- test recipes, Bruno conventions, known gotchas, and guide-maintenance rules. Machine-specific paths (workspace parent, ADM source checkout, Bruno collection) live in a gitignored AGENTS.local.md; commit AGENTS.local.example.md as the copyable template. CLAUDE.md imports both so Claude Code and other coding agents share a single source of truth. --- .gitignore | 3 + AGENTS.local.example.md | 33 ++++++++ AGENTS.md | 167 ++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 8 ++ 4 files changed, 211 insertions(+) create mode 100644 AGENTS.local.example.md create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/.gitignore b/.gitignore index 41e16e9d..91ad882e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # A place to store artifacts during local development (scripts, datasets, dotenv files, etc.) .nocommit/**/* +# Machine-specific agent configuration (copy AGENTS.local.example.md to create) +AGENTS.local.md + # Logs logs *.log diff --git a/AGENTS.local.example.md b/AGENTS.local.example.md new file mode 100644 index 00000000..f59b5dcd --- /dev/null +++ b/AGENTS.local.example.md @@ -0,0 +1,33 @@ +# AGENTS.local.md — machine-specific agent configuration + +Copy this file to `AGENTS.local.md` (gitignored) and fill in the paths for +your machine. Agents consult this file for local resource locations referenced +by `AGENTS.md`. + +## Workbench workspace + +Parent directory containing the sibling Workbench repos +(`attack-workbench-frontend`, `attack-workbench-deployment`, +`attack-workbench-taxii-server`, ...): + +``` +/path/to/workbench/ +``` + +## ADM source checkout + +Local clone of https://github.com/mitre-attack/attack-data-model +(Zod schemas under `src/schemas/{sdo,sro,smo,common}`): + +``` +/path/to/attack-data-model +``` + +## Bruno API collection + +Local Bruno collection mirroring this API (omit this section if you don't +maintain one — agents will then skip the Bruno step in the task workflow): + +``` +/path/to/bruno/workbench/ +``` diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..c473875f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,167 @@ +# ATT&CK Workbench REST API — Agent Guide + +Node.js/Express + MongoDB (Mongoose) REST API for managing ATT&CK objects +(STIX 2.x). Part of the multi-repo ATT&CK Workbench ecosystem. + +## Related repositories and local environment + +Machine-specific absolute paths live in `AGENTS.local.md` at the repo root +(gitignored). If it does not exist, copy `AGENTS.local.example.md` to +`AGENTS.local.md` and fill in the values — or ask the developer. Consult it +before searching the filesystem for any of the resources below. + +- **Sibling Workbench repos** — conventionally cloned side-by-side under one + parent directory: `attack-workbench-frontend` (Angular UI that consumes this + API), `attack-workbench-deployment` (Docker Compose configs), + `attack-workbench-taxii-server`. Use them when you need consumer or + deployment context. +- **ADM (ATT&CK Data Model)** — this API validates STIX objects against the + published `@mitre-attack/attack-data-model` package. A local checkout of the + ADM source (`src/schemas/{sdo,sro,smo,common}`) is the authoritative + reference for STIX shapes: valid enum values, required fields, refinements. + Consult it when authoring payloads, especially for regression tests. +- **Bruno API collection** — manual smoke-test requests maintained outside + this repo (see Bruno section below). + +## Read the docs first + +Before designing or coding, read the relevant docs — they explain the API +surface, system design, and adopted patterns. Do not re-derive them from code: + +- `docs/README.md` — index of all documentation +- `docs/user/**` — endpoint behavior and workflows (describes *what is*) +- `docs/developer/**` — architecture and patterns (describes *why and how*), + notably: `data-model.md`, `event-bus-architecture.md`, + `cross-service-reads-pattern.md`, `service-exception-middleware.md`, + `crud-regression-test-taxonomy.md`, and `release-tracks/` +- `CONTRIBUTING.md` — branching and commit conventions + +## Architecture + +Layered request pipeline; keep new code in the matching layer: + +``` +app/routes/*-routes.js Express routers + authn/authz middleware. + Auto-mounted by routes/index.js (any *-routes.js). +app/controllers/ Parse & validate requests (Zod), delegate to a + service, forward errors via next(). No business logic. +app/services/ Business logic. meta-classes/base.service.js is the + generic CRUD base (create pipeline: strip + server-controlled fields → generate ATT&CK ID → + compose → ADM-validate → save). Facade pattern for + multi-service domains (e.g. release-tracks-service.js). +app/repository/ Mongo access; _base.repository.js is the generic base. +app/models/ Mongoose schemas. STIX documents have the shape + { workspace: {...}, stix: {...} }. +``` + +Key mechanics: + +- **Validation is layered**: (1) `express-openapi-validator` against + `app/api/definitions/openapi.yml` (+ `paths/*.yml`, `components/*.yml`); + (2) Zod request schemas in controllers (newer endpoints validate bodies/query + in Zod, with the OpenAPI schema kept loose); (3) ADM validation of the + composed STIX object (`config.validateRequests.withAttackDataModel`). + `work-in-progress` objects use ADM *partial* schemas (fields may be omitted, + but present fields must be valid); all other workflow states use full schemas. + Validation-bypass rules (`/api/config/validation-bypasses`) can suppress + specific ADM errors. +- **Every query parameter must be declared in the OpenAPI paths YAML** or the + validator rejects the request. Comma-separated list params need + `allowReserved: true` and a loose `oneOf` string/array schema, with real + validation in Zod. +- **Server-controlled fields**: on create, the server strips client-supplied + ATT&CK external references and `workspace.attack_id` (then generates them), + strips `revoked` and `x_mitre_attack_spec_version`, and stamps + `created_by_ref` with the organization identity. +- **Event-driven architecture**: cross-service *writes* must go through the + EventBus (`app/lib/event-bus.js`); direct repository *reads* across services + are permitted (see `cross-service-reads-pattern.md`). +- **Errors**: throw typed exceptions from `app/exceptions`; centralized + handlers in `app/lib/error-handler.js` map them to HTTP responses. +- **Config**: convict-based, `app/config/config.js`, env-var driven. + +## Commands + +```bash +npm run lint # eslint (includes prettier rules) +npm run format # prettier + eslint --fix +npm run test:file -- # one spec file +npm run test:api # all API regression tests (~1-2 min) +npm test # full suite: openapi + config + api + middleware +``` + +Tests use `mongodb-memory-server` — no external MongoDB or env setup needed. + +## Task workflow + +1. **Plan in a committable scratchpad**: track multi-step work as checkboxes in + `docs/developer/TODO.md` so progress survives context-window resets and + sessions. Check items off as they complete. Throwaway artifacts (notes, + datasets, one-off scripts) go in `.nocommit/` (gitignored). +2. **Definition of done** — a task is complete only when it includes: + - implementation, + - regression tests (see below), + - test verification, strictly in this order: run the relevant spec files + with `npm run test:file -- ` while iterating, then run the **full** + `npm test` suite — all of it must pass before the task is done, + - OpenAPI spec updates for any API-surface change, + - documentation updates (`docs/user/**` = what the behavior *is*; + `docs/developer/**` = why/how, including how behavior evolved), + - Bruno collection updates for any API-surface change, + - a proposed conventional commit message. +3. **Commits**: conventional commits are enforced (commitlint + + semantic-release; see `CONTRIBUTING.md`). Propose the message (type(scope): + imperative subject + body); do not run `git commit` unless asked. Put + unrelated fixes discovered along the way in their own commit. + +## Writing regression tests + +Follow the existing pattern in `app/tests/api//*.spec.js` (mocha + +supertest + expect; see `docs/developer/crud-regression-test-taxonomy.md`): + +- `before()`: `database.initializeConnection()` → + `databaseConfiguration.checkSystemConfiguration()` → set + `config.validateRequests` flags → `initializeApp()` → `login.loginAnonymous()`. +- **Always enable ADM validation** (`config.validateRequests.withAttackDataModel + = true`) and make payloads ADM-valid — check the ADM Zod sources when unsure. + Common traps: `kill_chain_phases[].kill_chain_name` must be + `mitre-attack` / `mitre-mobile-attack` / `mitre-ics-attack`; + `x_mitre_platforms` must use real platform names (e.g. `Windows`). +- Account for server-controlled fields: read generated values + (`workspace.attack_id`, `stix.created_by_ref`, ATT&CK external refs) from the + POST response rather than asserting on what you sent. To simulate states the + API won't accept on create (`revoked`, missing ATT&CK ID), update the + document directly via the Mongoose model. +- Startup seeds four static marking definitions (e.g. TLP:WHITE + `marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9`) and a placeholder + organization identity; the MITRE identity is *not* seeded. + +## Bruno smoke tests + +The Bruno collection (location in `AGENTS.local.md`) mirrors the API for +manual testing — one `.bru` file per request, grouped in folders, environments +in `environments/`. When changing the API surface, update the affected `.bru` +files: keep the `url` line consistent with enabled `params:query` entries, add +new optional params as disabled toggles (`~name: value`), and document +parameter semantics in the `docs { }` block. + +## Gotchas + +- STIX version rules: the bundle envelope carries `spec_version` only in STIX + 2.0 (2.1 removed it; each 2.1 *object* declares its own `spec_version`). + Marking definitions have no `stix.modified`. +- `p-limit` is not a dependency and recent versions are ESM-only — use a small + inline concurrency runner instead. +- Legacy endpoints under deprecation (e.g. `GET /api/stix-bundles`) are + replaced by release-tracks equivalents — check + `docs/developer/release-tracks/bundle-export.md` before extending them. + +## Maintaining this guide + +Treat this file like code. At the end of a task, consider whether a durable, +non-obvious lesson was learned (a validation trap, a pattern decision, a +workflow correction) and propose adding it here; prune entries that are stale +or no longer earn their token cost — this file is loaded into every agent +session. Machine-specific paths never belong in this file; they go in +`AGENTS.local.md`. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..7d706559 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,8 @@ +# CLAUDE.md + +The canonical agent guide for this repository is AGENTS.md (shared across all +coding agents; edit that file, not this one). Machine-specific paths live in +AGENTS.local.md (gitignored; copy from AGENTS.local.example.md if missing). + +@AGENTS.md +@AGENTS.local.md From e8cf697f00196a9c4536fd4e68af8bf42890d6c4 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:05:06 -0400 Subject: [PATCH 04/55] test: stabilize the in-memory test database across spec files Fix two flake sources in the mocha suites: - Reuse a single mongodb-memory-server instance for all spec files in the process. Per-file stop/start intermittently failed with "Port already in use" (a port not yet released by the previous instance), breaking that file's before() hook and cascading failures through the whole file. closeConnection now drops the database and disconnects but keeps the server running; the mocha scripts run with --exit. - Explicitly rebuild schema indexes after each reconnect. Dropping the database also drops its indexes, and mongoose's per-model init() is memoized per process, so unique-index constraints (stix.id + stix.modified) intermittently vanished for later spec files, letting duplicate-POST tests and dependent count assertions fail in roaming pairs. Also records the (now historic) flake signature in the AGENTS.md gotchas. --- AGENTS.md | 7 +++++++ app/lib/database-in-memory.js | 21 +++++++++++++++++---- package.json | 14 +++++++------- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c473875f..7c8fa090 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,6 +156,13 @@ parameter semantics in the `docs { }` block. - Legacy endpoints under deprecation (e.g. `GET /api/stix-bundles`) are replaced by release-tracks equivalents — check `docs/developer/release-tracks/bundle-export.md` before extending them. +- Historic full-suite flake (fixed 2026-07-10): per-spec-file mongod + restarts hit "Port already in use", failing a random file's `before` hook + (visible as `loginAnonymous` 404s). `database-in-memory.js` now reuses one + mongod across spec files and the mocha scripts use `--exit`. If roaming + single-file failures reappear, re-run that spec file in isolation before + treating them as real, and check mongod startup errors at the top of the + run output. ## Maintaining this guide diff --git a/app/lib/database-in-memory.js b/app/lib/database-in-memory.js index 3666b4ff..66564f09 100644 --- a/app/lib/database-in-memory.js +++ b/app/lib/database-in-memory.js @@ -5,6 +5,11 @@ const logger = require('./logger'); let mongod; exports.initializeConnection = async function () { + // Reuse a single MongoMemoryServer for all spec files in the process. + // Starting a fresh mongod per spec file intermittently collides with a + // port the previous instance has not fully released ("Port already in + // use"), which fails the spec's before() hook and cascades failures + // through that whole file. if (!mongod) { mongod = await MongoMemoryServer.create(); } @@ -24,16 +29,24 @@ exports.initializeConnection = async function () { } catch (error) { handleError(error); } + + // Rebuild schema indexes for models compiled in an earlier spec file. + // closeConnection drops the database (including its indexes), and + // mongoose's per-model init() is memoized per process — without this, + // unique-index constraints (e.g. stix.id + stix.modified) intermittently + // vanish for later spec files. + await Promise.all(Object.values(mongoose.models).map((model) => model.createIndexes())); + logger.info('Mongoose connected to ' + uri); }; exports.closeConnection = async function () { - if (mongod) { + // Drop data and disconnect, but leave the mongod instance running for the + // next spec file. The mocha scripts run with --exit, so the process does + // not linger after the last spec. + if (mongod && mongoose.connection.readyState !== 0) { await mongoose.connection.dropDatabase(); await mongoose.connection.close(); - await mongod.stop(); - - mongod = null; } }; diff --git a/package.json b/package.json index 914f9954..6b446070 100644 --- a/package.json +++ b/package.json @@ -27,15 +27,15 @@ "format": "npm run prettier:fix && npm run lint:fix", "start": "node ./bin/www", "test": "npm run test:openapi && npm run test:config && npm run test:api && npm run test:middleware", - "test:api": "mocha --timeout 20000 --recursive ./app/tests/api", - "test:config": "mocha --timeout 20000 --recursive ./app/tests/config", - "test:import": "mocha --timeout 20000 --recursive ./app/tests/import", - "test:openapi": "mocha --timeout 20000 ./app/tests/openapi", + "test:api": "mocha --timeout 20000 --recursive ./app/tests/api --exit", + "test:config": "mocha --timeout 20000 --recursive ./app/tests/config --exit", + "test:import": "mocha --timeout 20000 --recursive ./app/tests/import --exit", + "test:openapi": "mocha --timeout 20000 ./app/tests/openapi --exit", "test:middleware": "mocha --timeout 20000 ./app/tests/middleware --exit", "test:authn": "./app/tests/run-mocha-separate-jobs.sh ./app/tests/authn", - "test:fuzz": "mocha --timeout 10000 --recursive ./app/tests/fuzz", - "test:scheduler": "mocha --timeout 60000 --recursive ./app/tests/scheduler", - "test:file": "mocha --timeout 10000", + "test:fuzz": "mocha --timeout 10000 --recursive ./app/tests/fuzz --exit", + "test:scheduler": "mocha --timeout 60000 --recursive ./app/tests/scheduler --exit", + "test:file": "mocha --timeout 10000 --exit", "check:lockfile": "bash scripts/check-package-lock.sh" }, "dependencies": { From a83c03262d8fcffbf1873980d0c89b99c60de6b0 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:09:18 -0400 Subject: [PATCH 05/55] feat(release-tracks): add bidirectional refs between objects and tracks Add workspace.release_tracks backrefs ([{ id, tier, status }]) to STIX object documents so release-track membership is visible from any standard object getter without scanning tracks. Tier values match the snapshot tier array names (members/staged/candidates/quarantine), consistent with the tier field on the object-versions endpoint. Backrefs are maintained by snapshot-driven reconciliation: every snapshot persistence choke point (cloneSnapshot, track clone, snapshot/track deletion, bump) emits release-track::contents-changed with the track's latest snapshot; AttackObjectsService and RelationshipsService listeners diff desired vs current entries and bulk-write the difference. The design is idempotent and self-healing, covering all membership mutation routes with one code path. Sparse multikey indexes on workspace.release_tracks.id support the reverse lookups. The field is server-controlled: stripped from client create/update/import input and excluded from every revision-clone path (revoke, relationship deprecation, technique conversion, identity propagation). The regression suite for this feature lands at the end of this commit series (release-tracks-backrefs.spec.js), since it also covers the follow-up behavior changes that build on the same infrastructure. --- app/api/definitions/components/workspace.yml | 22 ++ app/lib/event-constants.js | 7 + app/lib/release-tracks/backref-reconciler.js | 199 ++++++++++++++++++ app/models/attack-object-model.js | 4 + app/models/relationship-model.js | 4 + app/models/subschemas/workspace.js | 19 ++ app/repository/_base.repository.js | 90 ++++++++ app/services/meta-classes/base.service.js | 12 ++ .../release-tracks/snapshot-service.js | 32 +++ .../release-tracks/versioning-service.js | 6 + app/services/stix/attack-objects-service.js | 35 ++- app/services/stix/relationships-service.js | 40 ++++ app/services/stix/techniques-service.js | 6 + docs/README.md | 2 + docs/developer/event-bus-architecture.md | 1 + .../release-tracks/backref-reconciliation.md | 131 ++++++++++++ docs/developer/release-tracks/entities.md | 41 ++-- docs/user/release-tracks/object-backrefs.md | 77 +++++++ 18 files changed, 703 insertions(+), 25 deletions(-) create mode 100644 app/lib/release-tracks/backref-reconciler.js create mode 100644 docs/developer/release-tracks/backref-reconciliation.md create mode 100644 docs/user/release-tracks/object-backrefs.md diff --git a/app/api/definitions/components/workspace.yml b/app/api/definitions/components/workspace.yml index 32051cbf..fd145501 100644 --- a/app/api/definitions/components/workspace.yml +++ b/app/api/definitions/components/workspace.yml @@ -14,9 +14,31 @@ components: type: array items: $ref: '#/components/schemas/collection_reference' + release_tracks: + type: array + description: 'Server-controlled. Reverse pointers to the release tracks whose current (latest) snapshot references this object revision. Maintained automatically as objects move through release-track tiers; client-supplied values are ignored.' + items: + $ref: '#/components/schemas/release_track_reference' attack_id: type: string description: 'ATT&CK ID (e.g., T1234, G0001). When creating a new version of an existing object, this must match the existing attack_id. When creating a new object, this field is generated by the backend and cannot be set.' + release_track_reference: + type: object + properties: + id: + type: string + description: 'The release track ID (release-track--)' + tier: + type: string + enum: ['members', 'staged', 'candidates', 'quarantine'] + description: 'The tier of the release track that references this object revision; values match the snapshot tier array names' + status: + type: string + enum: ['work-in-progress', 'awaiting-review', 'reviewed'] + description: 'Track-scoped workflow status. Members are always reviewed; quarantined entries carry no status.' + required: + - id + - tier collection_reference: type: object properties: diff --git a/app/lib/event-constants.js b/app/lib/event-constants.js index 57afcd72..fdfe22d9 100644 --- a/app/lib/event-constants.js +++ b/app/lib/event-constants.js @@ -158,4 +158,11 @@ module.exports = Object.freeze({ // Validation VALIDATION_BYPASS_CHECK_REQUESTED: 'validation-bypass::check-requested', + + // Release Tracks + // Emitted after any persisted change to a release track's current (latest) + // snapshot. Payload: { trackId, snapshot } where snapshot is the track's + // latest snapshot, or null when the track (or its only snapshot) was deleted. + // Listeners reconcile workspace.release_tracks backrefs on their own documents. + RELEASE_TRACK_CONTENTS_CHANGED: 'release-track::contents-changed', }); diff --git a/app/lib/release-tracks/backref-reconciler.js b/app/lib/release-tracks/backref-reconciler.js new file mode 100644 index 00000000..b6f9b49c --- /dev/null +++ b/app/lib/release-tracks/backref-reconciler.js @@ -0,0 +1,199 @@ +'use strict'; + +// ============================================================================= +// Release Track Backref Reconciler +// +// Maintains the reverse pointers (`workspace.release_tracks`) that STIX object +// documents carry back to the release tracks that reference them. Each entry +// has the shape: +// +// { +// id: 'release-track--', +// tier: 'members'|'staged'|'candidates'|'quarantine', +// status: 'work-in-progress'|'awaiting-review'|'reviewed' +// } +// +// Backrefs are pinned to specific object revisions: the entry lives on the +// exact (stix.id, stix.modified) document that the track's tier entry pins. +// +// Reconciliation is snapshot-driven and idempotent: given a track's current +// (latest) snapshot, compute the desired set of backrefs and diff it against +// the documents that currently carry an entry for that track. This single +// code path covers every membership mutation (add/remove/review/promote/ +// demote/bump/member-sync/clone/bundle-import/updateContents) as well as +// snapshot deletion (membership reverts to the new latest snapshot) and +// track deletion (snapshot = null removes all entries). +// +// Called from EventBus listeners (RELEASE_TRACK_CONTENTS_CHANGED) in +// attack-objects-service and relationships-service — each service reconciles +// only the documents in its own collection, selected via `includeRef`. +// ============================================================================= + +const logger = require('../logger'); + +// Snapshot tier array names, also used verbatim as the backref `tier` value. +// Order matters: if a revision somehow appears in multiple tiers, the first +// tier listed here wins. +const TIERS = ['members', 'staged', 'candidates', 'quarantine']; + +function versionKey(objectRef, objectModified) { + return `${objectRef}|${new Date(objectModified).toISOString()}`; +} + +/** + * Derive the backref status for a tier entry. + * Members are inherently 'reviewed'; quarantined entries (virtual tracks) + * carry no workflow status. + */ +function entryStatus(tierName, entry) { + switch (tierName) { + case 'members': + return 'reviewed'; + case 'staged': + return entry.object_status || 'reviewed'; + case 'candidates': + return entry.object_status || 'work-in-progress'; + default: + return entry.object_status || undefined; + } +} + +/** + * Compute the desired backref entries from a snapshot. + * + * @param {Object|null} snapshot - The track's latest snapshot (null = no membership) + * @param {function(string): boolean} includeRef - Filter on object_ref; lets each + * collection's listener reconcile only its own documents + * @returns {Map} + */ +function computeDesiredEntries(snapshot, includeRef) { + const desired = new Map(); + if (!snapshot) return desired; + + for (const tierName of TIERS) { + for (const entry of snapshot[tierName] || []) { + if (!includeRef(entry.object_ref)) continue; + + const key = versionKey(entry.object_ref, entry.object_modified); + if (desired.has(key)) continue; // earlier tier wins + + desired.set(key, { + objectRef: entry.object_ref, + objectModified: entry.object_modified, + tier: tierName, + status: entryStatus(tierName, entry), + }); + } + } + + return desired; +} + +/** + * Reconcile workspace.release_tracks backrefs for one track against one + * document collection. + * + * @param {Object} repository - A BaseRepository instance (provides + * retrieveReleaseTrackRefsLean, retrieveVersionRefsLean, bulkWrite) + * @param {string} trackId - The release track ID + * @param {Object|null} snapshot - The track's latest snapshot (null = remove all) + * @param {function(string): boolean} includeRef - Filter on object_ref + * @returns {Promise<{added: number, updated: number, removed: number}>} + */ +async function reconcile(repository, trackId, snapshot, includeRef) { + const desired = computeDesiredEntries(snapshot, includeRef); + const current = await repository.retrieveReleaseTrackRefsLean(trackId); + + const operations = []; + const counts = { added: 0, updated: 0, removed: 0 }; + const satisfied = new Set(); + + for (const document of current) { + const key = versionKey(document.stix.id, document.stix.modified); + const want = desired.get(key); + + if (!want) { + operations.push({ + updateOne: { + filter: { _id: document._id }, + update: { $pull: { 'workspace.release_tracks': { id: trackId } } }, + }, + }); + counts.removed++; + continue; + } + + satisfied.add(key); + const existing = (document.workspace.release_tracks || []).find((e) => e.id === trackId); + if (existing && existing.tier === want.tier && (existing.status || undefined) === want.status) { + continue; // already correct + } + + const update = { $set: { 'workspace.release_tracks.$.tier': want.tier } }; + if (want.status === undefined) { + update.$unset = { 'workspace.release_tracks.$.status': '' }; + } else { + update.$set['workspace.release_tracks.$.status'] = want.status; + } + operations.push({ + updateOne: { + filter: { _id: document._id, 'workspace.release_tracks.id': trackId }, + update, + }, + }); + counts.updated++; + } + + // Add entries to pinned revisions that don't carry one yet + const missing = [...desired.entries()].filter(([key]) => !satisfied.has(key)); + if (missing.length > 0) { + const revisions = await repository.retrieveVersionRefsLean( + missing.map(([, want]) => ({ + object_ref: want.objectRef, + object_modified: want.objectModified, + })), + ); + const documentsByKey = new Map( + revisions.map((doc) => [versionKey(doc.stix.id, doc.stix.modified), doc]), + ); + + for (const [key, want] of missing) { + const document = documentsByKey.get(key); + if (!document) { + // Pinned revision does not exist in this collection — either it lives + // in the other collection (handled by that listener) or the pin is + // dangling. Reconciliation self-heals on the next contents change. + continue; + } + + const entry = { id: trackId, tier: want.tier }; + if (want.status !== undefined) { + entry.status = want.status; + } + operations.push({ + updateOne: { + filter: { _id: document._id }, + update: { $push: { 'workspace.release_tracks': entry } }, + }, + }); + counts.added++; + } + } + + if (operations.length > 0) { + await repository.bulkWrite(operations); + logger.verbose( + `BackrefReconciler: track "${trackId}" — added ${counts.added}, ` + + `updated ${counts.updated}, removed ${counts.removed} backref(s)`, + ); + } + + return counts; +} + +module.exports = { + reconcile, + // exported for unit testing + computeDesiredEntries, + versionKey, +}; diff --git a/app/models/attack-object-model.js b/app/models/attack-object-model.js index 2349b451..51d829ff 100644 --- a/app/models/attack-object-model.js +++ b/app/models/attack-object-model.js @@ -46,6 +46,10 @@ const attackObjectSchema = new mongoose.Schema(attackObjectDefinition, options); // This improves the efficiency of queries and enforces uniqueness on this combination of properties attackObjectSchema.index({ 'stix.id': 1, 'stix.modified': -1 }, { unique: true }); +// Multikey index supporting reverse lookups from release tracks +// (release-track backref reconciliation queries by workspace.release_tracks.id) +attackObjectSchema.index({ 'workspace.release_tracks.id': 1 }, { sparse: true }); + // Create the model const attackObjectModel = mongoose.model('AttackObject', attackObjectSchema); diff --git a/app/models/relationship-model.js b/app/models/relationship-model.js index a01194e9..223aa4d6 100644 --- a/app/models/relationship-model.js +++ b/app/models/relationship-model.js @@ -40,6 +40,10 @@ const relationshipSchema = new mongoose.Schema(relationshipDefinition); relationshipSchema.index({ 'stix.id': 1, 'stix.modified': -1 }, { unique: true }); +// Multikey index supporting reverse lookups from release tracks +// (release-track backref reconciliation queries by workspace.release_tracks.id) +relationshipSchema.index({ 'workspace.release_tracks.id': 1 }, { sparse: true }); + // Create the model const RelationshipModel = mongoose.model(ModelName.Relationship, relationshipSchema); diff --git a/app/models/subschemas/workspace.js b/app/models/subschemas/workspace.js index 7bedcd31..0190e8c0 100644 --- a/app/models/subschemas/workspace.js +++ b/app/models/subschemas/workspace.js @@ -30,6 +30,24 @@ const validationIssue = { }; const validationIssueSchema = new mongoose.Schema(validationIssue, { _id: false }); +const releaseTrackRef = { + id: { type: String, required: true }, + // Which tier of the track references this revision; values match the + // snapshot tier array names. + tier: { + type: String, + enum: ['members', 'staged', 'candidates', 'quarantine'], + required: true, + }, + // Track-scoped workflow status. Members are inherently 'reviewed'; + // quarantined entries (virtual tracks) carry no status. + status: { + type: String, + enum: ['work-in-progress', 'awaiting-review', 'reviewed'], + }, +}; +const releaseTrackRefSchema = new mongoose.Schema(releaseTrackRef, { _id: false }); + /** * Workspace property definition for most object types */ @@ -43,6 +61,7 @@ module.exports.common = { }, attack_id: String, collections: [collectionVersionSchema], + release_tracks: { type: [releaseTrackRefSchema], default: undefined }, embedded_relationships: { type: [embeddedRelationshipSchema], default: undefined }, validation: { errors: { type: [validationIssueSchema], default: undefined }, diff --git a/app/repository/_base.repository.js b/app/repository/_base.repository.js index fbd7abc7..7140cd57 100644 --- a/app/repository/_base.repository.js +++ b/app/repository/_base.repository.js @@ -527,6 +527,96 @@ class BaseRepository extends AbstractRepository { } } + /** + * Retrieve the workspace.release_tracks backrefs of one object revision. + * Lean, minimal projection — used to refresh a create/update response + * after domain-event listeners (member sync → backref reconciliation) + * may have stamped backrefs onto the persisted document. + * + * @param {string} stixId - The STIX ID + * @param {Date|string} stixModified - The revision's modified timestamp + * @returns {Promise} The release_tracks entries, if any + */ + async retrieveBackrefsByVersionLean(stixId, stixModified) { + try { + const document = await this.model + .findOne({ 'stix.id': stixId, 'stix.modified': new Date(stixModified) }) + .select('workspace.release_tracks') + .lean() + .exec(); + return document?.workspace?.release_tracks; + } catch (err) { + throw new DatabaseError(err); + } + } + + /** + * Retrieve all documents carrying a workspace.release_tracks entry for the + * given release track. Lean, minimal projection — used by release-track + * backref reconciliation. + * + * @param {string} trackId - The release track ID + * @returns {Promise} Lean documents with _id, stix.id, stix.modified, workspace.release_tracks + */ + async retrieveReleaseTrackRefsLean(trackId) { + try { + return await this.model + .find({ 'workspace.release_tracks.id': trackId }) + .select('_id stix.id stix.modified workspace.release_tracks') + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + /** + * Resolve specific object revisions to their document _ids. Lean, minimal + * projection — used by release-track backref reconciliation. + * + * @param {Array<{object_ref: string, object_modified: Date|string}>} versions + * @returns {Promise} Lean documents with _id, stix.id, stix.modified + */ + async retrieveVersionRefsLean(versions) { + const BATCH_SIZE = 500; + try { + const results = []; + for (let i = 0; i < versions.length; i += BATCH_SIZE) { + const batch = versions.slice(i, i + BATCH_SIZE); + const documents = await this.model + .find({ + $or: batch.map((v) => ({ + 'stix.id': v.object_ref, + 'stix.modified': new Date(v.object_modified), + })), + }) + .select('_id stix.id stix.modified') + .lean() + .exec(); + results.push(...documents); + } + return results; + } catch (err) { + throw new DatabaseError(err); + } + } + + /** + * Execute a set of bulk write operations, batched to bound memory usage. + * + * @param {Object[]} operations - MongoDB bulkWrite operations + */ + async bulkWrite(operations) { + const BATCH_SIZE = 500; + try { + for (let i = 0; i < operations.length; i += BATCH_SIZE) { + await this.model.bulkWrite(operations.slice(i, i + BATCH_SIZE), { ordered: false }); + } + } catch (err) { + throw new DatabaseError(err); + } + } + async unsetField(documentId, fieldPath) { try { return await this.model.updateOne({ _id: documentId }, { $unset: { [fieldPath]: '' } }); diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 62aa8cba..820034ec 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -370,8 +370,12 @@ class BaseService extends ServiceWithHooks { // Strip workspace.validation — server-controlled; recomputed on every // create/update so a stale entry from a prior GET cannot ride along. + // Strip workspace.release_tracks — server-controlled; maintained by + // release-track backref reconciliation, and pinned to specific revisions, + // so a copy from a prior GET must not ride along onto a new version. if (data.workspace) { delete data.workspace.validation; + delete data.workspace.release_tracks; } if (!options.preserveAttackId) { @@ -776,8 +780,11 @@ class BaseService extends ServiceWithHooks { async composeForImport(data, options) { // Strip workspace.validation — server-controlled; the fail-open block // below is the only legitimate writer of this field on the import path. + // Strip workspace.release_tracks — server-controlled (see + // stripServerControlledFields); imported objects must not claim membership. if (data.workspace) { delete data.workspace.validation; + delete data.workspace.release_tracks; } // Extract ATT&CK ID from external_references and propagate to workspace.attack_id @@ -1081,6 +1088,11 @@ class BaseService extends ServiceWithHooks { delete objectAData.__t; objectAData.stix.revoked = true; objectAData.stix.modified = new Date().toISOString(); + // Release-track backrefs are pinned to specific revisions — the new + // revoked revision is not referenced by any track. + if (objectAData.workspace) { + delete objectAData.workspace.release_tracks; + } if (options.userAccountId) { objectAData.workspace = objectAData.workspace || {}; objectAData.workspace.workflow = objectAData.workspace.workflow || {}; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 674b1674..eaa62a6b 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -17,6 +17,8 @@ const registryRepo = require('../../repository/release-tracks/release-track-regi const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const modelFactory = require('../../models/release-tracks/model-factory'); const logger = require('../../lib/logger'); +const EventBus = require('../../lib/event-bus'); +const EventConstants = require('../../lib/event-constants'); const { TrackNotFoundError, NotFoundError } = require('../../exceptions'); // ============================================================================= @@ -73,6 +75,22 @@ async function syncRegistryCounters(trackId) { }); } +/** + * Notify listeners that a track's current (latest) snapshot changed so they + * can reconcile workspace.release_tracks backrefs on their own documents. + * + * Emissions are awaited (request/response blocking): backrefs are consistent + * by the time the triggering API call returns. + * + * @param {string} trackId + * @param {Object|null} snapshot - The track's latest snapshot, or null when the + * track (or its only snapshot) was deleted + */ +async function emitContentsChanged(trackId, snapshot) { + await EventBus.emit(EventConstants.RELEASE_TRACK_CONTENTS_CHANGED, { trackId, snapshot }); +} +exports.emitContentsChanged = emitContentsChanged; + // ============================================================================= // Track management // ============================================================================= @@ -226,6 +244,9 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov const saved = await dynamicRepo.saveSnapshot(trackId, clone); await syncRegistryCounters(trackId); + // The clone (modified = now) is the track's new latest snapshot + await emitContentsChanged(trackId, saved); + logger.verbose(`SnapshotService: Cloned snapshot for track "${trackId}"`); return saved; }; @@ -290,6 +311,9 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { updated_at: now, }); + // The new track's initial snapshot carries the source track's contents + await emitContentsChanged(newTrackId, saved); + logger.verbose(`SnapshotService: Cloned track to new track "${clone.name}" (${newTrackId})`); return saved; } @@ -486,6 +510,9 @@ exports.deleteTrack = async function deleteTrack(trackId) { await dynamicRepo.dropCollection(trackId); await registryRepo.deleteByTrackId(trackId); + // Remove all backrefs to the deleted track + await emitContentsChanged(trackId, null); + logger.verbose(`SnapshotService: Deleted track "${trackId}"`); }; @@ -507,5 +534,10 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { await dynamicRepo.deleteSnapshot(trackId, modified); await syncRegistryCounters(trackId); + // Deleting the latest snapshot reverts membership to the previous snapshot + // (or clears it if no snapshots remain) + const latest = await dynamicRepo.getLatestSnapshot(trackId); + await emitContentsChanged(trackId, latest); + logger.verbose(`SnapshotService: Deleted snapshot '${modified}' from track "${trackId}"`); }; diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 2b015c93..8150b163 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -132,6 +132,12 @@ async function _doBump(trackId, snapshot, options) { updated_at: now, }); + // The staged → members promotion changed tier membership. Re-read the + // latest snapshot rather than using `tagged` — bumpByModified may have + // tagged an older snapshot, and backrefs track the latest one. + const latest = await dynamicRepo.getLatestSnapshot(trackId); + await snapshotService.emitContentsChanged(trackId, latest); + logger.verbose( `VersioningService: Tagged track "${trackId}" as v${version} ` + `(promoted ${promotedCount} staged → members)`, diff --git a/app/services/stix/attack-objects-service.js b/app/services/stix/attack-objects-service.js index 9809b635..d7e97a3e 100644 --- a/app/services/stix/attack-objects-service.js +++ b/app/services/stix/attack-objects-service.js @@ -199,9 +199,39 @@ class AttackObjectsService extends BaseService { AttackObjectsService.handleOrganizationIdentityChanged, ); + EventBus.on( + Events.RELEASE_TRACK_CONTENTS_CHANGED, + AttackObjectsService.handleReleaseTrackContentsChanged, + ); + logger.info('AttackObjectsService: Event listeners initialized'); } + /** + * Reconcile workspace.release_tracks backrefs on attackObjects documents + * when a release track's contents change. Covers every STIX type stored in + * the attackObjects collection; relationship refs are handled by + * RelationshipsService (separate collection). + * + * @param {Object} payload - { trackId, snapshot } (snapshot null = track deleted) + */ + static async handleReleaseTrackContentsChanged(payload) { + const backrefReconciler = require('../../lib/release-tracks/backref-reconciler'); + + try { + await backrefReconciler.reconcile( + attackObjectsRepository, + payload.trackId, + payload.snapshot, + (objectRef) => !objectRef.startsWith('relationship--'), + ); + } catch (error) { + logger.error( + `AttackObjectsService: Error reconciling release track backrefs for ${payload.trackId}: ${error.message}`, + ); + } + } + /** * Handle organization identity changes by creating new versions of affected objects. * Objects are updated based on field-specific provenance: @@ -238,12 +268,15 @@ class AttackObjectsService extends BaseService { ); const newVersion = { - workspace: obj.workspace, + workspace: { ...obj.workspace }, stix: { ...obj.stix, modified: new Date().toISOString(), }, }; + // Release-track backrefs are pinned to specific revisions — the new + // revision is not referenced by any track. + delete newVersion.workspace.release_tracks; if (createdByInHistory) { newVersion.stix.created_by_ref = newIdentityRef; diff --git a/app/services/stix/relationships-service.js b/app/services/stix/relationships-service.js index 1d275186..68eb89d3 100644 --- a/app/services/stix/relationships-service.js +++ b/app/services/stix/relationships-service.js @@ -56,9 +56,39 @@ class RelationshipsService extends BaseService { this.handleSubtechniqueConvertedToTechnique.bind(this), ); + EventBus.on( + EventConstants.RELEASE_TRACK_CONTENTS_CHANGED, + this.handleReleaseTrackContentsChanged.bind(this), + ); + logger.info('RelationshipsService: Event listeners initialized'); } + /** + * Reconcile workspace.release_tracks backrefs on relationship documents + * when a release track's contents change. Relationships live in their own + * collection, so this service handles the relationship refs while + * AttackObjectsService handles everything else. + * + * @param {Object} payload - { trackId, snapshot } (snapshot null = track deleted) + */ + static async handleReleaseTrackContentsChanged(payload) { + const backrefReconciler = require('../../lib/release-tracks/backref-reconciler'); + + try { + await backrefReconciler.reconcile( + relationshipsRepository, + payload.trackId, + payload.snapshot, + (objectRef) => objectRef.startsWith('relationship--'), + ); + } catch (error) { + logger.error( + `RelationshipsService: Error reconciling release track backrefs for ${payload.trackId}: ${error.message}`, + ); + } + } + /** * Create a subtechnique-of SRO when a technique is converted to a subtechnique. * @@ -156,6 +186,11 @@ class RelationshipsService extends BaseService { deprecatedVersion.stix.x_mitre_deprecated = true; deprecatedVersion.stix.modified = new Date().toISOString(); + // Backrefs are pinned to the exact revision a track references — + // never carried onto a new revision. + if (deprecatedVersion.workspace) { + delete deprecatedVersion.workspace.release_tracks; + } const saved = await relationshipsRepository.save(deprecatedVersion); deprecatedDocs.push(saved); @@ -221,6 +256,11 @@ class RelationshipsService extends BaseService { relData.stix.x_mitre_deprecated = true; relData.stix.modified = new Date().toISOString(); + // Backrefs are pinned to the exact revision a track references — + // never carried onto a new revision. + if (relData.workspace) { + delete relData.workspace.release_tracks; + } const saved = await relationshipsRepository.save(relData); deprecatedDocs.push(saved); diff --git a/app/services/stix/techniques-service.js b/app/services/stix/techniques-service.js index 73735e1c..2c23cfad 100644 --- a/app/services/stix/techniques-service.js +++ b/app/services/stix/techniques-service.js @@ -335,6 +335,9 @@ class TechniquesService extends BaseService { newVersion.stix.modified = new Date().toISOString(); newVersion.workspace = newVersion.workspace || {}; newVersion.workspace.attack_id = newAttackId; + // Backrefs are pinned to the exact revision a track references — never + // carried onto a new revision. + delete newVersion.workspace.release_tracks; // Rebuild external references: replace ATT&CK ref with the new one const userRefs = removeAttackExternalReferences(newVersion.stix.external_references); @@ -417,6 +420,9 @@ class TechniquesService extends BaseService { newVersion.stix.modified = new Date().toISOString(); newVersion.workspace = newVersion.workspace || {}; newVersion.workspace.attack_id = newAttackId; + // Backrefs are pinned to the exact revision a track references — never + // carried onto a new revision. + delete newVersion.workspace.release_tracks; // Rebuild external references: replace ATT&CK ref with the new one const userRefs = removeAttackExternalReferences(newVersion.stix.external_references); diff --git a/docs/README.md b/docs/README.md index 812e5d9a..8f956eb7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ Guides for consumers of the REST API — endpoints, workflows, and terminology. - [Release Workflow](user/release-tracks/release-workflow.md): Workflow integration and candidacy - [Output Formats](user/release-tracks/output-formats.md): Output format specifications - [Workflow Examples](user/release-tracks/workflow-examples.md): End-to-end workflow examples +- [Object Backrefs](user/release-tracks/object-backrefs.md): Release-track membership pointers on object documents (`workspace.release_tracks`) ## Developer Documentation @@ -39,6 +40,7 @@ Architecture, patterns, and implementation details for contributors. ### Release Tracks (Internals) - [Entities](developer/release-tracks/entities.md): Database schemas and data models +- [Backref Reconciliation](developer/release-tracks/backref-reconciliation.md): How `workspace.release_tracks` backrefs stay in sync with snapshots - [Member Sync Strategies](developer/release-tracks/member-sync-strategies.md): Automatic tracking of member object revisions - [Error Handling](developer/release-tracks/error-handling.md): Error handling patterns - [Implementation Notes](developer/release-tracks/implementation-notes.md): Implementation notes and decisions diff --git a/docs/developer/event-bus-architecture.md b/docs/developer/event-bus-architecture.md index 5ce78bcf..66ad2fc0 100644 --- a/docs/developer/event-bus-architecture.md +++ b/docs/developer/event-bus-architecture.md @@ -160,6 +160,7 @@ Where `{type}` is the STIX type (e.g., `attack-pattern`, `x-mitre-analytic`, `x- | `x-mitre-detection-strategy::analytics-referenced` | DetectionStrategiesService | When detection strategy references analytics (create/update) | `{ detectionStrategyId, detectionStrategy, analyticIds }` | AnalyticsService | | `x-mitre-detection-strategy::analytics-removed` | DetectionStrategiesService | When analytics removed from detection strategy | `{ detectionStrategyId, analyticIds }` | AnalyticsService | | `x-mitre-analytic::parent-changed` | AnalyticsService | When analytic's parent detection strategy changes | `{ analyticId, oldParentId, newParentId, analytic }` | (Future: for cascading updates) | +| `release-track::contents-changed` | snapshot-service / versioning-service | After any persisted change to a track's latest snapshot (or track/snapshot deletion) | `{ trackId, snapshot }` (`snapshot` null when the track or its only snapshot was deleted) | AttackObjectsService, RelationshipsService (reconcile `workspace.release_tracks` backrefs; see [backref-reconciliation.md](release-tracks/backref-reconciliation.md)) | ## Workflow Examples diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md new file mode 100644 index 00000000..d553eaff --- /dev/null +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -0,0 +1,131 @@ +# Release Track Backref Reconciliation + +How and why `workspace.release_tracks` (see the +[user doc](../../user/release-tracks/object-backrefs.md) for the field's +behavior) is kept in sync with release-track snapshots. + +## Why + +Before backrefs, release-track membership was only discoverable from the track +side: answering "which tracks reference this object?" required scanning every +track's latest snapshot for the object's `stix.id` across the `candidates`, +`staged`, `members`, and `quarantine` tiers. The predecessor system (Workbench +collections) solved the same problem with `workspace.collections` backrefs, +maintained imperatively by `AttackObjectsService.insertCollection`. Release +tracks follow that precedent but maintain the pointers event-driven. + +## Why reconciliation instead of incremental updates + +Membership changes through many routes: add/remove candidates, review, +manual and auto promotion, demotion, bump (staged → members), member sync, +`updateContents`, track cloning, bundle import, snapshot deletion, and track +deletion. Patching each route with a bespoke incremental backref update would +be error-prone and would drift. + +Instead, every route already funnels through a small set of persistence choke +points, and each choke point triggers a full **snapshot-driven reconciliation**: +compute the desired backref set from the track's latest snapshot, diff it +against the documents currently carrying an entry for that track, and issue +bulk add/update/remove operations. The reconciler is idempotent and +self-healing — a missed or failed pass is corrected by the next one. + +## Event flow + +``` +snapshot-service.cloneSnapshot ┐ (every tier/config/metadata mutation, +snapshot-service._cloneToNewTrack │ member sync, auto-promotion, +snapshot-service.deleteSnapshot │ bundle import, updateContents, ...) +snapshot-service.deleteTrack │ +versioning-service._doBump ┘ (staged → members via tagSnapshotInPlace) + │ + ▼ awaited EventBus.emit release-track::contents-changed { trackId, snapshot } + │ snapshot = track's latest snapshot, + │ or null when the track (or its only + │ snapshot) was deleted + │ + ├──► AttackObjectsService.handleReleaseTrackContentsChanged + │ reconciles the attackObjects collection + │ (refs where !object_ref.startsWith('relationship--')) + │ + └──► RelationshipsService.handleReleaseTrackContentsChanged + reconciles the relationships collection + (refs where object_ref.startsWith('relationship--')) +``` + +Two listeners because relationships live in their own MongoDB collection; +per the event-bus ownership rules each service modifies only its own +documents. Both delegate to the shared logic in +`app/lib/release-tracks/backref-reconciler.js`, parameterized by repository +and an `includeRef` predicate. + +`createTrack` does not emit — a brand-new track's tiers are empty and nothing +can reference its ID yet. `bumpByModified` may tag an older snapshot; the bump +path therefore re-reads the *latest* snapshot before emitting rather than +using the tagged one. + +Emissions are awaited (the request/response-blocking convention), so backrefs +are consistent by the time the triggering API call returns. + +## Reconciliation algorithm + +For one `(repository, trackId, snapshot, includeRef)`: + +1. **Desired set** — walk the snapshot tiers in order `members`, `staged`, + `candidates`, `quarantine` (first tier wins if a revision somehow appears + twice), keyed by `(object_ref, object_modified)`. Status mapping: + members → `reviewed`; staged/candidates → the entry's `object_status`; + quarantine → none. +2. **Current set** — `find({ 'workspace.release_tracks.id': trackId })`, + supported by a sparse multikey index on both collections. +3. **Diff → bulkWrite** (batched, unordered): + - current but not desired → `$pull` the track's entry; + - both, but phase/status differ → positional `$set`/`$unset`; + - desired but not current → resolve the pinned revision to its `_id` + (batched `$or` on the `stix.id + stix.modified` index) and `$push` the + entry. Pins whose revision document doesn't exist (dangling pin, or a + ref belonging to the other collection) are skipped. + +Repository support lives in `BaseRepository` +(`retrieveReleaseTrackRefsLean`, `retrieveVersionRefsLean`, `bulkWrite`), so +both `attackObjectsRepository` and `relationshipsRepository` inherit it. + +## Server-controlled invariants + +`workspace.release_tracks` is stripped from client input in +`BaseService.stripServerControlledFields` (create/update) and +`composeForImport` (import), alongside `workspace.validation`. Because +backrefs are pinned to specific revisions, code paths that clone a document +into a *new* revision must not carry the field forward; this is handled in: + +- `BaseService.revoke` (revoked revision clone), +- `AttackObjectsService.handleOrganizationIdentityChanged` (identity + propagation clones), +- `RelationshipsService.handleObjectRevoked` and + `handleSubtechniqueConvertedToTechnique` (relationship deprecation clones), +- `TechniquesService.convertToSubtechnique` / `convertToTechnique` + (conversion clones). + +(Clones routed through `create()` — e.g. the relationship *transfer* during +revoke — are already covered by `stripServerControlledFields`.) + +Relatedly, revision identity is immutable in place: `BaseService.updateFull` +rejects (400) a PUT whose body `stix.id`/`stix.modified` differ from the path +parameters, so a pinned revision can never be re-keyed out from under a +track's pin (which would strand the pin and orphan the backref). + +New revisions created through `create()` are covered by the strip; if any +track references the object (members, candidates, or staged), member sync +enrolls or re-pins the new revision and the resulting snapshot clone triggers +reconciliation, which stamps the backref on the new revision (see +`member-sync-strategies.md`). + +## Known limitations + +- **Deleted-then-recreated revisions.** If an object revision document is + deleted while pinned by a track, the backref disappears with the document + and the track keeps a dangling pin (pre-existing behavior). If an identical + revision is later re-created, its backref is restored on the next + contents-changed event for that track, not immediately. +- **Historical snapshots.** Backrefs describe only the *latest* snapshot per + track. Membership in older snapshots remains discoverable only from the + track side. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 0c374bf2..f90895bb 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -126,6 +126,7 @@ Each release track snapshot will be tracked as an individual MongoDB Document in status_threshold: "reviewed" }, promotion_conflicts: { + into_candidates: "prefer_latest", // "always_overwrite" | "always_reject" | "prefer_latest" | "abort" candidates_to_staged: "prefer_latest", // "always_overwrite" | "always_reject" | "prefer_latest" staged_to_members: "abort" // "always_overwrite" | "always_reject" | "prefer_latest" | "abort" }, @@ -185,7 +186,11 @@ This provides: ### Object (SDO/SRO/SMO) Document Schema -Objects maintain a simple reference to which release tracks reference them: +Objects maintain a simple reverse reference to the release tracks that +currently reference them (implemented as `workspace.release_tracks`; see +[backref-reconciliation.md](backref-reconciliation.md) for how it is kept in +sync and the [user doc](../../user/release-tracks/object-backrefs.md) for +field semantics): ```javascript { @@ -197,30 +202,17 @@ Objects maintain a simple reference to which release tracks reference them: // ... other STIX properties }, workspace: { - // NO global workflow status - status is tracked per-release-track - - // Simple reverse reference for efficient queries - referenced_by: [ + // Reverse references for efficient "which tracks contain this revision?" queries + release_tracks: [ { - release_track_id: "release-track--123", - snapshot_id: "2024-12-15T16:20:00.000Z", - membership_tier: "members", // "members" | "staged" | "candidates" - review_status: "reviewed" // "work-in-progress" | "awaiting-review" | "reviewed" + id: "release-track--123", + tier: "members", // "members" | "staged" | "candidates" | "quarantine" + status: "reviewed" // "work-in-progress" | "awaiting-review" | "reviewed" }, { - release_track_id: "release-track--456", - snapshot_id: "2025-01-10T11:00:00.000Z", - membership_tier: "candidates", - review_status: "work-in-progress" - } - ], - - // Attribution metadata - workflow_history: [ - { - timestamp: "2024-01-12T09:00:00Z", - modified_by: "alice@example.com", - action: "created" + id: "release-track--456", + tier: "candidates", + status: "work-in-progress" } ] } @@ -228,10 +220,11 @@ Objects maintain a simple reference to which release tracks reference them: ``` **Key Points:** -- **No global `workflow.status`** - status is release-track-specific -- `referenced_by` provides reverse lookup for queries like "show me all release tracks containing this object" +- `workspace.release_tracks` provides reverse lookup for queries like "show me all release tracks containing this object" +- Entries reflect each track's **latest** snapshot and are pinned to the specific object revision the tier entry references - Same object version can have different statuses in different release tracks - Multiple versions of same object can exist, each potentially referenced by different release tracks +- The field is server-controlled and maintained by event-driven reconciliation (`release-track::contents-changed`) ### Virtual Release Track Snapshot Schema diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md new file mode 100644 index 00000000..1eb2d7f7 --- /dev/null +++ b/docs/user/release-tracks/object-backrefs.md @@ -0,0 +1,77 @@ +# Release Track Backrefs on Objects + +Every STIX object document carries reverse pointers to the release tracks that +currently reference it, in `workspace.release_tracks`. This lets you retrieve +an object through any standard getter (e.g. `GET /api/techniques/:stixId`, +`GET /api/attack-objects`) and see its release-track membership without +scanning tracks. + +## Shape + +```json +{ + "workspace": { + "release_tracks": [ + { + "id": "release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d", + "tier": "candidates", + "status": "work-in-progress" + } + ] + }, + "stix": { "...": "..." } +} +``` + +| Field | Values | Meaning | +|-------|--------|---------| +| `id` | `release-track--` | The referencing release track | +| `tier` | `members`, `staged`, `candidates`, `quarantine` | Which tier of the track references this revision; values match the snapshot tier array names | +| `status` | `work-in-progress`, `awaiting-review`, `reviewed` | Track-scoped workflow status | + +An object referenced by multiple tracks carries one entry per track. + +## Semantics + +- **Revision-pinned.** Release-track tiers pin specific object revisions + (`object_ref` + `object_modified`). The backref lives on exactly the pinned + revision document. If a track's candidate pin is moved to a newer revision + (`POST /:id/candidates/:objectRef/update-version`), the backref moves with + it. Different revisions of the same object can carry entries for the same + track — e.g. after member sync auto-enrolls a new revision as a candidate, + the released revision keeps its `members` entry and the new revision gets a + `candidates` entry. +- **Follows new revisions under `track_latest`.** Creating a new revision of + a tracked object keeps the backref on the object's latest revision: for + `members`, the new revision is auto-enrolled as a candidate; for + `candidates`/`staged` pins, the pin (and its backref) moves to the new + revision per the track's member-sync supplant config. Under the `manual` + strategy, pins stay where they are — the old pinned revision keeps the + backref, and the new revision (which the track genuinely does not + reference) has none; use `?versions=all` to see membership across + revisions. +- **Reflects the latest snapshot.** Backrefs mirror the track's *current* + (most recent) snapshot. Deleting the latest snapshot reverts backrefs to the + previous snapshot's membership; deleting a track removes all of its entries. +- **Status mapping.** Candidates and staged entries carry their track-scoped + workflow status. Members are always `reviewed` (promotion to member implies + review). Quarantined entries (virtual tracks) have no workflow status, so + `status` is omitted. +- **Server-controlled.** Like `workspace.attack_id` and + `workspace.validation`, the field is maintained by the server. Values + supplied in `POST`/`PUT` bodies are silently ignored, and updates through + the standard object endpoints cannot remove or alter existing entries. +- **Read-your-own-writes.** `POST`/`PUT` responses include backrefs produced + by the request's own side effects — e.g. when revision sync re-pins a + track to the newly created revision, the response body already carries the + resulting `workspace.release_tracks` entry. + +## Lifecycle example + +``` +POST /api/release-tracks/:id/candidates → { tier: "candidates", status: "work-in-progress" } +POST /api/release-tracks/:id/candidates/review → { tier: "candidates", status: "awaiting-review" } +POST /api/release-tracks/:id/candidates/promote → { tier: "staged", status: "awaiting-review" } +POST /api/release-tracks/:id/bump → { tier: "members", status: "reviewed" } +DELETE /api/release-tracks/:id → entry removed +``` From 5a9bfdff552f3e594143657c4262a6f3ff031e8e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:10:48 -0400 Subject: [PATCH 06/55] fix(api): reject revision re-keying on in-place updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PUT merged body stix.id/stix.modified over the stored document, so an update could silently re-key a revision — stranding release-track pins and orphaning workspace.release_tracks backrefs. updateFull now returns 400 when the body identity fields differ from the path parameters; re-keying must go through POST, which creates a new revision that release-track revision sync captures. Rewrites the legacy PUT regression tests across all SDO types, which encoded the old bump-modified-through-the-body convention. The Angular frontend is unaffected: its PUT factory always serializes the body with the same modified value used in the URL. --- app/services/meta-classes/base.service.js | 20 +++ app/tests/api/analytics/analytics.spec.js | 5 +- app/tests/api/assets/assets.spec.js | 5 +- .../update-identity-guard.spec.js | 119 ++++++++++++++++++ app/tests/api/campaigns/campaigns.spec.js | 5 +- .../data-components/data-components.spec.js | 10 +- .../api/data-sources/data-sources.spec.js | 5 +- .../detection-strategies-spec.js | 8 +- app/tests/api/groups/groups.spec.js | 5 +- app/tests/api/identities/identities.spec.js | 7 +- app/tests/api/matrices/matrices.spec.js | 5 +- app/tests/api/mitigations/mitigations.spec.js | 5 +- app/tests/api/notes/notes.spec.js | 5 +- .../api/relationships/relationships.spec.js | 10 +- app/tests/api/software/software.spec.js | 5 +- app/tests/api/tactics/tactics.spec.js | 5 +- app/tests/api/techniques/techniques.spec.js | 5 +- 17 files changed, 168 insertions(+), 61 deletions(-) create mode 100644 app/tests/api/base-services/update-identity-guard.spec.js diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 820034ec..f4908aad 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -867,6 +867,26 @@ class BaseService extends ServiceWithHooks { throw new MissingParameterError('modified'); } + // Revision identity is immutable in place: a PUT may not re-key the + // document (release tracks pin revisions by stix.id + stix.modified; + // re-keying would strand those pins). Re-keying must go through POST, + // which creates a new revision that member sync captures. + if (data.stix?.id && data.stix.id !== stixId) { + throw new BadRequestError({ + details: `Body stix.id (${data.stix.id}) must match the stixId path parameter (${stixId})`, + }); + } + if ( + data.stix?.modified && + new Date(data.stix.modified).getTime() !== new Date(stixModified).getTime() + ) { + throw new BadRequestError({ + details: + `Body stix.modified (${data.stix.modified}) must match the modified path parameter ` + + `(${stixModified}) — revision identity cannot be changed by an in-place update`, + }); + } + const document = await this.repository.retrieveOneByVersion(stixId, stixModified); if (!document) { return null; diff --git a/app/tests/api/analytics/analytics.spec.js b/app/tests/api/analytics/analytics.spec.js index 4348e4ed..c0499970 100644 --- a/app/tests/api/analytics/analytics.spec.js +++ b/app/tests/api/analytics/analytics.spec.js @@ -185,13 +185,10 @@ describe('Analytics API', function () { }); it('PUT /api/analytics updates a analytic', async function () { - const originalModified = analytic1.stix.modified; - const timestamp = new Date().toISOString(); - analytic1.stix.modified = timestamp; analytic1.stix.description = 'This is an updated analytic.'; const body = analytic1; const res = await request(app) - .put('/api/analytics/' + analytic1.stix.id + '/modified/' + originalModified) + .put('/api/analytics/' + analytic1.stix.id + '/modified/' + analytic1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/assets/assets.spec.js b/app/tests/api/assets/assets.spec.js index c8b9be55..b2c742a4 100644 --- a/app/tests/api/assets/assets.spec.js +++ b/app/tests/api/assets/assets.spec.js @@ -194,13 +194,10 @@ describe('Assets API', function () { }); it('PUT /api/assets updates an asset', async function () { - const originalModified = asset1.stix.modified; - const timestamp = new Date().toISOString(); - asset1.stix.modified = timestamp; asset1.stix.description = 'This is an updated asset.'; const body = asset1; const res = await request(app) - .put('/api/assets/' + asset1.stix.id + '/modified/' + originalModified) + .put('/api/assets/' + asset1.stix.id + '/modified/' + asset1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/base-services/update-identity-guard.spec.js b/app/tests/api/base-services/update-identity-guard.spec.js new file mode 100644 index 00000000..03ea2437 --- /dev/null +++ b/app/tests/api/base-services/update-identity-guard.spec.js @@ -0,0 +1,119 @@ +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +const logger = require('../../../lib/logger'); +logger.level = 'debug'; + +const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { + workflow: { + state: 'work-in-progress', + }, + }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +// Revision identity (stix.id + stix.modified) is immutable in place: a PUT +// whose body identity fields differ from the path parameters must be +// rejected. Release tracks pin revisions by (stix.id, stix.modified) — +// re-keying a document in place would strand those pins. Re-keying goes +// through POST (a new revision) instead. +describe('PUT revision identity guard', function () { + let app; + let passportCookie; + let technique; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + + const res = await request(app) + .post('/api/techniques') + .send(buildTechnique('Identity Guard')) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201); + technique = res.body; + }); + + function putTechnique(body) { + return request(app) + .put(`/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + it('rejects a PUT whose body stix.modified differs from the path parameter', async function () { + const update = buildTechnique('Identity Guard (re-keyed modified)'); + update.stix.id = technique.stix.id; + update.stix.created = technique.stix.created; + update.stix.modified = new Date( + new Date(technique.stix.modified).getTime() + 1000, + ).toISOString(); + + await putTechnique(update).expect(400); + }); + + it('rejects a PUT whose body stix.id differs from the path parameter', async function () { + const update = buildTechnique('Identity Guard (re-keyed id)'); + update.stix.id = 'attack-pattern--00000000-0000-4000-8000-000000000000'; + update.stix.created = technique.stix.created; + update.stix.modified = technique.stix.modified; + + await putTechnique(update).expect(400); + }); + + it('did not alter the stored revision on the rejected PUTs', async function () { + const res = await request(app) + .get(`/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + expect(res.body.stix.name).toBe('Identity Guard'); + expect(res.body.stix.modified).toBe(technique.stix.modified); + }); + + it('accepts a PUT whose body identity matches the path parameters', async function () { + const update = buildTechnique('Identity Guard (updated)'); + update.stix.id = technique.stix.id; + update.stix.created = technique.stix.created; + update.stix.modified = technique.stix.modified; + + const res = await putTechnique(update).expect(200); + expect(res.body.stix.name).toBe('Identity Guard (updated)'); + expect(res.body.stix.modified).toBe(technique.stix.modified); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/campaigns/campaigns.spec.js b/app/tests/api/campaigns/campaigns.spec.js index c500ecc8..2cca1846 100644 --- a/app/tests/api/campaigns/campaigns.spec.js +++ b/app/tests/api/campaigns/campaigns.spec.js @@ -218,13 +218,10 @@ describe('Campaigns API', function () { }); it('PUT /api/campaigns updates a campaign', async function () { - const originalModified = campaign1.stix.modified; - const timestamp = new Date().toISOString(); - campaign1.stix.modified = timestamp; campaign1.stix.description = 'This is an updated campaign. Blue.'; const body = campaign1; const res = await request(app) - .put('/api/campaigns/' + campaign1.stix.id + '/modified/' + originalModified) + .put('/api/campaigns/' + campaign1.stix.id + '/modified/' + campaign1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/data-components/data-components.spec.js b/app/tests/api/data-components/data-components.spec.js index 6637c8a0..592fb400 100644 --- a/app/tests/api/data-components/data-components.spec.js +++ b/app/tests/api/data-components/data-components.spec.js @@ -277,13 +277,15 @@ describe('Data Components API', function () { }); it('PUT /api/data-components updates a data component', async function () { - const originalModified = dataComponent1.stix.modified; - const timestamp = new Date().toISOString(); - dataComponent1.stix.modified = timestamp; dataComponent1.stix.description = 'This is an updated data component.'; const body = dataComponent1; const res = await request(app) - .put('/api/data-components/' + dataComponent1.stix.id + '/modified/' + originalModified) + .put( + '/api/data-components/' + + dataComponent1.stix.id + + '/modified/' + + dataComponent1.stix.modified, + ) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/data-sources/data-sources.spec.js b/app/tests/api/data-sources/data-sources.spec.js index 3cdf4c07..f760ab4a 100644 --- a/app/tests/api/data-sources/data-sources.spec.js +++ b/app/tests/api/data-sources/data-sources.spec.js @@ -248,13 +248,10 @@ describe('Data Sources API', function () { }); it('PUT /api/data-sources updates a data source', async function () { - const originalModified = dataSource1.stix.modified; - const timestamp = new Date().toISOString(); - dataSource1.stix.modified = timestamp; dataSource1.stix.description = 'This is an updated data source.'; const body = cloneForCreate(dataSource1); const res = await request(app) - .put('/api/data-sources/' + dataSource1.stix.id + '/modified/' + originalModified) + .put('/api/data-sources/' + dataSource1.stix.id + '/modified/' + dataSource1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/detection-strategies/detection-strategies-spec.js b/app/tests/api/detection-strategies/detection-strategies-spec.js index 3c041f8e..16d12a1d 100644 --- a/app/tests/api/detection-strategies/detection-strategies-spec.js +++ b/app/tests/api/detection-strategies/detection-strategies-spec.js @@ -263,14 +263,14 @@ describe('Detection Strategies API', function () { }); it('PUT /api/detection-strategies updates a detection strategy', async function () { - const originalModified = detectionStrategy1.stix.modified; - const timestamp = new Date().toISOString(); - detectionStrategy1.stix.modified = timestamp; detectionStrategy1.stix.name = 'This is an updated detection strategy.'; const body = detectionStrategy1; const res = await request(app) .put( - '/api/detection-strategies/' + detectionStrategy1.stix.id + '/modified/' + originalModified, + '/api/detection-strategies/' + + detectionStrategy1.stix.id + + '/modified/' + + detectionStrategy1.stix.modified, ) .send(body) .set('Accept', 'application/json') diff --git a/app/tests/api/groups/groups.spec.js b/app/tests/api/groups/groups.spec.js index c8d7fd3e..cca8ebe8 100644 --- a/app/tests/api/groups/groups.spec.js +++ b/app/tests/api/groups/groups.spec.js @@ -200,13 +200,10 @@ describe('Groups API', function () { }); it('PUT /api/groups updates a group', async function () { - const originalModified = group1.stix.modified; - const timestamp = new Date().toISOString(); - group1.stix.modified = timestamp; group1.stix.description = 'This is an updated group. Blue.'; const body = group1; const res = await request(app) - .put('/api/groups/' + group1.stix.id + '/modified/' + originalModified) + .put('/api/groups/' + group1.stix.id + '/modified/' + group1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/identities/identities.spec.js b/app/tests/api/identities/identities.spec.js index 0c3e6add..3efd1676 100644 --- a/app/tests/api/identities/identities.spec.js +++ b/app/tests/api/identities/identities.spec.js @@ -195,7 +195,7 @@ describe('Identity API', function () { const body = JSON.parse(JSON.stringify(mitreIdentity)); delete body.warnings; body.stix.description = 'Updated MITRE identity description.'; - body.stix.modified = new Date(Date.now() + 1000).toISOString(); + body.stix.modified = modified; const res = await request(app) .put('/api/identities/' + xMitreIdentity + '/modified/' + modified) @@ -326,13 +326,10 @@ describe('Identity API', function () { }); it('PUT /api/identities updates an identity', async function () { - const originalModified = identity1.stix.modified; - const timestamp = new Date().toISOString(); - identity1.stix.modified = timestamp; identity1.stix.description = 'This is an updated identity.'; const body = identity1; const res = await request(app) - .put('/api/identities/' + identity1.stix.id + '/modified/' + originalModified) + .put('/api/identities/' + identity1.stix.id + '/modified/' + identity1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/matrices/matrices.spec.js b/app/tests/api/matrices/matrices.spec.js index a77adfd4..f61ac4b4 100644 --- a/app/tests/api/matrices/matrices.spec.js +++ b/app/tests/api/matrices/matrices.spec.js @@ -172,14 +172,11 @@ describe('Matrices API', function () { }); it('PUT /api/matrices updates a matrix', async function () { - const originalModified = matrix1.stix.modified; - const timestamp = new Date().toISOString(); - matrix1.stix.modified = timestamp; matrix1.stix.description = 'This is an updated matrix.'; const body = matrix1; const res = await request(app) - .put('/api/matrices/' + matrix1.stix.id + '/modified/' + originalModified) + .put('/api/matrices/' + matrix1.stix.id + '/modified/' + matrix1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/mitigations/mitigations.spec.js b/app/tests/api/mitigations/mitigations.spec.js index 191de6f6..d8775eb8 100644 --- a/app/tests/api/mitigations/mitigations.spec.js +++ b/app/tests/api/mitigations/mitigations.spec.js @@ -168,13 +168,10 @@ describe('Mitigations API', function () { }); it('PUT /api/mitigations updates a mitigation', async function () { - const originalModified = mitigation1.stix.modified; - const timestamp = new Date().toISOString(); - mitigation1.stix.modified = timestamp; mitigation1.stix.description = 'This is an updated mitigation.'; const body = mitigation1; const res = await request(app) - .put('/api/mitigations/' + mitigation1.stix.id + '/modified/' + originalModified) + .put('/api/mitigations/' + mitigation1.stix.id + '/modified/' + mitigation1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/notes/notes.spec.js b/app/tests/api/notes/notes.spec.js index 6493d747..eb61c527 100644 --- a/app/tests/api/notes/notes.spec.js +++ b/app/tests/api/notes/notes.spec.js @@ -185,13 +185,10 @@ describe('Notes API', function () { }); it('PUT /api/notes should update a note', async function () { - const originalModified = note1.stix.modified; - const timestamp = new Date().toISOString(); - note1.stix.modified = timestamp; note1.stix.description = 'This is an updated note.'; const body = note1; const res = await request(app) - .put('/api/notes/' + note1.stix.id + '/modified/' + originalModified) + .put('/api/notes/' + note1.stix.id + '/modified/' + note1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/relationships/relationships.spec.js b/app/tests/api/relationships/relationships.spec.js index a18e7f05..9e71a4b8 100644 --- a/app/tests/api/relationships/relationships.spec.js +++ b/app/tests/api/relationships/relationships.spec.js @@ -167,13 +167,15 @@ describe('Relationships API', function () { }); it('PUT /api/relationships updates a relationship', async function () { - const originalModified = relationship1a.stix.modified; - const timestamp = new Date().toISOString(); - relationship1a.stix.modified = timestamp; relationship1a.stix.description = 'This is an updated relationship.'; const body = relationship1a; const res = await request(app) - .put('/api/relationships/' + relationship1a.stix.id + '/modified/' + originalModified) + .put( + '/api/relationships/' + + relationship1a.stix.id + + '/modified/' + + relationship1a.stix.modified, + ) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/software/software.spec.js b/app/tests/api/software/software.spec.js index d6894487..c6552cae 100644 --- a/app/tests/api/software/software.spec.js +++ b/app/tests/api/software/software.spec.js @@ -213,13 +213,10 @@ describe('Software API', function () { }); it('PUT /api/software updates a software', async function () { - const originalModified = software1.stix.modified; - const timestamp = new Date().toISOString(); - software1.stix.modified = timestamp; software1.stix.description = 'This is an updated software.'; const body = software1; const res = await request(app) - .put('/api/software/' + software1.stix.id + '/modified/' + originalModified) + .put('/api/software/' + software1.stix.id + '/modified/' + software1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/tactics/tactics.spec.js b/app/tests/api/tactics/tactics.spec.js index 217f7844..c325a972 100644 --- a/app/tests/api/tactics/tactics.spec.js +++ b/app/tests/api/tactics/tactics.spec.js @@ -160,13 +160,10 @@ describe('Tactics API', function () { }); it('PUT /api/tactics updates a tactic', async function () { - const originalModified = tactic1.stix.modified; - const timestamp = new Date().toISOString(); - tactic1.stix.modified = timestamp; tactic1.stix.description = 'This is an updated tactic.'; const body = tactic1; const res = await request(app) - .put('/api/tactics/' + tactic1.stix.id + '/modified/' + originalModified) + .put('/api/tactics/' + tactic1.stix.id + '/modified/' + tactic1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) diff --git a/app/tests/api/techniques/techniques.spec.js b/app/tests/api/techniques/techniques.spec.js index 7bf95fdc..35bd9023 100644 --- a/app/tests/api/techniques/techniques.spec.js +++ b/app/tests/api/techniques/techniques.spec.js @@ -200,13 +200,10 @@ describe('Techniques Basic API', function () { }); it('PUT /api/techniques updates a technique', async function () { - const originalModified = technique1.stix.modified; - const timestamp = new Date().toISOString(); - technique1.stix.modified = timestamp; technique1.stix.description = 'This is an updated technique.'; const body = cloneForCreate(technique1); const res = await request(app) - .put('/api/techniques/' + technique1.stix.id + '/modified/' + originalModified) + .put('/api/techniques/' + technique1.stix.id + '/modified/' + technique1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) From 9bccaba060364b97edd41c6a9b2adcefe35413be Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:12:11 -0400 Subject: [PATCH 07/55] fix(release-tracks): sync candidate and staged pins to new object revisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Member sync only watched the members tier, so creating a new revision of a candidate- or staged-pinned object silently stranded the pin on the old revision: the release would ship stale content and the object's latest view lost its workspace.release_tracks backref. Under track_latest, pins in all three tiers now follow new revisions per the supplant config (replace moves the pin, queue adds a second candidate, ignore skips); manual tracks are unchanged. This reverses the documented members-only scope — see the behavior evolution note in member-sync-strategies.md. Also make create/update responses read their own writes: the awaited created/updated events can re-pin a track to the new revision, so BaseService now refreshes workspace.release_tracks after event processing instead of returning a response composed before the backref was stamped. --- app/services/meta-classes/base.service.js | 27 ++++++++ .../release-tracks/member-sync-service.js | 63 ++++++++++++------- .../release-tracks/member-sync-strategies.md | 17 ++++- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index f4908aad..c6f7be7a 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -698,9 +698,35 @@ class BaseService extends ServiceWithHooks { const result = createdDocument.toObject ? createdDocument.toObject() : createdDocument; result.warnings = warnings; + await this._refreshReleaseTrackBackrefs(result); return result; } + /** + * Refresh workspace.release_tracks on a response object after domain + * events have run. The created/updated event is awaited, and its listeners + * (member sync → backref reconciliation) may stamp release-track backrefs + * onto the persisted document after the in-memory copy was composed — + * without this, the response would hide backrefs the request itself + * produced. + * + * @param {Object} result - The plain response object ({ workspace, stix }) + * @private + */ + async _refreshReleaseTrackBackrefs(result) { + if (!result?.stix?.id || !result?.stix?.modified) { + return; + } + const backrefs = await this.repository.retrieveBackrefsByVersionLean( + result.stix.id, + result.stix.modified, + ); + if (backrefs) { + result.workspace = result.workspace || {}; + result.workspace.release_tracks = backrefs; + } + } + /** * Import path for create(): handles STIX bundle imports where the object * already has server-controlled fields populated by the source system. @@ -984,6 +1010,7 @@ class BaseService extends ServiceWithHooks { await this.emitUpdatedEvent(newDocument, document); const result = newDocument.toObject ? newDocument.toObject() : newDocument; result.warnings = warnings; + await this._refreshReleaseTrackBackrefs(result); return result; } else { throw new DatabaseError({ diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index 502ccb68..8b204038 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -3,16 +3,21 @@ // ============================================================================= // Member Sync Service // -// Handles automatic enrollment of new object revisions as candidates when -// the object is already a member of a release track. This service implements -// the "Member Sync Strategies" feature documented in 08_MEMBER_SYNC_STRATEGIES.md. +// Keeps release tracks in sync with new object revisions under the +// track_latest strategy (see member-sync-strategies.md): +// - Objects in `members`: new revisions are auto-enrolled as candidates. +// - Objects pinned in `candidates`/`staged`: the pin follows the new +// revision per the supplant config — otherwise the pin silently goes +// stale while the author keeps editing, and the release would ship an +// old revision (the object's latest view would also lose its +// workspace.release_tracks backref). // // Core functionality: // - Listens for STIX object modification events via EventBus -// - Identifies release tracks where the modified object is a member +// - Identifies release tracks that reference the modified object // - Applies the configured member sync strategy (track_latest vs manual) // - Handles supplant behavior (replace/queue/ignore) -// - Creates new draft snapshots with auto-enrolled candidates +// - Creates new draft snapshots with the updated tiers // // This service is event-driven and operates independently of the main // release track workflow. It integrates with workflow-service for @@ -21,7 +26,9 @@ // Event Integration: // Subscribes to BaseService CRUD events ({type}::created, {type}::updated) // via the EventBus. When a STIX object is created or updated, this service -// checks if it's a member of any release track and auto-enrolls if configured. +// checks whether any release track references it and syncs if configured. +// Relationships are deliberately not subscribed: bundle export pulls +// active relationships dynamically. // ============================================================================= const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); @@ -52,17 +59,16 @@ const EventConstants = require('../../lib/event-constants'); exports.handleObjectModified = async function handleObjectModified(event) { const { objectRef, newModified, modifiedBy } = event; - // 1. Find all release tracks where this object is in members - const affectedTracks = await findTracksWithObjectInMembers(objectRef); + // 1. Find all release tracks that reference this object (members, + // candidates, or staged) + const affectedTracks = await findTracksReferencingObject(objectRef); if (affectedTracks.length === 0) { - logger.debug(`[member-sync] No release tracks contain ${objectRef} in members`); + logger.debug(`[member-sync] No release tracks reference ${objectRef}`); return []; } - logger.debug( - `[member-sync] Found ${affectedTracks.length} track(s) with ${objectRef} in members`, - ); + logger.debug(`[member-sync] Found ${affectedTracks.length} track(s) referencing ${objectRef}`); // 2. Process each track according to its member_sync config const results = []; @@ -72,6 +78,7 @@ exports.handleObjectModified = async function handleObjectModified(event) { objectRef, newModified, modifiedBy, + isMember: trackInfo.isMember, }); if (result) results.push(result); } catch (err) { @@ -88,12 +95,18 @@ exports.handleObjectModified = async function handleObjectModified(event) { // ============================================================================= /** - * Find all release tracks where the given object is in the members array. + * Find all release tracks whose latest snapshot references the given object + * in the members, candidates, or staged tiers. + * + * Members enroll new revisions as candidates; candidate/staged pins follow + * new revisions per the supplant config — otherwise a pin silently goes + * stale while the author keeps editing, and the release would ship an old + * revision. * * @param {string} objectRef - The STIX ID to search for - * @returns {Promise>} + * @returns {Promise>} */ -async function findTracksWithObjectInMembers(objectRef) { +async function findTracksReferencingObject(objectRef) { // Get all track IDs from registry const allTracks = await registryRepo.findAll({ limit: 10000 }); const results = []; @@ -105,12 +118,17 @@ async function findTracksWithObjectInMembers(objectRef) { const snapshot = await dynamicRepo.getLatestSnapshot(trackInfo.track_id); if (!snapshot) continue; - // Check if object is in members - const memberEntry = snapshot.members?.find((m) => m.object_ref === objectRef); - if (memberEntry) { + const isMember = (snapshot.members || []).some((m) => m.object_ref === objectRef); + const isTracked = + isMember || + (snapshot.candidates || []).some((c) => c.object_ref === objectRef) || + (snapshot.staged || []).some((s) => s.object_ref === objectRef); + + if (isTracked) { results.push({ trackId: trackInfo.track_id, snapshot, + isMember, }); } } @@ -137,7 +155,7 @@ async function findTracksWithObjectInMembers(objectRef) { * @returns {Promise} New snapshot if changes made, null otherwise */ async function processMemberSync(trackId, snapshot, event) { - const { objectRef, newModified, modifiedBy } = event; + const { objectRef, newModified, modifiedBy, isMember } = event; // Get member sync config with defaults const config = getMemberSyncConfig(snapshot); @@ -158,7 +176,10 @@ async function processMemberSync(trackId, snapshot, event) { // Determine action based on supplant.behavior let action = null; if (!existingEntry) { - // No existing entry → simple enrollment + // No candidate/staged entry. Only members enroll new revisions from + // scratch; a non-member object can only be here via a pin that has + // since disappeared (snapshot changed between discovery and processing). + if (!isMember) return null; action = { type: 'enroll', tier: 'candidates' }; } else { // Existing entry → apply supplant behavior @@ -383,7 +404,7 @@ initializeEventListeners(); // Expose internal functions for unit testing exports._internal = { - findTracksWithObjectInMembers, + findTracksReferencingObject, processMemberSync, getMemberSyncConfig, handleStixObjectEvent, diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index 267dc305..a0a2ced7 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -70,16 +70,27 @@ A **Member Sync Strategy** is a configuration setting on a release track that de ### When Does Member Sync Apply? -Member sync logic is triggered by **object modification events**. Specifically, when a STIX object is created or updated (resulting in a new `modified` timestamp), the system checks whether that object is a member of any release tracks. For each release track where the object is a member, the configured member sync strategy determines what action (if any) to take. +Member sync logic is triggered by **object modification events**. Specifically, when a STIX object is created or updated (resulting in a new `modified` timestamp), the system checks whether that object is referenced by any release track's latest snapshot — in `members`, `candidates`, or `staged`. For each referencing track, the configured member sync strategy determines what action (if any) to take: -**Important:** Member sync only applies to objects that are currently in the `members` array of a release track. It does not apply to objects that are only in `candidates` or `staged`. The rationale is that objects in `candidates` or `staged` are still progressing through the workflow and have not yet been "committed" to the release track as official members. +- **Object in `members`:** the new revision is auto-enrolled as a candidate (the original behavior). If a candidate/staged entry for the object already exists, the supplant config governs the overlap. +- **Object pinned only in `candidates`/`staged`:** the pin follows the new revision per the supplant config (`replace` moves the pin — to the same tier under `status_policy: preserve`, back to `candidates` under `reset`; `queue` adds a second candidate entry; `ignore` does nothing). + +> **Behavior evolution (2026-07-10):** member sync originally applied *only* to +> objects in `members`, on the rationale that candidates/staged entries were +> still in-flight. In practice that meant a candidate pin silently went stale +> the moment the author kept editing — the release would ship the old pinned +> revision, and the object's latest view lost its `workspace.release_tracks` +> backref (the membership appeared to vanish). Under `track_latest`, pins now +> follow new revisions for all three tiers; `manual` tracks are unaffected. +> Relationships are deliberately excluded from sync — bundle export pulls +> active relationships dynamically. ### Relationship to Existing Features Member sync strategies integrate with several existing release track features: - **Candidacy Threshold:** When a new revision is auto-enrolled as a candidate, it may be immediately promoted to `staged` if its status meets the candidacy threshold. -- **Conflict Resolution Policies:** When member sync adds a new revision and a previous revision already exists in `candidates` or `staged`, the configured conflict resolution policy (from `config.promotion_conflicts`) determines how to handle the overlap. +- **Conflict Resolution Policies:** Member sync resolves overlaps with existing `candidates`/`staged` entries through its own `supplant` config (below). *Manual* candidate adds and demotions instead go through `config.promotion_conflicts.into_candidates` (default `prefer_latest`) — see `release-workflow.md`. The two are deliberately separate: supplant expresses sync intent (replace/queue/ignore), while `into_candidates` uses the same policy vocabulary as the other tier transitions. - **Snapshot Creation:** Any change to a release track's object lists (`candidates`, `staged`, `members`) results in a new draft snapshot being created. Member sync follows this convention. --- From 21f6bddd6109bd41c1167b73d2c8bb845aac3b15 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:13:27 -0400 Subject: [PATCH 08/55] feat(release-tracks): add into_candidates conflict resolution policy Manual candidate adds and demotions appended blindly when the object_ref was already pinned in candidates at a different revision, duplicating candidates. Entries into the candidates tier now pass through the same conflict machinery as the other tier transitions, governed by config.promotion_conflicts.into_candidates (prefer_latest by default; abort returns 409). Exact (object_ref, object_modified) re-adds remain idempotent, and revision-sync enrollment keeps its own supplant semantics. Also adds the regression suite covering the whole release-track backref series: lifecycle backrefs across every tier transition, server-controlled stripping (create/update/import and all revision-clone paths), revision sync for member/candidate/staged pins, read-your-own-writes responses, manual re-adds, and the conflict policies. --- .../definitions/components/release-tracks.yml | 11 +- .../release-tracks/release-track-schemas.js | 1 + .../release-track-snapshot-schema.js | 7 + .../release-tracks/standard-track-service.js | 32 +- .../release-tracks-backrefs.spec.js | 700 ++++++++++++++++++ docs/developer/TODO.md | 266 +++++++ docs/user/release-tracks/release-workflow.md | 14 +- 7 files changed, 1026 insertions(+), 5 deletions(-) create mode 100644 app/tests/api/release-tracks/release-tracks-backrefs.spec.js create mode 100644 docs/developer/TODO.md diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index d5459d1c..29ed70d2 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -175,8 +175,17 @@ components: default: 'reviewed' promotion_conflicts: type: object - description: 'Conflict resolution policies for tier promotions' + description: 'Conflict resolution policies for tier transitions' properties: + into_candidates: + type: string + enum: + - always_overwrite + - always_reject + - prefer_latest + - abort + description: 'How to handle conflicts when a manually added or demoted entry targets an object_ref already pinned in candidates at a different revision' + default: 'prefer_latest' candidates_to_staged: type: string enum: diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index fd070454..6f3daa5c 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -366,6 +366,7 @@ const updateCandidateVersionBodySchema = z.object({ /** PUT /release-tracks/:id/config */ const promotionConflictsSchema = z.object({ + into_candidates: conflictPolicySchema.optional(), candidates_to_staged: conflictPolicySchema.exclude(['abort']).optional(), staged_to_members: conflictPolicySchema.optional(), }); diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index bc16ceee..42e261ee 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -177,6 +177,13 @@ const compositionResolutionSchema = new mongoose.Schema(compositionResolutionDef // --- Config sub-schemas --- const promotionConflictsDefinition = { + // Applies when an entry enters the candidates tier (manual add, demote) + // and the object_ref is already pinned at a different revision. + into_candidates: { + type: String, + enum: ['always_overwrite', 'always_reject', 'prefer_latest', 'abort'], + default: 'prefer_latest', + }, candidates_to_staged: { type: String, enum: ['always_overwrite', 'always_reject', 'prefer_latest'], diff --git a/app/services/release-tracks/standard-track-service.js b/app/services/release-tracks/standard-track-service.js index b5bc4787..03c2030d 100644 --- a/app/services/release-tracks/standard-track-service.js +++ b/app/services/release-tracks/standard-track-service.js @@ -122,7 +122,20 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId }); } - const mergedCandidates = [...existingCandidates, ...newEntries]; + // Same-object conflicts (the object_ref is already pinned in candidates at + // a different revision) are resolved by the into_candidates policy. + const conflictPolicy = source.config?.promotion_conflicts?.into_candidates || 'prefer_latest'; + const { merged: mergedCandidates, rejected } = conflictResolution.applyConflictPolicy( + existingCandidates, + newEntries, + conflictPolicy, + ); + if (rejected.length > 0) { + logger.verbose( + `StandardTrackService: into_candidates policy "${conflictPolicy}" rejected ` + + `${rejected.length} candidate(s) for track "${trackId}"`, + ); + } let snapshot = await snapshotService.cloneSnapshot(trackId, source, { candidates: mergedCandidates, @@ -448,9 +461,24 @@ exports.demoteStaged = async function demoteStaged(trackId, objectRefs, userId) }); } + // Demoted entries re-enter candidates through the same conflict policy as + // manual adds. + const conflictPolicy = source.config?.promotion_conflicts?.into_candidates || 'prefer_latest'; + const { merged: mergedCandidates, rejected } = conflictResolution.applyConflictPolicy( + existingCandidates, + demotedEntries, + conflictPolicy, + ); + if (rejected.length > 0) { + logger.verbose( + `StandardTrackService: into_candidates policy "${conflictPolicy}" rejected ` + + `${rejected.length} demoted entry/entries for track "${trackId}"`, + ); + } + const snapshot = await snapshotService.cloneSnapshot(trackId, source, { staged: remainingStaged, - candidates: [...existingCandidates, ...demotedEntries], + candidates: mergedCandidates, }); logger.verbose( diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js new file mode 100644 index 00000000..3a2d7ece --- /dev/null +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -0,0 +1,700 @@ +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +const logger = require('../../../lib/logger'); +logger.level = 'debug'; + +const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { + workflow: { + state: 'work-in-progress', + }, + }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release Track Backrefs (workspace.release_tracks) API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function postObject(path, body, expectedStatus = 201) { + const res = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return res.body; + } + + async function getObjectVersion(path) { + const res = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return res.body; + } + + async function getTechniqueVersion(technique) { + return getObjectVersion( + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + ); + } + + async function createTrack(name) { + const res = await postObject('/api/release-tracks/new', { name, type: 'standard' }); + return res.id; + } + + async function addCandidates(trackId, objects) { + return postObject( + `/api/release-tracks/${trackId}/candidates`, + { + object_refs: objects.map((o) => ({ id: o.stix.id, modified: o.stix.modified })), + }, + 200, + ); + } + + function trackEntries(object) { + return object.workspace.release_tracks || []; + } + + function entryForTrack(object, trackId) { + return trackEntries(object).find((e) => e.id === trackId); + } + + describe('candidate lifecycle', function () { + let trackId; + let technique; + + before(async function () { + technique = await postObject('/api/techniques', buildTechnique('Backref Lifecycle')); + trackId = await createTrack('Backref Lifecycle Track'); + }); + + it('adding a candidate sets a candidate backref on the pinned revision', async function () { + await addCandidates(trackId, [technique]); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + + it('reviewing candidates updates the backref status', async function () { + await postObject( + `/api/release-tracks/${trackId}/candidates/review`, + { from: 'work-in-progress', to: 'awaiting-review' }, + 200, + ); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'awaiting-review', + }); + }); + + it('promoting candidates flips the backref tier to staged', async function () { + await postObject( + `/api/release-tracks/${trackId}/candidates/promote`, + { object_refs: [technique.stix.id] }, + 200, + ); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'staged', + status: 'awaiting-review', + }); + }); + + it('demoting staged entries returns the backref tier to candidates', async function () { + await postObject( + `/api/release-tracks/${trackId}/staged/demote`, + { object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }] }, + 200, + ); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'awaiting-review', + }); + }); + + it('bumping the track promotes staged backrefs to member/reviewed', async function () { + await postObject( + `/api/release-tracks/${trackId}/candidates/promote`, + { object_refs: [technique.stix.id] }, + 200, + ); + await postObject(`/api/release-tracks/${trackId}/bump`, { type: 'minor' }, 200); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'members', + status: 'reviewed', + }); + }); + + it('deleting the track removes its backrefs', async function () { + await request(app) + .delete(`/api/release-tracks/${trackId}`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(204); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toBeUndefined(); + }); + }); + + describe('candidate removal and version pins', function () { + it('removing a candidate removes the backref', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Backref Removal')); + const trackId = await createTrack('Backref Removal Track'); + await addCandidates(trackId, [technique]); + + await request(app) + .delete(`/api/release-tracks/${trackId}/candidates/${technique.stix.id}`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(204); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toBeUndefined(); + }); + + it('updating a candidate version pin moves the backref to the new revision', async function () { + const revisionA = await postObject('/api/techniques', buildTechnique('Backref Pin Move')); + + // Create a second revision of the same object + const revisionBData = buildTechnique('Backref Pin Move v2'); + revisionBData.stix.id = revisionA.stix.id; + revisionBData.stix.created = revisionA.stix.created; + revisionBData.stix.modified = new Date( + new Date(revisionA.stix.modified).getTime() + 1000, + ).toISOString(); + const revisionB = await postObject('/api/techniques', revisionBData); + + const trackId = await createTrack('Backref Pin Move Track'); + await addCandidates(trackId, [revisionA]); + + await postObject( + `/api/release-tracks/${trackId}/candidates/${revisionA.stix.id}/update-version`, + { old_modified: revisionA.stix.modified, new_modified: revisionB.stix.modified }, + 200, + ); + + const retrievedA = await getTechniqueVersion(revisionA); + const retrievedB = await getTechniqueVersion(revisionB); + expect(entryForTrack(retrievedA, trackId)).toBeUndefined(); + expect(entryForTrack(retrievedB, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + }); + + describe('members and snapshots', function () { + it('setting track contents adds member backrefs and reverts on snapshot delete', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Backref Contents')); + const trackId = await createTrack('Backref Contents Track'); + + const contentsSnapshot = await postObject( + `/api/release-tracks/${trackId}/contents`, + { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }, + 200, + ); + + let retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'members', + status: 'reviewed', + }); + + // Deleting the latest snapshot reverts membership to the previous + // (empty) snapshot — the backref disappears + await request(app) + .delete(`/api/release-tracks/${trackId}/snapshots/${contentsSnapshot.modified}`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(204); + + retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toBeUndefined(); + }); + + it('an object referenced by two tracks carries one backref per track', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Backref Two Tracks')); + const trackA = await createTrack('Backref Two Tracks A'); + const trackB = await createTrack('Backref Two Tracks B'); + + await addCandidates(trackA, [technique]); + await addCandidates(trackB, [technique]); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackA)).toMatchObject({ tier: 'candidates' }); + expect(entryForTrack(retrieved, trackB)).toMatchObject({ tier: 'candidates' }); + expect(trackEntries(retrieved)).toHaveLength(2); + }); + + it('cloning a track adds backrefs for the new track', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Backref Clone')); + const trackId = await createTrack('Backref Clone Track'); + await addCandidates(trackId, [technique]); + + const cloned = await postObject( + `/api/release-tracks/${trackId}/clone`, + { name: 'Backref Clone Track Copy' }, + 201, + ); + + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, trackId)).toMatchObject({ tier: 'candidates' }); + expect(entryForTrack(retrieved, cloned.id)).toMatchObject({ tier: 'candidates' }); + }); + }); + + describe('member sync', function () { + it('a new revision of a member object gets a candidate backref while the member revision keeps its own', async function () { + const revisionA = await postObject('/api/techniques', buildTechnique('Backref Member Sync')); + const trackId = await createTrack('Backref Member Sync Track'); + + await postObject( + `/api/release-tracks/${trackId}/contents`, + { + x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], + }, + 200, + ); + + // Creating a new revision triggers member sync (default strategy: + // track_latest) which auto-enrolls the new revision as a candidate + const revisionBData = buildTechnique('Backref Member Sync v2'); + revisionBData.stix.id = revisionA.stix.id; + revisionBData.stix.created = revisionA.stix.created; + revisionBData.stix.modified = new Date( + new Date(revisionA.stix.modified).getTime() + 1000, + ).toISOString(); + const revisionB = await postObject('/api/techniques', revisionBData); + + const retrievedA = await getTechniqueVersion(revisionA); + const retrievedB = await getTechniqueVersion(revisionB); + expect(entryForTrack(retrievedA, trackId)).toEqual({ + id: trackId, + tier: 'members', + status: 'reviewed', + }); + expect(entryForTrack(retrievedB, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + + it('a new revision of a candidate object moves the pin and backref to the new revision', async function () { + const revisionA = await postObject( + '/api/techniques', + buildTechnique('Backref Candidate Sync'), + ); + const trackId = await createTrack('Backref Candidate Sync Track'); + await addCandidates(trackId, [revisionA]); + + const revisionBData = buildTechnique('Backref Candidate Sync v2'); + revisionBData.stix.id = revisionA.stix.id; + revisionBData.stix.created = revisionA.stix.created; + revisionBData.stix.modified = new Date( + new Date(revisionA.stix.modified).getTime() + 1000, + ).toISOString(); + const revisionB = await postObject('/api/techniques', revisionBData); + + // The POST response itself reflects the moved backref — the events + // that re-pin the track are awaited before the response is composed + expect(entryForTrack(revisionB, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + + const retrievedA = await getTechniqueVersion(revisionA); + const retrievedB = await getTechniqueVersion(revisionB); + expect(entryForTrack(retrievedA, trackId)).toBeUndefined(); + expect(entryForTrack(retrievedB, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + + it('a new revision of a staged object returns the pin to candidates (default supplant)', async function () { + const revisionA = await postObject('/api/techniques', buildTechnique('Backref Staged Sync')); + const trackId = await createTrack('Backref Staged Sync Track'); + await addCandidates(trackId, [revisionA]); + await postObject( + `/api/release-tracks/${trackId}/candidates/promote`, + { object_refs: [revisionA.stix.id] }, + 200, + ); + + const revisionBData = buildTechnique('Backref Staged Sync v2'); + revisionBData.stix.id = revisionA.stix.id; + revisionBData.stix.created = revisionA.stix.created; + revisionBData.stix.modified = new Date( + new Date(revisionA.stix.modified).getTime() + 1000, + ).toISOString(); + const revisionB = await postObject('/api/techniques', revisionBData); + + const retrievedA = await getTechniqueVersion(revisionA); + const retrievedB = await getTechniqueVersion(revisionB); + expect(entryForTrack(retrievedA, trackId)).toBeUndefined(); + expect(entryForTrack(retrievedB, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + + it('manual strategy leaves candidate pins on the original revision', async function () { + const revisionA = await postObject('/api/techniques', buildTechnique('Backref Manual Sync')); + const trackId = await createTrack('Backref Manual Sync Track'); + await request(app) + .put(`/api/release-tracks/${trackId}/config`) + .send({ member_sync: { strategy: 'manual' } }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + await addCandidates(trackId, [revisionA]); + + const revisionBData = buildTechnique('Backref Manual Sync v2'); + revisionBData.stix.id = revisionA.stix.id; + revisionBData.stix.created = revisionA.stix.created; + revisionBData.stix.modified = new Date( + new Date(revisionA.stix.modified).getTime() + 1000, + ).toISOString(); + const revisionB = await postObject('/api/techniques', revisionBData); + + const retrievedA = await getTechniqueVersion(revisionA); + const retrievedB = await getTechniqueVersion(revisionB); + expect(entryForTrack(retrievedA, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + expect(entryForTrack(retrievedB, trackId)).toBeUndefined(); + }); + }); + + describe('relationships', function () { + it('relationship documents get backrefs in their own collection', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Backref Rel Target')); + const group = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Backref Rel Group', + spec_version: '2.1', + type: 'intrusion-set', + description: 'Group used to verify relationship backrefs.', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + const relationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: technique.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + + const trackId = await createTrack('Backref Relationship Track'); + await addCandidates(trackId, [relationship]); + + const retrieved = await getObjectVersion( + `/api/relationships/${relationship.stix.id}/modified/${relationship.stix.modified}`, + ); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + }); + + describe('manual re-adds and the into_candidates policy', function () { + async function setTrackConfig(trackId, config) { + await request(app) + .put(`/api/release-tracks/${trackId}/config`) + .send(config) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + } + + async function listCandidates(trackId) { + const res = await getObjectVersion(`/api/release-tracks/${trackId}/candidates`); + return res.candidates; + } + + function buildNextRevision(previous, name) { + const data = buildTechnique(name); + data.stix.id = previous.stix.id; + data.stix.created = previous.stix.created; + data.stix.modified = new Date( + new Date(previous.stix.modified).getTime() + 1000, + ).toISOString(); + return data; + } + + it('re-adding an object after a new revision replaces the stale pin (prefer_latest default)', async function () { + const revisionA = await postObject('/api/techniques', buildTechnique('Backref Readd')); + const trackId = await createTrack('Backref Readd Track'); + // manual strategy isolates the add-candidates path from revision sync + await setTrackConfig(trackId, { member_sync: { strategy: 'manual' } }); + await postObject( + `/api/release-tracks/${trackId}/candidates`, + { object_refs: [{ id: revisionA.stix.id }] }, + 200, + ); + + const revisionB = await postObject( + '/api/techniques', + buildNextRevision(revisionA, 'Backref Readd v2'), + ); + + // Re-add without modified — resolves to the latest revision and + // replaces the stale pin instead of duplicating it + await postObject( + `/api/release-tracks/${trackId}/candidates`, + { object_refs: [{ id: revisionA.stix.id }] }, + 200, + ); + + const candidates = await listCandidates(trackId); + expect(candidates).toHaveLength(1); + expect(new Date(candidates[0].object_modified).toISOString()).toBe(revisionB.stix.modified); + + expect(entryForTrack(await getTechniqueVersion(revisionA), trackId)).toBeUndefined(); + expect(entryForTrack(await getTechniqueVersion(revisionB), trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + + // An exact re-add of the same revision is idempotent + await postObject( + `/api/release-tracks/${trackId}/candidates`, + { object_refs: [{ id: revisionA.stix.id }] }, + 200, + ); + expect(await listCandidates(trackId)).toHaveLength(1); + }); + + it('into_candidates=abort rejects a conflicting re-add with 409', async function () { + const revisionA = await postObject('/api/techniques', buildTechnique('Backref Abort')); + const trackId = await createTrack('Backref Abort Track'); + await setTrackConfig(trackId, { + member_sync: { strategy: 'manual' }, + promotion_conflicts: { into_candidates: 'abort' }, + }); + await addCandidates(trackId, [revisionA]); + + await postObject('/api/techniques', buildNextRevision(revisionA, 'Backref Abort v2')); + + await postObject( + `/api/release-tracks/${trackId}/candidates`, + { object_refs: [{ id: revisionA.stix.id }] }, + 409, + ); + + // Track state unchanged: still pinned at revision A, backref intact + const candidates = await listCandidates(trackId); + expect(candidates).toHaveLength(1); + expect(new Date(candidates[0].object_modified).toISOString()).toBe(revisionA.stix.modified); + expect(entryForTrack(await getTechniqueVersion(revisionA), trackId)).toBeDefined(); + }); + }); + + describe('server-controlled field', function () { + it('strips client-supplied workspace.release_tracks on create', async function () { + const data = buildTechnique('Backref Injection Create'); + data.workspace.release_tracks = [ + { id: 'release-track--00000000-0000-4000-8000-000000000000', tier: 'members' }, + ]; + + const created = await postObject('/api/techniques', data); + expect(created.workspace.release_tracks).toBeUndefined(); + }); + + it('preserves server-managed backrefs when a PUT omits or fakes them', async function () { + const technique = await postObject( + '/api/techniques', + buildTechnique('Backref Injection Put'), + ); + const trackId = await createTrack('Backref Injection Track'); + await addCandidates(trackId, [technique]); + + const update = buildTechnique('Backref Injection Put (updated)'); + update.stix.id = technique.stix.id; + update.stix.created = technique.stix.created; + update.stix.modified = technique.stix.modified; + update.workspace.release_tracks = [ + { id: 'release-track--00000000-0000-4000-8000-000000000000', tier: 'members' }, + ]; + + const res = await request(app) + .put(`/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`) + .send(update) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + expect(entryForTrack(res.body, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + expect(trackEntries(res.body)).toHaveLength(1); + }); + }); + + describe('revision clones never inherit backrefs', function () { + it('revoking an object strips backrefs from the revoked and deprecated revisions', async function () { + const techniqueA = await postObject('/api/techniques', buildTechnique('Backref Revoke A')); + const techniqueB = await postObject('/api/techniques', buildTechnique('Backref Revoke B')); + const group = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Backref Revoke Group', + spec_version: '2.1', + type: 'intrusion-set', + description: 'Group used to verify revoke backref stripping.', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + const relationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: techniqueA.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + + const trackId = await createTrack('Backref Revoke Track'); + await addCandidates(trackId, [techniqueA, relationship]); + + const res = await request(app) + .post(`/api/techniques/${techniqueA.stix.id}/revoke`) + .send({ revoking: { stixId: techniqueB.stix.id, modified: techniqueB.stix.modified } }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + // The revoked revision is a new version — it must not inherit backrefs + expect(res.body.primary.stix.revoked).toBe(true); + expect(res.body.primary.workspace.release_tracks).toBeUndefined(); + + // The relationship referencing the revoked object was deprecated into a + // new revision — it must not inherit backrefs either + const latestRels = await getObjectVersion(`/api/relationships/${relationship.stix.id}`); + const latestRel = latestRels[0]; + expect(latestRel.stix.x_mitre_deprecated).toBe(true); + expect(latestRel.stix.modified).not.toBe(relationship.stix.modified); + expect(latestRel.workspace.release_tracks).toBeUndefined(); + + // The pinned revisions keep their backrefs + const pinnedTechnique = await getTechniqueVersion(techniqueA); + expect(entryForTrack(pinnedTechnique, trackId)).toMatchObject({ tier: 'candidates' }); + const pinnedRel = await getObjectVersion( + `/api/relationships/${relationship.stix.id}/modified/${relationship.stix.modified}`, + ); + expect(entryForTrack(pinnedRel, trackId)).toMatchObject({ tier: 'candidates' }); + }); + + it('technique conversion strips backrefs from the converted revision', async function () { + const parent = await postObject('/api/techniques', buildTechnique('Backref Convert Parent')); + const technique = await postObject( + '/api/techniques', + buildTechnique('Backref Convert Child'), + ); + const trackId = await createTrack('Backref Convert Track'); + await addCandidates(trackId, [technique]); + + const res = await request(app) + .post(`/api/techniques/${technique.stix.id}/convert-to-subtechnique`) + .send({ parentTechniqueAttackId: parent.workspace.attack_id }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + // The converted revision is a new version — no inherited backrefs + expect(res.body.primary.stix.x_mitre_is_subtechnique).toBe(true); + expect(res.body.primary.workspace.release_tracks).toBeUndefined(); + + // The pinned revision keeps its backref + const pinned = await getTechniqueVersion(technique); + expect(entryForTrack(pinned, trackId)).toMatchObject({ tier: 'candidates' }); + }); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md new file mode 100644 index 00000000..4aba92d5 --- /dev/null +++ b/docs/developer/TODO.md @@ -0,0 +1,266 @@ +# Release Track TODOs + +## Regression Tests + +- [ ] Implement regression tests + +- [x] **Investigate the recurring full-suite flake.** Two root causes found and fixed (2026-07-10) in `app/lib/database-in-memory.js`: + 1. *Port collision*: every spec file stopped and restarted the `mongodb-memory-server` instance, and a fresh mongod would intermittently fail with `Port already in use` — breaking that file's `before` hook (surfacing as `loginAnonymous` 404s) and cascading failures through the file. Fixed by reusing one mongod for all spec files in the process (`closeConnection` drops the database and disconnects but keeps the server running) plus `--exit` on the mocha scripts. + 2. *Vanishing unique indexes*: dropping the database between spec files also drops its indexes, and mongoose's per-model `init()` is memoized per process — so the `stix.id + stix.modified` unique index was intermittently missing for later files, letting duplicate-POST tests (and dependent count tests) fail in roaming pairs. Fixed by explicitly awaiting `createIndexes()` for all registered models after each reconnect. + + Residual: rare (≈1 per run under heavy machine load) single-test failures of a different character (a count assertion, a 20s timeout in a pagination GET) still appear occasionally and pass in isolation — likely load-related; keep observing before chasing further. + + +## Snapshot Output Format + +**TASK Summary**: Implement support for the `bundle` output format for snapshots + +`bundle` refers to a STIX 2.1 bundle that contains all of the objects in the snapshot. The bundle should be emitted as a JSON object with the following structure: + +```json +{ + "type": "bundle", + "id": "bundle--", + "spec_version": "2.0", // omit if STIX 2.1, include for STIX 2.0 + "objects": [ + // All objects in the snapshot + ] +} +``` + +The following release-track snapshot retrieval endpoints support `include` and +`format` query parameters: + +- `GET /api/release-tracks/:id` (get latest snapshot) +- `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) + +> [!Note] +> The ephemeral bundle endpoint (`GET /api/release-tracks/ephemeral/{domain}`) supports `format`, but not tier `include`, because it does not read from a persisted release-track snapshot. Rather, it "blindly" includes all objects in the domain. + + +**Include Parameter** (controls which tiers are returned): +``` +GET /api/release-tracks/:id # Default: all tiers +GET /api/release-tracks/:id?include=members # Members tier only +GET /api/release-tracks/:id?include=staged # Members and staged tiers +GET /api/release-tracks/:id?include=candidates # Members and candidates tiers +GET /api/release-tracks/:id?include=quarantine # Members and quarantine tiers +GET /api/release-tracks/:id?include=all # All tiers +``` + +**Format Parameter** (controls output format): +``` +GET /api/release-tracks/:id?format=workbench # Workbench snapshot with metadata (default) +GET /api/release-tracks/:id?format=bundle # Standard STIX 2.1 bundle +GET /api/release-tracks/:id?format=filesystemstore # Not implemented; returns 501 +``` + +**Combined Example:** +``` +GET /api/release-tracks/:id?include=all&format=workbench +``` + +> [!Note] +> The `workbench` format is the default output format and is already implemented. The `bundle` format is a new output format that needs to be implemented. The `filesystemstore` format is not implemented and will return a 501 error if requested. + +### Replacing the legacy `GET /api/stix-bundles` endpoint + +Importantly, the release track retrieval method with `format=bundle` as well as the ephemeral bundle endpoint will supplant the `GET /api/stix-bundles/` endpoint defined in `stix-bundles-routes.js`. The `stix-bundles` endpoint will be deprecated and removed in a future release. We thus need to inspect the `stix-bundles-controller.js` module and identify any logic that needs to be preserved with respect to preserving existing functionality in the new endpoints. + +The `stix-bundles` endpoint currently supports generating a `x-mitre-collection` object that is emitted in the bundle. We need to ensure that this functionality is preserved in the new endpoints. Users specify how the `x-mitre-collection` object is generated via the `includeCollectionObject`, `collectionObjectVersion`, `collectionObjectModified`, and `collectionAttackSpecVersion` query parameters. We can simplify this functionality in the new endpoints: + +- `collectionObjectVersion` can just default to `v0.1` to signify that the collection was generated ephemerally and is not connected to a particular release track. +- `collectionObjectModified` can default to the current timestamp. +- `collectionAttackSpecVersion` can default to the global default attack spec version (tracked in `config.js` and exposed via `app.attackSpecVersion`). +- The `includeCollectionObject` parameter can be renamed to `includeToc` to signify that the user wants to include a table of contents object in the bundle (which is what the `x-mitre-collection` object effectively is; moreover, the term, "collection", is oversaturated in the context of STIX and Workbench, so this renaming will help reduce confusion). The `includeToc` parameter can default to `true`. + +Here is how each of the other query parameters should be handled/mapped to the newer ephemeral bundle retrieval endpoint (`/api/release-tracks/ephemeral/{domain}`): + +- `includeNotes` can be **removed**. We originally implemented notes in Workbench such that they could be included in emitted STIX bundles because we treat notes as STIX objects. However, this concept never really took off, and we have decided to treat notes as second-class Workbench-native objects that are not STIX objects, and thus cannot be included in emitted STIX bundles. +- `includeMissingAttackId` should be **preserved** as `includeObjectsWithMissingAttackId`. This parameter allows users to control whether or not objects without ATT&CK IDs are included in the emitted bundle. It defaults to `false`. +- `stixVersion` should be **preserved**. This parameter allows users to control which STIX version is used in the emitted bundle (`2.0` or `2.1`). It defaults to `2.1`. +- `useLegacyMethod` should be **removed**. The `stix-bundles-service.js` module has a legacy method for generating STIX bundles that we no longer use. The new endpoints should not support this legacy method, and thus this parameter can be removed. +- `includeDataSources` should be **removed**. For context, Data Sources are officially considered a deprecated concept in ATT&CK as of ATT&CK Spec v3.3.0. They were marked as either deprecated or revoked in the corresponding ATT&CK content release (v18.0). Because we already have `includeDeprecated` and `includeRevoked` query parameters, we can remove `includeDataSources` and instead rely on the `includeDeprecated` and `includeRevoked` query parameters to control whether or not deprecated/revoked Data Sources are included in the emitted bundle. This will simplify the API and reduce confusion. +- `state` can be **removed**. The `state` parameter was originally implemented to allow users to control which objects are included based on their workflow status (`work-in-progress`, `awaiting-review`, `reviewed`). Before the introduction of release tracks, workflow status was globally scoped. Now, with release tracks, workflow status is scoped to a release track. The ephemeral bundle endpoint is domain scoped, not release-track scoped, and thus it does not have a concept of workflow status. The `state` parameter can be removed from the new endpoints. + +### Updates to the release-track retrieval endpoints + +For release track retrieval requests that include the `format=bundle` query parameter, the following query parameters must be supported: + +- `include: ['candidate', 'staged']`: If specified, the value must be equal to an array of at least one value. The parameter acts as a filter, allowing users to specify whether release-track candidates and/or staged objects should be included in the bundle. If the `include` parameter is omitted, only members should be included. +- `state: ['work-in-progress', 'awaiting-review']`: If specified, the value must be equal to an array of at least one value. Notably, objects marked as `"reviewed"` are always included (by nature of all members being included —— all members are inherently "reviewed"), irrespective of this query parameter. The parameter acts as a union filter that logically combines with `include`. In other words, when `include` and `state` are both specified, `include` is applied first, then `state` is applied to the remaining `include`-filtered subset. (i.e., Of the candidates and/or staged objects that are ready to be included in the emitted bundle, only include the ones that are marked as "work-in-progress", "awaiting-review", or either). +- `stixVersion` should be **preserved**. This parameter allows users to control which STIX version is used in the emitted bundle (`2.0` or `2.1`). It defaults to `2.1`. + +### In Summary: + +- [x] Read the existing release track user + developer documentation in `docs/user/release-tracks/` and `docs/developer/release-tracks/`, respectively. +- [x] Review the new `GET /api/release-tracks/ephemeral/:domain` endpoint implementation as well as the legacy `GET /api/stix-bundles` endpoint. +- [x] Implement support for the `format=bundle` query parameter in the following two endpoints: + - `GET /api/release-tracks/:id` (get latest snapshot) + - `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) +- [x] Ensure that all required logic (query parameters) is/are implemented in the new endpoints as outlined above. +- [x] Implement regression tests for the new functionality (`release-tracks-bundle.spec.js`, `ephemeral-bundle.spec.js`) +- [x] Update the aforementioned user + developer documentation. The user documentation should simply describe how the behavior _is_ while the developer documentation should described _why_ and _how_, and additionally cover what has been described here: explaining what _was_ and how the functionality has evolved from before the introduction of release tracks to after. (See `docs/developer/release-tracks/bundle-export.md`.) + + +## Bidirectional References + +- [x] Implement bidirectional refs between objects and snapshots. Users should be able to get individual objects via standard getters (e.g., `GET /api/techniques/:id`) and see which snapshots they are part of in the object's metadata. + +> **Implemented** as `workspace.release_tracks` (`[{ id, tier, status }]`, tiers `members`/`staged`/`candidates`/`quarantine` — matching the snapshot tier array names; the sketch below predates the rename of `phase` → `tier`), maintained via snapshot-driven reconciliation over the `release-track::contents-changed` EventBus event. See `docs/developer/release-tracks/backref-reconciliation.md` (why/how) and `docs/user/release-tracks/object-backrefs.md` (behavior). Regression tests: `app/tests/api/release-tracks/release-tracks-backrefs.spec.js`. + +Currently, it is impossible to delineate which release tracks (if any) an object belongs to _from the object's perspective_. By "the object's perspective", I mean from a given STIX object document in the `attackObjects` Mongo collection —— you cannot look at a document in the `attackObjects` collection and see which release track(s) the object is a part of. Instead, you must scan all existing release tracks for the object's `stix.id` value in either the `candidates`, `staged`, `members`, or `quarantine` list. + +This is easily correctable. When an object is either added or removed from a release track, the object document should be updated. We just need to include a small piece of metadata in the STIX object's document. Fortunately, we already have a pattern in place for tracking metadata: `workspace`. Moreover, we actually have an equivalent bidirectional ref tracker in place for the release tracks' predecessor: Workbench collections. They are/were tracked in each object's `workspace.collection` field. So, we may be able to copy/mimic this existing workflow. + +I am imagining STIX object documents containing backwards pointers to their containing release track(s) looking something like this: + +```yaml +# A Technique document +workspace: + release_tracks: + - id: String + phase: String; Options: ['candidate', 'staged', 'member'] + status: String; Options: ['work-in-progress', 'awaiting-review', or 'reviewed'] +stix: # ... +``` + +For example: + +```yaml +workspace: + release_tracks: + - id: 'release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d' + phase: 'candidate' + status: 'work-in-progress' +stix: # ... +``` + +The `phase` and `status` fields will need to change for the appropriate `release_tracks` list element when user moves the object between the candidate, staged, and member phases; and when the object's status changes. We can make use of the event bus architecture here, following the same pattern that some services (like `detection-strategies-service.js` and `analytics-service.js`) use to track embedded relationships between two objects. Similarly, the release tracks service would just need to fire off an event that each of the STIX services listen; and when heard, they set the `workspace.release_tracks` field for the relevant STIX object document(s) accordingly. + +## Release-Track Change Capture (in-place mutation hardening) + +Object CRUD paths can mutate or destroy revisions that release tracks pin, without the track ever hearing about it. Design decisions locked in 2026-07-10. The `workspace.release_tracks` backrefs make every guard below a cheap document-local check (no track scanning). + +- [x] **Reject revision re-keying on PUT.** `updateFull` merged body `stix.id`/`stix.modified` over the stored document, so a PUT could silently re-key a revision and strand any track pins. Now returns 400 when the body identity fields differ from the path parameters. Re-keying must go through POST (a new revision), which member sync captures. Tests: `app/tests/api/base-services/update-identity-guard.spec.js`. + +- [ ] **Capture in-place PUTs of pinned revisions.** Decision: reject the PUT (409) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. When pinned only in `staged`/`candidates`, allow the PUT but flag the tier entry (reset/annotate `object_status`; tentative value `modified-in-place`) via a snapshot clone so the change is re-reviewed. This also covers in-place deprecation (`x_mitre_deprecated` set via PUT) — tracks must never be blind to it. Also fix the current member-sync misfire on `::updated` events: an in-place PUT of a member-pinned revision today enrolls a candidate with the *same* `(stix.id, modified)` key as the member entry, creating a duplicate cross-tier reference in the snapshot. Needs: guard in `BaseService.updateFull` (read the document's backrefs), a release-tracks event/handler for flagging an entry, snapshot-schema status addition, workflow/auto-promotion interaction review, tests, docs. + +- [ ] **DELETE of tracked objects → deprecation.** Decision: deleting an object revision pinned as `members`/`staged` must not hard-delete (today the backref dies with the document and the track keeps a silent dangling pin). Convert the delete into a new revision with `x_mitre_deprecated: true`, which member sync enrolls as a candidate — aligns with ATT&CK's deprecate-don't-delete release convention (released objects never vanish between releases; the custom deleted-flag + remove-on-merge idea was considered and rejected for that reason). Hard delete remains available for untracked / work-in-progress objects. True removal from a release is a track-side operation (remove the member entry), not an object-side delete. + +- [ ] **Revoke must reach member sync.** `revoke()` saves the revoked revision via `repository.save` directly, so no `::created`/`::updated` event fires and member sync never enrolls the revoked revision in the tracks where the object is a member — a track can silently keep exporting the pre-revoke revision. Fix: subscribe member sync to the existing per-type `::revoked` events (payload shape differs from created/updated — needs a small adapter in member-sync-service). Scoping is inherent and safe: member sync only enrolls in tracks that already hold the object in `members`, so a revoke can never pull an object into another team's track. Decision: do NOT extend member sync to relationships — bundle export pulls active relationships dynamically and deprecated ones drop out on their own; tracks may still pin relationships manually. + +## Diffing Endpoint + +- [ ] Implement object diffing endpoints for snapshots. Users should be able to effectively preview changes to objects before tier transitions (candidates, staged, members). + +### Idea 1 - Diffing endpoint specifically for release tracks + +In this approach, we would implement a workflow-driven diffing endpoint that is specific to release tracks. The endpoint would allow users to diff objects in the candidate snapshot against their previous revisions in the staged or member snapshots. + +``` +GET /api/release-tracks/:id/candidates/:objectRef/diff +GET /api/release-tracks/:id/staged/:objectRef/diff +``` + +If `:objectRef` is a reference to an object that is not part of the candidate snapshot, the endpoint should return a 404 error. If it is part of the candidate snapshot, the endpoint should return a diff between the object in the candidate snapshot and the object in the next lifecycle stage. + +To clarify, snapshot objects transition linearly and unidirectionally through the following tier transitions: Candidate -> Staged -> Member + +An object exists as a set of one or more revisions. An object is identified by its `stix.id` field, whereas an object revision is identified by its `stix.id` and `stix.modified` fields. + +A revision can exist in exactly one tier at a time. + +- If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. +- If it exists in the staged snapshot, it will not exist in the candidate or member snapshots. +- If it exists in the member snapshot, it will not exist in the candidate or staged snapshots. + +If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. However, a _previous_ revision may exist in the staged or member tiers (though it is not guaranteed). Because the tier transitions are unidirectional, revisions must be temporally ordered as it relates to how they are distributed across the tiers. It should not be possible for a newer revision to exist in a previous tier. For example, if a revision exists in the candidate snapshot, it is not possible for a newer revision to exist in the staged or member snapshots. + +This rigidity allows us to implement a diffing endpoint that is specific to release tracks. The diffing endpoint should return a diff between the candidate revision and the next lifecycle stage revision (staged or member). + +So, if an object exists in the candidate snapshot, and another/previous revision of it exists in the members state, the diff endpoint should return a diff between the candidate revision and the member revision. If no previous revision exists in the members state, the diff endpoint should return a diff between the candidate revision and an empty object. + +As another example, if an object exists in the staged tier, the `GET /api/release-tracks/:id/staged/:objectRef/diff` endpoint should return a diff between it and the previous revision that exists in the member tier. If no previous revision exists in the member state, the diff endpoint should return a diff between the candidate revision and an empty object. + +Member revisions are considered immutable and thus cannot be diffed from. Hence, there is no `GET /api/release-tracks/:id/members/:objectRef/diff` endpoint. + +There is one edge case that needs special consideration. If a revision exists as a candidate, a previous revision exists as a member, but no previous revision exists in the staged tier, the diff endpoint now becomes unclear: If the candidate transitions to the next tier, one could argue that the diff should be between the candidate revision and an empty object (since no previous revision exists in the staged tier). However, one could also argue that the diff should be between the candidate revision and the previous member revision. I think the most intuitive approach is to return a diff between the candidate revision and the previous member revision. This is because the candidate revision will eventually transition to the staged tier, and it is more intuitive to compare it against the most recent revision that exists in the next lifecycle stage (member) rather than an empty object. + +To stick with the example, if a revision exists as a candidate, a previous revision exists as staged, and a previous revision exists as a member, the `GET /api/release-tracks/:id/candidates/:objectRef/diff` diff endpoint should return a diff between the candidate revision and the previous staged revision. This is because the candidate revision will eventually transition to the staged tier, and it is more intuitive to compare it against the most recent revision that exists in the next lifecycle stage (staged) rather than an empty object. Similarly, the `GET /api/release-tracks/:id/staged/:objectRef/diff` diff endpoint should return a diff between the staged revision and the previous member revision. This is because the staged revision will eventually transition to the member tier. + +### Idea 2 - Diffing endpoint for all objects (not just release tracks) + +Type-centric: +``` +GET /api/:type/:id/diff +GET /api/:type/:id/modified/:modified/diff +``` + +Type-agnostic: + +Embed the +``` +GET /api/attack-objects/:id/diff +GET /api/attack-objects/:id/modified/:modified/diff +{ + "compareTo": { + "type": "attack-pattern", + "id": "attack-pattern--1234", + "modified": "2024-02-01T00:00:00.000Z", + } +} +``` + +Set up a diffing endpoint that is type-agnostic and allows users to compare any two revisions of an object. The endpoint should accept a request body that specifies the `compareTo` revision, and the endpoint should return a diff between the current revision and the specified `compareTo` revision. +``` +GET /api/compare +{ + "compareFrom": { + "type": "attack-pattern", + "id": "attack-pattern--1234", + "modified": "2024-01-01T00:00:00.000Z", + }, + "compareTo": { + "type": "attack-pattern", + "id": "attack-pattern--1234", + "modified": "2024-02-01T00:00:00.000Z", + } +} +``` + + + +## Reimagining Notes + +- [ ] Implement support for tracking notes on snapshot objects (can be candidates, staged, or members). Notes should be stored in a separate Mongo collection and linked to the snapshot object via a reference field. Users should be able to add, edit, and delete notes via the API. Notably, we already have a notes service that can be leveraged for this purpose. However, it needs some modifications. The service was originally implemented with STIX in mind. The idea was to treat/represent notes as STIX objects and enable users to include them in emitted STIX bundles. However, the concept never really took off. We should modify the service to treat notes as second-class objects that are entirely separate from STIX, but rather as Workbench-native objects. Notes should be capable of being linked/attached to snapshot objects (candidates, staged, or members) as well as to objects independent of snapshots (documents in the `attackObjects` collection). + +Make a new Mongo collection called `notes` to store notes. Each note should have the following fields: + +```json +{ + "_id": "ObjectId", + "content": "string", + "created_by": "string", + "last_modified_by": "string", + "created_at": "Date", + "modified_at": "Date", + "snapshot_object_id": "ObjectId", // Reference to the snapshot object (if applicable) + "object_id": "ObjectId" // Reference to the attack object (if applicable) +} +``` + +Notes will NOT be version controlled. If they are edited or deleted, the changes will be reflected immediately in the database, and recovery and undo functionality will not be supported. + +Links/references between notes and snapshot objects will be one-to-many. A single snapshot object can have multiple notes attached to it, but a note can only be linked to one snapshot object at a time. Similarly, links/references between notes and attack objects will also be one-to-many. These should be bidirectionally tracked, meaning that if a note is linked to an attack object, the attack object should have a reference to the note in its metadata, and vice versa. + +```json +// attackObjects collection +{ + "_id": "ObjectId", + "workspace": { + "notes": ["ObjectId"] // Array of references to notes linked to this attack object + }, + "stix": "StixObject", +} +``` \ No newline at end of file diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 7f8783bb..0e5fc952 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -251,27 +251,35 @@ POST /api/release-tracks/:id/candidates/promote When promoting objects between tiers, conflicts can occur if multiple versions of the same object (same `stix.id`, different `stix.modified` timestamps) exist. Release tracks use **conflict resolution policies** to determine how to handle these situations. **When do conflicts occur?** +- Adding an object to `candidates` (manual add or demotion) when a different version of the object is already pinned in `candidates` - Promoting from `candidates` to `staged` when a different version of the object already exists in `staged` - Promoting from `staged` to `members` (during tagging/release) when a different version already exists in `members` -**Promotions can happen via:** +**Transitions can happen via:** +- **Manual candidate adds** via REST API endpoint (e.g., `POST /api/release-tracks/:id/candidates`) — adding without `modified` resolves the object's latest revision +- **Demotion** back to candidates (`POST /api/release-tracks/:id/staged/demote`) - **Manual promotion** via REST API endpoint (e.g., `POST /api/release-tracks/:id/candidates/promote`) - **Auto-promotion** based on candidacy threshold (e.g., object status changes to `awaiting-review`) - **Tagging/release operations** (e.g., `POST /api/release-tracks/:id/bump`) +Note: revision-sync enrollment (`config.member_sync`, strategy `track_latest`) resolves its overlaps through the `supplant` config rather than these policies — see [member-sync-strategies.md](../../developer/release-tracks/member-sync-strategies.md). + #### Conflict Resolution Policies -Release tracks can be configured with different policies for handling promotion conflicts: +Release tracks can be configured with different policies for handling tier-transition conflicts: ```javascript config: { promotion_conflicts: { + into_candidates: "prefer_latest", // Manual adds / demotions into Candidates candidates_to_staged: "prefer_latest", // Candidates → Staged promotions staged_to_members: "abort" // Staged → Members promotions (during release) } } ``` +Exact duplicates (same `stix.id` *and* same `stix.modified`) are never conflicts: re-adding an identical revision to `candidates` is idempotent and simply skipped. + #### Policy Options ##### 1. `always_overwrite` @@ -433,6 +441,7 @@ PUT /api/release-tracks/:id/config ```json { "promotion_conflicts": { + "into_candidates": "prefer_latest", "candidates_to_staged": "prefer_latest", "staged_to_members": "abort" } @@ -440,6 +449,7 @@ PUT /api/release-tracks/:id/config ``` **Default values:** +- `into_candidates`: `"prefer_latest"` - `candidates_to_staged`: `"prefer_latest"` - `staged_to_members`: `"abort"` From 1ba88339212d934ac8092fc41a36ac8af4ea73e2 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 13 Jul 2026 11:08:44 -0400 Subject: [PATCH 09/55] refactor(controllers): forward delete errors to the exception middleware Legacy deleteById/deleteVersionById handlers in 13 controllers caught all service errors and returned a blanket 500, hiding typed exceptions from the centralized error handler. Migrate them to next(err) per service-exception-middleware.md; also removes stray console.log debugging in the software controller. --- app/controllers/analytics-controller.js | 8 ++++---- app/controllers/assets-controller.js | 8 ++++---- app/controllers/campaigns-controller.js | 8 ++++---- app/controllers/collections-controller.js | 4 ++-- app/controllers/data-sources-controller.js | 8 ++++---- app/controllers/groups-controller.js | 8 ++++---- app/controllers/matrices-controller.js | 8 ++++---- app/controllers/mitigations-controller.js | 8 ++++---- app/controllers/notes-controller.js | 8 ++++---- app/controllers/relationships-controller.js | 8 ++++---- app/controllers/software-controller.js | 12 ++++-------- app/controllers/tactics-controller.js | 8 ++++---- app/controllers/techniques-controller.js | 8 ++++---- 13 files changed, 50 insertions(+), 54 deletions(-) diff --git a/app/controllers/analytics-controller.js b/app/controllers/analytics-controller.js index 961a1a20..188f167f 100644 --- a/app/controllers/analytics-controller.js +++ b/app/controllers/analytics-controller.js @@ -136,7 +136,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const analytic = await analyticsService.deleteVersionById( req.params.stixId, @@ -150,11 +150,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete analytic failed. ' + err); - return res.status(500).send('Unable to delete analytic. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const analytics = await analyticsService.deleteById(req.params.stixId); if (analytics.deletedCount === 0) { @@ -165,6 +165,6 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete analytic failed. ' + err); - return res.status(500).send('Unable to delete analytic. Server error.'); + return next(err); } }; diff --git a/app/controllers/assets-controller.js b/app/controllers/assets-controller.js index 1de1f880..9f0e30ad 100644 --- a/app/controllers/assets-controller.js +++ b/app/controllers/assets-controller.js @@ -143,7 +143,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const assets = await assetsService.deleteById(req.params.stixId); @@ -155,11 +155,11 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete asset failed. ' + err); - return res.status(500).send('Unable to delete asset. Server error.'); + return next(err); } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const asset = await assetsService.deleteVersionById(req.params.stixId, req.params.modified); if (!asset) { @@ -170,7 +170,7 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete asset failed. ' + err); - return res.status(500).send('Unable to delete asset. Server error.'); + return next(err); } }; diff --git a/app/controllers/campaigns-controller.js b/app/controllers/campaigns-controller.js index ebcfd8b3..e9735fb2 100644 --- a/app/controllers/campaigns-controller.js +++ b/app/controllers/campaigns-controller.js @@ -142,7 +142,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const campaign = await campaignsService.deleteVersionById( req.params.stixId, @@ -156,11 +156,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete campaign failed. ' + err); - return res.status(500).send('Unable to delete campaign. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const campaigns = await campaignsService.deleteById(req.params.stixId); if (campaigns.deletedCount === 0) { @@ -171,7 +171,7 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete campaign failed. ' + err); - return res.status(500).send('Unable to delete campaign. Server error.'); + return next(err); } }; diff --git a/app/controllers/collections-controller.js b/app/controllers/collections-controller.js index 538a51a1..255a2a49 100644 --- a/app/controllers/collections-controller.js +++ b/app/controllers/collections-controller.js @@ -209,7 +209,7 @@ exports.delete = async function (req, res) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const removedCollection = await collectionsService.deleteVersionById( req.params.stixId, @@ -224,6 +224,6 @@ exports.deleteVersionById = async function (req, res) { } } catch (error) { logger.error('Delete collection failed. ' + error); - return res.status(500).send('Unable to delete collection. Server error.'); + return next(error); } }; diff --git a/app/controllers/data-sources-controller.js b/app/controllers/data-sources-controller.js index d7933d55..31241561 100644 --- a/app/controllers/data-sources-controller.js +++ b/app/controllers/data-sources-controller.js @@ -145,7 +145,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const dataSource = await dataSourcesService.deleteVersionById( req.params.stixId, @@ -159,11 +159,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete data source failed. ' + err); - return res.status(500).send('Unable to delete data source. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const dataSources = await dataSourcesService.deleteById(req.params.stixId); if (dataSources.deletedCount === 0) { @@ -174,7 +174,7 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete data source failed. ' + err); - return res.status(500).send('Unable to delete data source. Server error.'); + return next(err); } }; diff --git a/app/controllers/groups-controller.js b/app/controllers/groups-controller.js index eeff37e3..1ebbc988 100644 --- a/app/controllers/groups-controller.js +++ b/app/controllers/groups-controller.js @@ -138,7 +138,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const group = await groupsService.deleteVersionById(req.params.stixId, req.params.modified); if (!group) { @@ -149,11 +149,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete group failed. ' + err); - return res.status(500).send('Unable to delete group. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const groups = await groupsService.deleteById(req.params.stixId); if (groups.deletedCount === 0) { @@ -164,7 +164,7 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete group failed. ' + err); - return res.status(500).send('Unable to delete group. Server error.'); + return next(err); } }; diff --git a/app/controllers/matrices-controller.js b/app/controllers/matrices-controller.js index 7e0e7738..a8585e9c 100644 --- a/app/controllers/matrices-controller.js +++ b/app/controllers/matrices-controller.js @@ -137,7 +137,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const matrix = await matricesService.deleteVersionById(req.params.stixId, req.params.modified); if (!matrix) { @@ -148,11 +148,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete matrix failed. ' + err); - return res.status(500).send('Unable to delete matrix. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const matrices = await matricesService.deleteById(req.params.stixId); @@ -164,7 +164,7 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete matrix failed. ' + err); - return res.status(500).send('Unable to delete matrix. Server error.'); + return next(err); } }; diff --git a/app/controllers/mitigations-controller.js b/app/controllers/mitigations-controller.js index ddaec0cf..a2fb4bb3 100644 --- a/app/controllers/mitigations-controller.js +++ b/app/controllers/mitigations-controller.js @@ -140,7 +140,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const mitigation = await mitigationsService.deleteVersionById( req.params.stixId, @@ -154,11 +154,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete mitigation failed. ' + err); - return res.status(500).send('Unable to delete mitigation. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const mitigations = await mitigationsService.deleteById(req.params.stixId); if (mitigations.deletedCount === 0) { @@ -169,7 +169,7 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete mitigation failed. ' + err); - return res.status(500).send('Unable to delete mitigation. Server error.'); + return next(err); } }; diff --git a/app/controllers/notes-controller.js b/app/controllers/notes-controller.js index 669e022a..de307536 100644 --- a/app/controllers/notes-controller.js +++ b/app/controllers/notes-controller.js @@ -134,7 +134,7 @@ exports.updateVersion = async function (req, res, next) { } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const results = await notesService.deleteById(req.params.stixId); if (results.deletedCount === 0) { @@ -145,11 +145,11 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete note failed. ' + err); - return res.status(500).send('Unable to delete note. Server error.'); + return next(err); } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const note = await notesService.deleteVersionById(req.params.stixId, req.params.modified); if (!note) { @@ -162,6 +162,6 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete note version failed. ' + err); - return res.status(500).send('Unable to delete note. Server error.'); + return next(err); } }; diff --git a/app/controllers/relationships-controller.js b/app/controllers/relationships-controller.js index 972d75ee..5a76b614 100644 --- a/app/controllers/relationships-controller.js +++ b/app/controllers/relationships-controller.js @@ -147,7 +147,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const relationship = await relationshipsService.deleteVersionById( req.params.stixId, @@ -161,11 +161,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete relationship failed. ' + err); - return res.status(500).send('Unable to delete relationship. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const relationships = await relationshipsService.deleteById(req.params.stixId); if (relationships.deletedCount === 0) { @@ -176,6 +176,6 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete relationship failed. ' + err); - return res.status(500).send('Unable to delete relationship. Server error.'); + return next(err); } }; diff --git a/app/controllers/software-controller.js b/app/controllers/software-controller.js index 4f1f7717..c357e0a5 100644 --- a/app/controllers/software-controller.js +++ b/app/controllers/software-controller.js @@ -150,7 +150,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const software = await softwareService.deleteVersionById( req.params.stixId, @@ -164,14 +164,12 @@ exports.deleteVersionById = async function (req, res) { return res.status(204).end(); } } catch (err) { - console.log('delete version by id error'); - console.log(err); logger.error('Delete software failed. ' + err); - return res.status(500).send('Unable to delete software. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const softwares = await softwareService.deleteById(req.params.stixId); @@ -182,10 +180,8 @@ exports.deleteById = async function (req, res) { return res.status(204).end(); } } catch (err) { - console.log('delete by id error'); - console.log(err); logger.error('Delete software failed. ' + err); - return res.status(500).send('Unable to delete software. Server error.'); + return next(err); } }; diff --git a/app/controllers/tactics-controller.js b/app/controllers/tactics-controller.js index 4c7cdfd2..adae839b 100644 --- a/app/controllers/tactics-controller.js +++ b/app/controllers/tactics-controller.js @@ -144,7 +144,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const tactic = await tacticsService.deleteVersionById(req.params.stixId, req.params.modified); @@ -156,11 +156,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete tactic failed. ' + err); - return res.status(500).send('Unable to delete tactic. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const tactics = await tacticsService.deleteById(req.params.stixId); @@ -172,7 +172,7 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete tactic failed. ' + err); - return res.status(500).send('Unable to delete tactic. Server error.'); + return next(err); } }; diff --git a/app/controllers/techniques-controller.js b/app/controllers/techniques-controller.js index 68394bde..d00692a0 100644 --- a/app/controllers/techniques-controller.js +++ b/app/controllers/techniques-controller.js @@ -143,7 +143,7 @@ exports.updateFull = async function (req, res, next) { } }; -exports.deleteVersionById = async function (req, res) { +exports.deleteVersionById = async function (req, res, next) { try { const technique = await techniquesService.deleteVersionById( req.params.stixId, @@ -157,11 +157,11 @@ exports.deleteVersionById = async function (req, res) { } } catch (err) { logger.error('Delete technique failed. ' + err); - return res.status(500).send('Unable to delete technique. Server error.'); + return next(err); } }; -exports.deleteById = async function (req, res) { +exports.deleteById = async function (req, res, next) { try { const techniques = await techniquesService.deleteById(req.params.stixId); if (techniques.deletedCount === 0) { @@ -172,7 +172,7 @@ exports.deleteById = async function (req, res) { } } catch (err) { logger.error('Delete technique failed. ' + err); - return res.status(500).send('Unable to delete technique. Server error.'); + return next(err); } }; From 812d5bcaaf3f93f2e46b18aa110228d68b631546 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 13 Jul 2026 11:09:49 -0400 Subject: [PATCH 10/55] feat(api): reject in-place mutation of members-pinned revisions Members-pinned revisions are released content: PUT and DELETE (single version or all versions) now return 409 (MemberPinnedRevisionError) when any release track pins the revision in its members tier, with guidance to create a new revision instead (x_mitre_deprecated to retire). Adds the 409 responses to the affected OpenAPI operations. --- app/api/definitions/paths/analytics-paths.yml | 6 +++ app/api/definitions/paths/assets-paths.yml | 4 ++ app/api/definitions/paths/campaigns-paths.yml | 6 +++ .../paths/data-components-paths.yml | 6 +++ .../definitions/paths/data-sources-paths.yml | 4 ++ .../paths/detection-strategies-paths.yml | 6 +++ app/api/definitions/paths/groups-paths.yml | 6 +++ .../definitions/paths/identities-paths.yml | 4 ++ app/api/definitions/paths/matrices-paths.yml | 6 +++ .../definitions/paths/mitigations-paths.yml | 6 +++ app/api/definitions/paths/notes-paths.yml | 6 +++ .../definitions/paths/relationships-paths.yml | 6 +++ app/api/definitions/paths/software-paths.yml | 6 +++ app/api/definitions/paths/tactics-paths.yml | 6 +++ .../definitions/paths/techniques-paths.yml | 6 +++ app/exceptions/index.js | 12 +++++ app/lib/error-handler.js | 2 + app/repository/_base.repository.js | 21 ++++++++ app/services/meta-classes/base.service.js | 53 ++++++++++++++++++- 19 files changed, 171 insertions(+), 1 deletion(-) diff --git a/app/api/definitions/paths/analytics-paths.yml b/app/api/definitions/paths/analytics-paths.yml index 0e5e9b2c..16d84795 100644 --- a/app/api/definitions/paths/analytics-paths.yml +++ b/app/api/definitions/paths/analytics-paths.yml @@ -205,6 +205,8 @@ paths: description: 'All the analytic versions were successfully deleted.' '404': description: 'A analytic with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/analytics/{stixId}/modified/{modified}: get: @@ -275,6 +277,8 @@ paths: description: 'Missing or invalid parameters were provided. The analytic was not updated.' '404': description: 'A analytic with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a analytic' operationId: 'analytic-delete' @@ -301,3 +305,5 @@ paths: description: 'The analytic was successfully deleted.' '404': description: 'A analytic with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/assets-paths.yml b/app/api/definitions/paths/assets-paths.yml index 5f8d1585..594eac6c 100644 --- a/app/api/definitions/paths/assets-paths.yml +++ b/app/api/definitions/paths/assets-paths.yml @@ -269,6 +269,8 @@ paths: description: 'Missing or invalid parameters were provided. The asset was not updated.' '404': description: 'An asset with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete an asset' operationId: 'asset-delete' @@ -295,6 +297,8 @@ paths: description: 'The asset was successfully deleted.' '404': description: 'An asset with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/assets/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/campaigns-paths.yml b/app/api/definitions/paths/campaigns-paths.yml index e643c3aa..37ec83bb 100644 --- a/app/api/definitions/paths/campaigns-paths.yml +++ b/app/api/definitions/paths/campaigns-paths.yml @@ -177,6 +177,8 @@ paths: description: 'All the campaign versions were successfully deleted.' '404': description: 'A campaign with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/campaigns/{stixId}/modified/{modified}: get: @@ -247,6 +249,8 @@ paths: description: 'Missing or invalid parameters were provided. The campaign was not updated.' '404': description: 'A campaign with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a campaign' operationId: 'campaign-delete' @@ -273,6 +277,8 @@ paths: description: 'The campaign was successfully deleted.' '404': description: 'A campaign with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/campaigns/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/data-components-paths.yml b/app/api/definitions/paths/data-components-paths.yml index 74dbeb04..1e260c73 100644 --- a/app/api/definitions/paths/data-components-paths.yml +++ b/app/api/definitions/paths/data-components-paths.yml @@ -189,6 +189,8 @@ paths: description: 'All the data component versions were successfully deleted.' '404': description: 'A data component with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/data-components/{stixId}/channels: get: @@ -313,6 +315,8 @@ paths: description: 'Missing or invalid parameters were provided. The data component was not updated.' '404': description: 'A data component with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a data component' operationId: 'data-component-delete' @@ -339,6 +343,8 @@ paths: description: 'The data component was successfully deleted.' '404': description: 'A data component with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/data-components/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/data-sources-paths.yml b/app/api/definitions/paths/data-sources-paths.yml index 2b609476..c301a6af 100644 --- a/app/api/definitions/paths/data-sources-paths.yml +++ b/app/api/definitions/paths/data-sources-paths.yml @@ -285,6 +285,8 @@ paths: description: 'Missing or invalid parameters were provided. The data source was not updated.' '404': description: 'A data source with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a data source' operationId: 'data-source-delete' @@ -311,6 +313,8 @@ paths: description: 'The data source was successfully deleted.' '404': description: 'A data source with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/data-sources/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/detection-strategies-paths.yml b/app/api/definitions/paths/detection-strategies-paths.yml index 9ff84498..a038c768 100644 --- a/app/api/definitions/paths/detection-strategies-paths.yml +++ b/app/api/definitions/paths/detection-strategies-paths.yml @@ -189,6 +189,8 @@ paths: description: 'All the detection strategy versions were successfully deleted.' '404': description: 'A detection strategy with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/detection-strategies/{stixId}/modified/{modified}: get: @@ -259,6 +261,8 @@ paths: description: 'Missing or invalid parameters were provided. The detection strategy was not updated.' '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a detection strategy' operationId: 'detection-strategy-delete' @@ -285,3 +289,5 @@ paths: description: 'The detection strategy was successfully deleted.' '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/groups-paths.yml b/app/api/definitions/paths/groups-paths.yml index 4b736e37..cbe8dd3b 100644 --- a/app/api/definitions/paths/groups-paths.yml +++ b/app/api/definitions/paths/groups-paths.yml @@ -177,6 +177,8 @@ paths: description: 'All the group versions were successfully deleted.' '404': description: 'A group with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/groups/{stixId}/modified/{modified}: get: @@ -247,6 +249,8 @@ paths: description: 'Missing or invalid parameters were provided. The group was not updated.' '404': description: 'A group with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a group' operationId: 'group-delete' @@ -273,6 +277,8 @@ paths: description: 'The group was successfully deleted.' '404': description: 'A group with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/groups/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/identities-paths.yml b/app/api/definitions/paths/identities-paths.yml index edca01cf..1c104789 100644 --- a/app/api/definitions/paths/identities-paths.yml +++ b/app/api/definitions/paths/identities-paths.yml @@ -226,6 +226,8 @@ paths: description: 'Missing or invalid parameters were provided. The identity was not updated.' '404': description: 'An identity with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a identity' operationId: 'identity-delete' @@ -252,3 +254,5 @@ paths: description: 'The identity was successfully deleted.' '404': description: 'An identity with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/matrices-paths.yml b/app/api/definitions/paths/matrices-paths.yml index f7f84956..a49d0123 100644 --- a/app/api/definitions/paths/matrices-paths.yml +++ b/app/api/definitions/paths/matrices-paths.yml @@ -177,6 +177,8 @@ paths: description: 'All the matrix versions were successfully deleted.' '404': description: 'A matrix with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}: get: @@ -247,6 +249,8 @@ paths: description: 'Missing or invalid parameters were provided. The matrix was not updated.' '404': description: 'A matrix with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a matrix' operationId: 'matrix-delete' @@ -273,6 +277,8 @@ paths: description: 'The matrix was successfully deleted.' '404': description: 'A matrix with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/mitigations-paths.yml b/app/api/definitions/paths/mitigations-paths.yml index 4189babd..f1253584 100644 --- a/app/api/definitions/paths/mitigations-paths.yml +++ b/app/api/definitions/paths/mitigations-paths.yml @@ -189,6 +189,8 @@ paths: description: 'All the mitigation versions were successfully deleted.' '404': description: 'A mitigation with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/mitigations/{stixId}/modified/{modified}: get: @@ -259,6 +261,8 @@ paths: description: 'Missing or invalid parameters were provided. The mitigation was not updated.' '404': description: 'A mitigation with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a mitigation' operationId: 'mitigation-delete' @@ -285,6 +289,8 @@ paths: description: 'The mitigation was successfully deleted.' '404': description: 'A mitigation with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/mitigations/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/notes-paths.yml b/app/api/definitions/paths/notes-paths.yml index c246d8de..f83752ef 100644 --- a/app/api/definitions/paths/notes-paths.yml +++ b/app/api/definitions/paths/notes-paths.yml @@ -175,6 +175,8 @@ paths: description: 'The note was successfully deleted.' '404': description: 'A note with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/notes/{stixId}/modified/{modified}: get: @@ -244,6 +246,8 @@ paths: description: 'Missing or invalid parameters were provided. The note was not updated.' '404': description: 'A note with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a note' operationId: 'note-delete-version' @@ -270,3 +274,5 @@ paths: description: 'The note was successfully deleted.' '404': description: 'A note with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/relationships-paths.yml b/app/api/definitions/paths/relationships-paths.yml index e25940fc..0de8060f 100644 --- a/app/api/definitions/paths/relationships-paths.yml +++ b/app/api/definitions/paths/relationships-paths.yml @@ -247,6 +247,8 @@ paths: description: 'All the relationship versions were successfully deleted.' '404': description: 'A relationship with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/relationships/{stixId}/modified/{modified}: get: @@ -317,6 +319,8 @@ paths: description: 'Missing or invalid parameters were provided. The relationship was not updated.' '404': description: 'A relationship with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a relationship' operationId: 'relationship-delete' @@ -343,3 +347,5 @@ paths: description: 'The relationship was successfully deleted.' '404': description: 'A relationship with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/software-paths.yml b/app/api/definitions/paths/software-paths.yml index e8d1fe25..33831f85 100644 --- a/app/api/definitions/paths/software-paths.yml +++ b/app/api/definitions/paths/software-paths.yml @@ -200,6 +200,8 @@ paths: description: 'All the tactic versions were successfully deleted.' '404': description: 'A tactic with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/software/{stixId}/modified/{modified}: get: @@ -269,6 +271,8 @@ paths: description: 'Missing or invalid parameters were provided. The software object was not updated.' '404': description: 'A software object with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a software object' operationId: 'software-delete' @@ -295,6 +299,8 @@ paths: description: 'The software object was successfully deleted.' '404': description: 'A software object with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/software/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/tactics-paths.yml b/app/api/definitions/paths/tactics-paths.yml index f402dfb9..6caadde9 100644 --- a/app/api/definitions/paths/tactics-paths.yml +++ b/app/api/definitions/paths/tactics-paths.yml @@ -189,6 +189,8 @@ paths: description: 'All the tactic versions were successfully deleted.' '404': description: 'A tactic with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}: get: @@ -259,6 +261,8 @@ paths: description: 'Missing or invalid parameters were provided. The tactic was not updated.' '404': description: 'A tactic with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a tactic' operationId: 'tactic-delete' @@ -285,6 +289,8 @@ paths: description: 'The tactic was successfully deleted.' '404': description: 'A tactic with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/techniques-paths.yml b/app/api/definitions/paths/techniques-paths.yml index 918372ee..050535e6 100644 --- a/app/api/definitions/paths/techniques-paths.yml +++ b/app/api/definitions/paths/techniques-paths.yml @@ -213,6 +213,8 @@ paths: description: 'All the technique versions were successfully deleted.' '404': description: 'A technique with the requested STIX id was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}: get: @@ -283,6 +285,8 @@ paths: description: 'Missing or invalid parameters were provided. The technique was not updated.' '404': description: 'A technique with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a technique' operationId: 'technique-delete' @@ -309,6 +313,8 @@ paths: description: 'The technique was successfully deleted.' '404': description: 'A technique with the requested STIX id and modified date was not found.' + '409': + description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}/tactics: get: diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 03df60f3..c84af8ad 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -297,6 +297,17 @@ class AlreadyReleasedError extends CustomError { } } +class MemberPinnedRevisionError extends CustomError { + constructor(options) { + super( + 'This revision is pinned in the members tier of a release track and is released content: ' + + 'it cannot be modified or deleted in place. Create a new revision instead ' + + '(set x_mitre_deprecated on a new revision to retire the object).', + options, + ); + } +} + class InvalidVersionError extends CustomError { constructor(message, options) { super(message || 'Invalid version', options); @@ -367,6 +378,7 @@ module.exports = { NoTaggedSnapshotsError, InvalidComponentTypeError, TrackNotFoundError, + MemberPinnedRevisionError, //** Database-related errors */ DuplicateIdError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 1fc1625f..78633c7e 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -41,6 +41,7 @@ const { NoTaggedSnapshotsError, InvalidComponentTypeError, TrackNotFoundError, + MemberPinnedRevisionError, ObjectHasValidationIssuesError, } = require('../exceptions'); @@ -130,6 +131,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof AlreadyRevokedError || err instanceof AlreadyReleasedError || err instanceof ReleaseConflictError || + err instanceof MemberPinnedRevisionError || err instanceof ObjectHasValidationIssuesError || err instanceof ActiveOrganizationIdentityDeleteError ) { diff --git a/app/repository/_base.repository.js b/app/repository/_base.repository.js index 7140cd57..fd024a38 100644 --- a/app/repository/_base.repository.js +++ b/app/repository/_base.repository.js @@ -550,6 +550,27 @@ class BaseRepository extends AbstractRepository { } } + /** + * Retrieve the revisions of an object that any release track pins in its + * members tier. Lean, minimal projection — used to guard delete + * operations (members-pinned revisions are released content and must not + * be destroyed). + * + * @param {string} stixId - The STIX ID + * @returns {Promise} Lean documents with stix.id, stix.modified, workspace.release_tracks + */ + async retrieveMemberPinnedVersionsLean(stixId) { + try { + return await this.model + .find({ 'stix.id': stixId, 'workspace.release_tracks.tier': 'members' }) + .select('stix.id stix.modified workspace.release_tracks') + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + /** * Retrieve all documents carrying a workspace.release_tracks entry for the * given release track. Lean, minimal projection — used by release-track diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index c6f7be7a..864ae03b 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -22,6 +22,7 @@ const { NotFoundError, AlreadyRevokedError, SelfRevocationError, + MemberPinnedRevisionError, } = require('../../exceptions'); const { getSchema } = require('../../lib/validation-schemas'); const { deepFreezeStix } = require('../../lib/import-safety'); @@ -702,6 +703,32 @@ class BaseService extends ServiceWithHooks { return result; } + /** + * Reject in-place mutation (PUT/DELETE) of a revision that any release + * track pins in its members tier. Members are released content: mutating + * or deleting the pinned document would silently change or break what the + * track ships. Changes go through a new revision (POST) — which revision + * sync captures — including retirement via x_mitre_deprecated. + * + * @param {Object} document - The stored document ({ workspace, stix }) + * @param {string} operation - Verb for the error message ('updated'|'deleted') + */ + static assertNotMemberPinned(document, operation) { + const memberPins = (document.workspace?.release_tracks || []).filter( + (entry) => entry.tier === 'members', + ); + if (memberPins.length > 0) { + throw new MemberPinnedRevisionError({ + details: + `Revision ${document.stix.id} (modified ` + + `${new Date(document.stix.modified).toISOString()}) is pinned in the members tier of ` + + `release track(s) ${memberPins.map((entry) => entry.id).join(', ')} and cannot be ` + + `${operation} in place. Create a new revision instead (set x_mitre_deprecated on a ` + + `new revision to retire the object).`, + }); + } + } + /** * Refresh workspace.release_tracks on a response object after domain * events have run. The created/updated event is awaited, and its listeners @@ -896,7 +923,7 @@ class BaseService extends ServiceWithHooks { // Revision identity is immutable in place: a PUT may not re-key the // document (release tracks pin revisions by stix.id + stix.modified; // re-keying would strand those pins). Re-keying must go through POST, - // which creates a new revision that member sync captures. + // which creates a new revision that revision sync captures. if (data.stix?.id && data.stix.id !== stixId) { throw new BadRequestError({ details: `Body stix.id (${data.stix.id}) must match the stixId path parameter (${stixId})`, @@ -917,6 +944,10 @@ class BaseService extends ServiceWithHooks { if (!document) { return null; } + + // Members-pinned revisions are released content — immutable in place. + BaseService.assertNotMemberPinned(document, 'updated'); + // TODO: diff analysis — detect field-level changes vs document // TODO: if no changes detected, short-circuit (no-op) @@ -1032,6 +1063,14 @@ class BaseService extends ServiceWithHooks { await this.beforeDeleteVersionById(stixId, stixModified); + // Members-pinned revisions are released content — they must never be + // deleted (the track's member entry would silently dangle). + const existing = await this.repository.retrieveOneByVersion(stixId, stixModified); + if (!existing) { + return null; + } + BaseService.assertNotMemberPinned(existing, 'deleted'); + const document = await this.repository.findOneAndDelete(stixId, stixModified); if (!document) { @@ -1317,6 +1356,11 @@ class BaseService extends ServiceWithHooks { }); result.mergeEventResults(eventResults); + // Revision sync (listening on the revoked event) may have enrolled or + // re-pinned the revoked revision in its tracks — refresh so the response + // carries the resulting backrefs. + await this._refreshReleaseTrackBackrefs(revokedDocument); + // ────────────────────────────────────────────── // 9. RETURN RESULT // ────────────────────────────────────────────── @@ -1329,6 +1373,13 @@ class BaseService extends ServiceWithHooks { throw new MissingParameterError('stixId'); } await this.beforeDeleteById(stixId); + + // Deleting all versions must not destroy a members-pinned revision + const memberPinned = await this.repository.retrieveMemberPinnedVersionsLean(stixId); + for (const pinnedDocument of memberPinned) { + BaseService.assertNotMemberPinned(pinnedDocument, 'deleted'); + } + const result = await this.repository.deleteMany(stixId); if (result.deletedCount > 0) { await this.afterDeleteById(stixId, result); From d4adfc09383943942815cd2563daa0534dc2d740 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 13 Jul 2026 11:09:55 -0400 Subject: [PATCH 11/55] fix(release-tracks): capture in-place edits and revocations in revision sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two blind spots closed: (1) member sync now subscribes to the per-type ::revoked events, so the revoked revision is enrolled as a candidate in member tracks and candidate/staged pins move to it — previously a track silently kept exporting the pre-revoke revision; the revoke response carries the resulting backrefs. (2) In-place PUTs of candidate/staged- pinned revisions reset the entry for re-review (staged demotes to candidates), including in-place deprecation; enrollment of already-pinned revisions and no-op snapshot clones are suppressed, fixing the same-key duplicate cross-tier enrollment misfire. --- .../release-tracks/member-sync-service.js | 107 +++++- .../release-tracks-backrefs.spec.js | 19 +- .../release-tracks-change-capture.spec.js | 359 ++++++++++++++++++ docs/developer/TODO.md | 10 +- .../release-tracks/member-sync-strategies.md | 19 + docs/user/release-tracks/object-backrefs.md | 25 ++ 6 files changed, 526 insertions(+), 13 deletions(-) create mode 100644 app/tests/api/release-tracks/release-tracks-change-capture.spec.js diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index 8b204038..12d302c3 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -205,6 +205,49 @@ async function processMemberSync(trackId, snapshot, event) { if (!action) return null; + const incomingTime = new Date(newModified).getTime(); + + if (action.type === 'enroll') { + // Skip if this exact revision is already pinned in any tier — enrolling + // it again would create a duplicate cross-tier reference (e.g. a + // re-import announcing an already-released revision). + const alreadyPinned = ['members', 'staged', 'candidates'].some((tier) => + (snapshot[tier] || []).some( + (e) => e.object_ref === objectRef && new Date(e.object_modified).getTime() === incomingTime, + ), + ); + if (alreadyPinned) { + logger.debug( + `[member-sync] Track ${trackId}: revision ${objectRef} @ ` + + `${new Date(newModified).toISOString()} is already pinned, skipping enrollment`, + ); + return null; + } + } + + if (action.type === 'replace') { + // An in-place edit of the pinned revision arrives with the same + // object_modified: the pin key does not change. Act only when the + // outcome differs — a reviewed entry resets for re-review, a staged + // entry demotes back to candidates — otherwise skip instead of cloning + // a no-op snapshot. + const existingTime = new Date(action.removeEntry.object_modified).getTime(); + const currentStatus = action.removeEntry.object_status || 'work-in-progress'; + const resultingStatus = + config.supplant.status_policy === 'preserve' ? currentStatus : 'work-in-progress'; + if ( + existingTime === incomingTime && + action.targetTier === existingTier && + resultingStatus === currentStatus + ) { + logger.debug( + `[member-sync] Track ${trackId}: in-place update of ${objectRef} leaves the ` + + `pinned entry unchanged, skipping`, + ); + return null; + } + } + // Build the new candidate/staged entry const now = new Date(); const newEntry = { @@ -379,19 +422,75 @@ async function handleStixObjectEvent(payload) { } } +/** + * All STIX object revoked events. The revoke workflow saves the revoked + * revision directly via the repository (no ::created/::updated fires), so + * without this subscription a track would silently keep exporting the + * pre-revoke revision. + */ +const STIX_OBJECT_REVOKED_EVENTS = [ + EventConstants.ATTACK_PATTERN_REVOKED, + EventConstants.TACTIC_REVOKED, + EventConstants.COURSE_OF_ACTION_REVOKED, + EventConstants.INTRUSION_SET_REVOKED, + EventConstants.MALWARE_REVOKED, + EventConstants.TOOL_REVOKED, + EventConstants.CAMPAIGN_REVOKED, + EventConstants.DATA_SOURCE_REVOKED, + EventConstants.DATA_COMPONENT_REVOKED, + EventConstants.MATRIX_REVOKED, + EventConstants.ASSET_REVOKED, +]; + +/** + * Handle a STIX object revoked event from BaseService.revoke(). + * + * The revoked payload shape differs from created/updated: the new revision + * (revoked: true) arrives as payload.revokedDocument. Treat it like any + * other new revision — enroll it in member tracks, move candidate/staged + * pins per the supplant config. + * + * @param {Object} payload - Event payload from BaseService.revoke() + * @param {string} payload.stixId - The STIX ID of the revoked object + * @param {Object} payload.revokedDocument - The new revoked revision + * @param {Object} [payload.options] - Revocation options + */ +async function handleStixObjectRevokedEvent(payload) { + const { stixId, revokedDocument, options } = payload; + + const event = { + objectRef: stixId, + newModified: revokedDocument?.stix?.modified, + modifiedBy: + options?.userAccountId || + revokedDocument?.workspace?.workflow?.created_by_user_account || + 'system', + }; + + try { + await exports.handleObjectModified(event); + } catch (err) { + logger.error(`[member-sync] Error handling object revocation: ${err.message}`, err); + } +} + /** * Initialize event listeners for member sync. * - * Subscribes to all STIX object created/updated events via the EventBus. - * Called automatically when this module is loaded. + * Subscribes to all STIX object created/updated/revoked events via the + * EventBus. Called automatically when this module is loaded. */ function initializeEventListeners() { for (const eventName of STIX_OBJECT_EVENTS) { EventBus.on(eventName, handleStixObjectEvent); } + for (const eventName of STIX_OBJECT_REVOKED_EVENTS) { + EventBus.on(eventName, handleStixObjectRevokedEvent); + } logger.info( - `[member-sync] Member sync service initialized, listening to ${STIX_OBJECT_EVENTS.length} event types`, + `[member-sync] Member sync service initialized, listening to ` + + `${STIX_OBJECT_EVENTS.length + STIX_OBJECT_REVOKED_EVENTS.length} event types`, ); } @@ -408,5 +507,7 @@ exports._internal = { processMemberSync, getMemberSyncConfig, handleStixObjectEvent, + handleStixObjectRevokedEvent, STIX_OBJECT_EVENTS, + STIX_OBJECT_REVOKED_EVENTS, }; diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 3a2d7ece..4f205141 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -647,21 +647,28 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); - // The revoked revision is a new version — it must not inherit backrefs + // The revoked revision carries a backref only via revision sync (the + // candidate pin moved to it) — never via clone-copying: the entry is + // the re-pinned candidate, not the fake members entry a copy would show expect(res.body.primary.stix.revoked).toBe(true); - expect(res.body.primary.workspace.release_tracks).toBeUndefined(); + expect(entryForTrack(res.body.primary, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + const oldTechniqueRevision = await getTechniqueVersion(techniqueA); + expect(entryForTrack(oldTechniqueRevision, trackId)).toBeUndefined(); // The relationship referencing the revoked object was deprecated into a - // new revision — it must not inherit backrefs either + // new revision — relationships are not revision-synced, so any backref + // here would be a clone leak const latestRels = await getObjectVersion(`/api/relationships/${relationship.stix.id}`); const latestRel = latestRels[0]; expect(latestRel.stix.x_mitre_deprecated).toBe(true); expect(latestRel.stix.modified).not.toBe(relationship.stix.modified); expect(latestRel.workspace.release_tracks).toBeUndefined(); - // The pinned revisions keep their backrefs - const pinnedTechnique = await getTechniqueVersion(techniqueA); - expect(entryForTrack(pinnedTechnique, trackId)).toMatchObject({ tier: 'candidates' }); + // The pinned relationship revision keeps its backref (its pin did not move) const pinnedRel = await getObjectVersion( `/api/relationships/${relationship.stix.id}/modified/${relationship.stix.modified}`, ); diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js new file mode 100644 index 00000000..3ba9914a --- /dev/null +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -0,0 +1,359 @@ +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +const logger = require('../../../lib/logger'); +logger.level = 'debug'; + +const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { + workflow: { + state: 'work-in-progress', + }, + }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +// Release tracks must never be blind to in-place mutations: +// - PUT/DELETE of a members-pinned revision is rejected (409) — released +// content is immutable in place; changes go through a new revision. +// - PUT of a candidate/staged-pinned revision resets the tier entry for +// re-review (staged entries demote back to candidates). +// - Revoking a tracked object enrolls/re-pins the revoked revision. +describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function postObject(path, body, expectedStatus = 201) { + const res = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return res.body; + } + + async function getJson(path, expectedStatus = 200) { + const res = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return res.body; + } + + async function getTechniqueVersion(stixId, modified) { + return getJson(`/api/techniques/${stixId}/modified/${modified}`); + } + + function putTechnique(technique, body) { + return request(app) + .put(`/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + async function createTrack(name) { + const res = await postObject('/api/release-tracks/new', { name, type: 'standard' }); + return res.id; + } + + async function addCandidate(trackId, technique) { + return postObject( + `/api/release-tracks/${trackId}/candidates`, + { object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }] }, + 200, + ); + } + + async function setMembers(trackId, technique) { + return postObject( + `/api/release-tracks/${trackId}/contents`, + { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }, + 200, + ); + } + + async function latestSnapshotModified(trackId) { + const snapshot = await getJson(`/api/release-tracks/${trackId}`); + return snapshot.modified; + } + + function entryForTrack(object, trackId) { + return (object.workspace.release_tracks || []).find((e) => e.id === trackId); + } + + function buildUpdateBody(technique, name) { + const update = buildTechnique(name); + update.stix.id = technique.stix.id; + update.stix.created = technique.stix.created; + update.stix.modified = technique.stix.modified; + return update; + } + + describe('members-pinned revisions are immutable in place', function () { + let trackId; + let technique; + + before(async function () { + technique = await postObject('/api/techniques', buildTechnique('Capture Member')); + trackId = await createTrack('Capture Member Track'); + await setMembers(trackId, technique); + }); + + it('rejects a PUT of a members-pinned revision with 409', async function () { + const res = await putTechnique( + technique, + buildUpdateBody(technique, 'Capture Member (edited)'), + ).expect(409); + expect(res.text).toContain('members tier'); + + const retrieved = await getTechniqueVersion(technique.stix.id, technique.stix.modified); + expect(retrieved.stix.name).toBe('Capture Member'); + expect(entryForTrack(retrieved, trackId)).toEqual({ + id: trackId, + tier: 'members', + status: 'reviewed', + }); + }); + + it('rejects a DELETE of a members-pinned revision with 409', async function () { + const res = await request(app) + .delete(`/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + expect(res.text).toContain('members tier'); + + await getTechniqueVersion(technique.stix.id, technique.stix.modified); + }); + + it('rejects a DELETE of all versions when any revision is members-pinned', async function () { + // Add a second (untracked) revision — the delete-all must still be + // rejected because the first revision is members-pinned + const revisionB = buildTechnique('Capture Member v2'); + revisionB.stix.id = technique.stix.id; + revisionB.stix.created = technique.stix.created; + revisionB.stix.modified = new Date( + new Date(technique.stix.modified).getTime() + 60000, + ).toISOString(); + await postObject('/api/techniques', revisionB); + + const res = await request(app) + .delete(`/api/techniques/${technique.stix.id}`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + expect(res.text).toContain('members tier'); + + await getTechniqueVersion(technique.stix.id, technique.stix.modified); + }); + }); + + describe('in-place edits of candidate/staged-pinned revisions', function () { + it('resets a reviewed candidate entry to work-in-progress on in-place PUT', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Candidate')); + const trackId = await createTrack('Capture Candidate Track'); + await addCandidate(trackId, technique); + await postObject( + `/api/release-tracks/${trackId}/candidates/review`, + { from: 'work-in-progress', to: 'awaiting-review' }, + 200, + ); + + const res = await putTechnique( + technique, + buildUpdateBody(technique, 'Capture Candidate (edited)'), + ).expect(200); + + // The PUT response reflects the reset (read-your-own-writes) + expect(entryForTrack(res.body, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + + const { candidates } = await getJson(`/api/release-tracks/${trackId}/candidates`); + expect(candidates).toHaveLength(1); + expect(candidates[0].object_status).toBe('work-in-progress'); + expect(new Date(candidates[0].object_modified).toISOString()).toBe(technique.stix.modified); + }); + + it('demotes a staged entry back to candidates on in-place PUT', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Staged')); + const trackId = await createTrack('Capture Staged Track'); + await addCandidate(trackId, technique); + await postObject( + `/api/release-tracks/${trackId}/candidates/promote`, + { object_refs: [technique.stix.id] }, + 200, + ); + + const res = await putTechnique( + technique, + buildUpdateBody(technique, 'Capture Staged (edited)'), + ).expect(200); + + expect(entryForTrack(res.body, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + + const snapshot = await getJson(`/api/release-tracks/${trackId}`); + expect(snapshot.staged).toHaveLength(0); + expect(snapshot.candidates).toHaveLength(1); + }); + + it('captures in-place deprecation of a reviewed candidate', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Deprecate')); + const trackId = await createTrack('Capture Deprecate Track'); + await addCandidate(trackId, technique); + await postObject( + `/api/release-tracks/${trackId}/candidates/review`, + { from: 'work-in-progress', to: 'awaiting-review' }, + 200, + ); + + const update = buildUpdateBody(technique, 'Capture Deprecate'); + update.stix.x_mitre_deprecated = true; + const res = await putTechnique(technique, update).expect(200); + + expect(res.body.stix.x_mitre_deprecated).toBe(true); + // The track saw the deprecation: the entry is back to work-in-progress + expect(entryForTrack(res.body, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + + it('does not clone a snapshot when the in-place PUT changes nothing track-visible', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Noop')); + const trackId = await createTrack('Capture Noop Track'); + await addCandidate(trackId, technique); + + const before = await latestSnapshotModified(trackId); + + // Candidate is already work-in-progress — the reset is a no-op, so no + // new snapshot should be created + await putTechnique(technique, buildUpdateBody(technique, 'Capture Noop (edited)')).expect( + 200, + ); + + const after = await latestSnapshotModified(trackId); + expect(after).toBe(before); + }); + + it('still allows DELETE of a candidate-pinned revision', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Del Cand')); + const trackId = await createTrack('Capture Del Cand Track'); + await addCandidate(trackId, technique); + + await request(app) + .delete(`/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(204); + }); + }); + + describe('revocation reaches the release track', function () { + async function revokeTechnique(revoked, revoker) { + const res = await request(app) + .post(`/api/techniques/${revoked.stix.id}/revoke`) + .send({ revoking: { stixId: revoker.stix.id, modified: revoker.stix.modified } }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return res.body; + } + + it('enrolls the revoked revision as a candidate in tracks where the object is a member', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Revoke M')); + const revoker = await postObject('/api/techniques', buildTechnique('Capture Revoker M')); + const trackId = await createTrack('Capture Revoke Member Track'); + await setMembers(trackId, technique); + + const result = await revokeTechnique(technique, revoker); + + // The revoke response carries the revoked revision's backref + expect(result.primary.stix.revoked).toBe(true); + expect(entryForTrack(result.primary, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + + // The member revision keeps its pin; the revoked revision is a candidate + const memberRevision = await getTechniqueVersion(technique.stix.id, technique.stix.modified); + expect(entryForTrack(memberRevision, trackId)).toEqual({ + id: trackId, + tier: 'members', + status: 'reviewed', + }); + const revokedRevision = await getTechniqueVersion( + technique.stix.id, + result.primary.stix.modified, + ); + expect(entryForTrack(revokedRevision, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + + it('moves a candidate pin to the revoked revision', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Revoke C')); + const revoker = await postObject('/api/techniques', buildTechnique('Capture Revoker C')); + const trackId = await createTrack('Capture Revoke Candidate Track'); + await addCandidate(trackId, technique); + + const result = await revokeTechnique(technique, revoker); + + const oldRevision = await getTechniqueVersion(technique.stix.id, technique.stix.modified); + expect(entryForTrack(oldRevision, trackId)).toBeUndefined(); + expect(entryForTrack(result.primary, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + }); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 4aba92d5..94d55b1a 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -144,11 +144,13 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho - [x] **Reject revision re-keying on PUT.** `updateFull` merged body `stix.id`/`stix.modified` over the stored document, so a PUT could silently re-key a revision and strand any track pins. Now returns 400 when the body identity fields differ from the path parameters. Re-keying must go through POST (a new revision), which member sync captures. Tests: `app/tests/api/base-services/update-identity-guard.spec.js`. -- [ ] **Capture in-place PUTs of pinned revisions.** Decision: reject the PUT (409) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. When pinned only in `staged`/`candidates`, allow the PUT but flag the tier entry (reset/annotate `object_status`; tentative value `modified-in-place`) via a snapshot clone so the change is re-reviewed. This also covers in-place deprecation (`x_mitre_deprecated` set via PUT) — tracks must never be blind to it. Also fix the current member-sync misfire on `::updated` events: an in-place PUT of a member-pinned revision today enrolls a candidate with the *same* `(stix.id, modified)` key as the member entry, creating a duplicate cross-tier reference in the snapshot. Needs: guard in `BaseService.updateFull` (read the document's backrefs), a release-tracks event/handler for flagging an entry, snapshot-schema status addition, workflow/auto-promotion interaction review, tests, docs. +- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the existing `::updated` → member-sync path: under `replace` the entry resets to `work-in-progress` (staged demotes to candidates) so the change is re-reviewed, covering in-place deprecation (`x_mitre_deprecated` via PUT). No new `modified-in-place` status was introduced — reset suffices and avoids rippling a new enum through workflow ordering, candidacy thresholds, and the frontend. The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. -- [ ] **DELETE of tracked objects → deprecation.** Decision: deleting an object revision pinned as `members`/`staged` must not hard-delete (today the backref dies with the document and the track keeps a silent dangling pin). Convert the delete into a new revision with `x_mitre_deprecated: true`, which member sync enrolls as a candidate — aligns with ATT&CK's deprecate-don't-delete release convention (released objects never vanish between releases; the custom deleted-flag + remove-on-merge idea was considered and rejected for that reason). Hard delete remains available for untracked / work-in-progress objects. True removal from a release is a track-side operation (remove the member entry), not an object-side delete. +- [x] **DELETE of tracked objects.** Implemented 2026-07-13 with a simplified decision: DELETE (single version or all versions) is *rejected* (409) when a revision is `members`-pinned, with guidance to retire the object via a new `x_mitre_deprecated` revision instead — members-pinned revisions are immutable and must never be deleted. (The earlier auto-convert-to-deprecation idea was dropped in favor of explicit rejection.) `candidates`/`staged`-pinned deletes remain allowed (the reconciler self-heals the dangling pin). Note: `CollectionsService` overrides `deleteVersionById`, so collections are not covered by the guard. Legacy delete controllers were migrated to the service-exception middleware (`next(err)`) so the 409 maps correctly. -- [ ] **Revoke must reach member sync.** `revoke()` saves the revoked revision via `repository.save` directly, so no `::created`/`::updated` event fires and member sync never enrolls the revoked revision in the tracks where the object is a member — a track can silently keep exporting the pre-revoke revision. Fix: subscribe member sync to the existing per-type `::revoked` events (payload shape differs from created/updated — needs a small adapter in member-sync-service). Scoping is inherent and safe: member sync only enrolls in tracks that already hold the object in `members`, so a revoke can never pull an object into another team's track. Decision: do NOT extend member sync to relationships — bundle export pulls active relationships dynamically and deprecated ones drop out on their own; tracks may still pin relationships manually. +- [x] **Revoke must reach member sync.** Implemented 2026-07-13: member sync subscribes to the 11 per-type `::revoked` events via a payload adapter (`handleStixObjectRevokedEvent`), so the revoked revision (`revoked: true`) is enrolled as a candidate in member tracks and candidate/staged pins move to it — treated exactly like any new revision. The revoke response's primary document carries the resulting backrefs. As decided, member sync is NOT extended to relationships: the revoked-by SRO and deprecation clones are pulled in dynamically at bundle export. + +- [ ] **Technique conversion should reach revision sync.** Same class as the (fixed) revoke gap: `convertToSubtechnique`/`convertToTechnique` save the new revision via `repository.save` directly — no `::created`/`::updated` fires, so a track pinning the converted object keeps pinning the pre-conversion revision with no capture. Fix candidates: emit the created/updated event from the conversion paths, or subscribe member sync to the `TECHNIQUE_CONVERTED_*` events with a payload adapter (same pattern as `handleStixObjectRevokedEvent`). ## Diffing Endpoint @@ -231,7 +233,7 @@ GET /api/compare -## Reimagining Notes +## Repurposing the `note` object - [ ] Implement support for tracking notes on snapshot objects (can be candidates, staged, or members). Notes should be stored in a separate Mongo collection and linked to the snapshot object via a reference field. Users should be able to add, edit, and delete notes via the API. Notably, we already have a notes service that can be leveraged for this purpose. However, it needs some modifications. The service was originally implemented with STIX in mind. The idea was to treat/represent notes as STIX objects and enable users to include them in emitted STIX bundles. However, the concept never really took off. We should modify the service to treat notes as second-class objects that are entirely separate from STIX, but rather as Workbench-native objects. Notes should be capable of being linked/attached to snapshot objects (candidates, staged, or members) as well as to objects independent of snapshots (documents in the `attackObjects` collection). diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index a0a2ced7..dbe9031e 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -84,6 +84,25 @@ Member sync logic is triggered by **object modification events**. Specifically, > follow new revisions for all three tiers; `manual` tracks are unaffected. > Relationships are deliberately excluded from sync — bundle export pulls > active relationships dynamically. +> +> **Behavior evolution (2026-07-13):** three further change-capture rules: +> +> - Sync also fires on the per-type `::revoked` events. The revoke workflow +> saves the revoked revision directly via the repository (no +> `::created`/`::updated` fires), so without this a track silently kept +> exporting the pre-revoke revision. +> - In-place `PUT`s of a pinned revision arrive as `::updated` with an +> unchanged `(stix.id, modified)` key. Under `replace`, the entry resets +> for re-review (staged demotes to candidates); if the outcome would be +> identical (entry already `work-in-progress` in the same tier), the sync +> skips instead of cloning a no-op snapshot. Enrollment is also skipped +> when the exact revision is already pinned in some tier (e.g. a re-import +> announcing an already-released revision) — previously this created a +> duplicate cross-tier reference. +> - `members`-pinned revisions never reach the in-place path at all: +> `BaseService` rejects `PUT`/`DELETE` of a members-pinned revision with +> 409 (`MemberPinnedRevisionError`) — released content is immutable in +> place. ### Relationship to Existing Features diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 1eb2d7f7..1ca81d59 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -66,6 +66,31 @@ An object referenced by multiple tracks carries one entry per track. track to the newly created revision, the response body already carries the resulting `workspace.release_tracks` entry. +## In-place edits, deletes, and revocations + +Release tracks are never blind to changes in the objects they pin: + +- **Members-pinned revisions are immutable in place.** `PUT` and `DELETE` + against a revision that any track pins in its `members` tier return + `409 Conflict` — released content cannot be changed or destroyed under the + track. Make changes by creating a new revision (`POST`); retire an object + by creating a new revision with `x_mitre_deprecated: true`. Revision sync + captures either one. +- **Candidate/staged-pinned revisions can be edited in place, but the track + sees it.** An in-place `PUT` (including one that only sets + `x_mitre_deprecated`) resets the pinned entry for re-review: a reviewed or + awaiting-review candidate drops back to `work-in-progress`, and a staged + entry is demoted back to `candidates` (per the track's member-sync + supplant config; `manual`-strategy tracks opt out). If the edit changes + nothing the track cares about (the entry was already `work-in-progress`), + no new snapshot is created. +- **Revoking a tracked object queues the revoked revision.** The revoke + workflow creates one new revision of the revoked object + (`revoked: true`); revision sync enrolls it as a candidate in tracks where + the object is a member and moves candidate/staged pins to it. The revoking + object and the `revoked-by` relationship are not tracked explicitly — + bundle export pulls secondary objects and their SROs in dynamically. + ## Lifecycle example ``` From ffedbbc2487bb9458a2c2eb9dc93436819ce3c2f Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:00:48 -0400 Subject: [PATCH 12/55] feat(release-tracks): add workflow gate and modified-in-place status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Centralize tier/status placement in a workflow gate (lib/release-tracks/workflow-gate.js): given the trigger (new revision, in-place edit, revocation), the entry's previous state, and the track config, the gate decides where a synced entry lands. The candidacy threshold is codified into placement — qualifying entries go directly to staged in a single snapshot instead of bouncing through candidates via a post-hoc auto-promotion pass, and workflow-service shares the gate's status ranking. In-place PUTs of pinned revisions now mark the entry with the server-assigned modified-in-place status: the content changed but carries no revision history, so reviewers are told that a re-review is required without pretending to know what changed. The marker ranks with work-in-progress, so permissive tracks (candidacy_threshold work-in-progress) keep in-place-edited staged entries staged, while strict tracks demote them to candidates. Cleared via the review endpoint (from: modified-in-place); never carried onto new revisions by the preserve policy. --- .../definitions/components/release-tracks.yml | 6 +- app/api/definitions/components/workspace.yml | 2 +- .../paths/release-tracks-paths.yml | 15 +- app/controllers/release-tracks-controller.js | 4 +- .../release-tracks/release-track-schemas.js | 17 +- app/lib/release-tracks/workflow-gate.js | 127 +++++++++++++++ .../release-track-snapshot-schema.js | 4 +- app/models/subschemas/workspace.js | 6 +- .../release-tracks/member-sync-service.js | 153 +++++++++--------- .../release-tracks/workflow-service.js | 19 +-- .../release-tracks-backrefs.spec.js | 5 +- .../release-tracks-change-capture.spec.js | 72 +++++++-- docs/developer/TODO.md | 2 +- .../release-tracks/member-sync-strategies.md | 27 +++- docs/user/release-tracks/object-backrefs.md | 20 ++- docs/user/release-tracks/release-workflow.md | 4 +- 16 files changed, 342 insertions(+), 141 deletions(-) create mode 100644 app/lib/release-tracks/workflow-gate.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 29ed70d2..254baec9 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -125,10 +125,11 @@ components: object_status: type: string enum: + - modified-in-place - work-in-progress - awaiting-review - reviewed - description: 'Workflow status (scoped to this track)' + description: 'Workflow status (scoped to this track). modified-in-place is server-assigned when the pinned revision is edited via an in-place PUT and needs re-review.' object_added_at: type: string format: date-time @@ -145,10 +146,11 @@ components: object_status: type: string enum: + - modified-in-place - work-in-progress - awaiting-review - reviewed - description: 'Workflow status (preserved from candidates)' + description: 'Workflow status (preserved from candidates). modified-in-place is server-assigned when the pinned revision is edited via an in-place PUT and needs re-review.' object_staged_at: type: string format: date-time diff --git a/app/api/definitions/components/workspace.yml b/app/api/definitions/components/workspace.yml index fd145501..a3c86367 100644 --- a/app/api/definitions/components/workspace.yml +++ b/app/api/definitions/components/workspace.yml @@ -34,7 +34,7 @@ components: description: 'The tier of the release track that references this object revision; values match the snapshot tier array names' status: type: string - enum: ['work-in-progress', 'awaiting-review', 'reviewed'] + enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'] description: 'Track-scoped workflow status. Members are always reviewed; quarantined entries carry no status.' required: - id diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 0999f816..512bcc62 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -244,9 +244,9 @@ paths: in: query description: | Workflow-status filter for the staged/candidate tiers selected via include - (bundle format only). Accepts work-in-progress and/or awaiting-review - (comma-separated or repeated). Entries marked reviewed are always included, - irrespective of this parameter. Members are unaffected. + (bundle format only). Accepts modified-in-place, work-in-progress and/or + awaiting-review (comma-separated or repeated). Entries marked reviewed are + always included, irrespective of this parameter. Members are unaffected. allowReserved: true schema: oneOf: @@ -451,6 +451,7 @@ paths: schema: type: string enum: + - modified-in-place - work-in-progress - awaiting-review - reviewed @@ -494,6 +495,8 @@ paths: description: | Transition candidates from one workflow status to another (forward-only). If auto_promote is enabled and candidates meet the threshold after transition, they are auto-promoted to staged. + `from` also accepts the server-assigned `modified-in-place` status; `to` accepts only the + user-settable statuses (work-in-progress, awaiting-review, reviewed). Request body validated via Zod: { from, to, object_refs? } tags: - 'Release Tracks' @@ -840,9 +843,9 @@ paths: in: query description: | Workflow-status filter for the staged/candidate tiers selected via include - (bundle format only). Accepts work-in-progress and/or awaiting-review - (comma-separated or repeated). Entries marked reviewed are always included, - irrespective of this parameter. Members are unaffected. + (bundle format only). Accepts modified-in-place, work-in-progress and/or + awaiting-review (comma-separated or repeated). Entries marked reviewed are + always included, irrespective of this parameter. Members are unaffected. allowReserved: true schema: oneOf: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 547b99da..db26435c 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -28,7 +28,7 @@ const { stixVersionQuerySchema, booleanQuerySchema, trackTypeQuerySchema, - workflowStatusSchema, + trackEntryStatusSchema, createTrackBodySchema, createFromBundleBodySchema, updateMetadataBodySchema, @@ -600,7 +600,7 @@ exports.addCandidates = async function addCandidates(req, res, next) { exports.listCandidates = async function listCandidates(req, res, next) { try { const options = { - status: parseOptionalQuery(req.query.status, workflowStatusSchema, undefined), + status: parseOptionalQuery(req.query.status, trackEntryStatusSchema, undefined), limit: req.query.limit ? parseInt(req.query.limit, 10) : undefined, offset: req.query.offset ? parseInt(req.query.offset, 10) : undefined, }; diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 6f3daa5c..526c955e 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -180,7 +180,7 @@ const bundleIncludeQuerySchema = z.preprocess( // value — reviewed objects are always included. const bundleStateQuerySchema = z.preprocess( (value) => normalizeQueryArray(value), - z.array(z.enum(['work-in-progress', 'awaiting-review'])).min(1), + z.array(z.enum(['modified-in-place', 'work-in-progress', 'awaiting-review'])).min(1), ); const stixVersionQuerySchema = z.enum(['2.0', '2.1']); @@ -195,6 +195,18 @@ const bumpTypeSchema = z.enum(['major', 'minor']); const workflowStatusSchema = z.enum(['work-in-progress', 'awaiting-review', 'reviewed']); +// Track-entry statuses include the server-assigned 'modified-in-place' +// marker (set by the workflow gate when a pinned revision is edited via an +// in-place PUT). Valid wherever an existing entry's status is read or +// matched (review `from`, status filters) — but not settable as a review +// target, and not a valid candidacy threshold. +const trackEntryStatusSchema = z.enum([ + 'modified-in-place', + 'work-in-progress', + 'awaiting-review', + 'reviewed', +]); + const candidacyThresholdSchema = z.enum(['work-in-progress', 'awaiting-review', 'reviewed']); const deduplicationStrategySchema = z.enum([ @@ -326,7 +338,7 @@ const addCandidatesBodySchema = z.object({ /** POST /release-tracks/:id/candidates/review */ const reviewCandidatesBodySchema = z.object({ - from: workflowStatusSchema, + from: trackEntryStatusSchema, to: workflowStatusSchema, object_refs: z .array( @@ -418,6 +430,7 @@ module.exports = { trackTypeQuerySchema, bumpTypeSchema, workflowStatusSchema, + trackEntryStatusSchema, candidacyThresholdSchema, deduplicationStrategySchema, resolutionStrategySchema, diff --git a/app/lib/release-tracks/workflow-gate.js b/app/lib/release-tracks/workflow-gate.js new file mode 100644 index 00000000..a10eb251 --- /dev/null +++ b/app/lib/release-tracks/workflow-gate.js @@ -0,0 +1,127 @@ +'use strict'; + +// ============================================================================= +// Release Track Workflow Gate +// +// Single decision point for where a tracked object's tier entry belongs +// after a change reaches revision sync. All placement rules — the supplant +// status policy, the modified-in-place marker, and the candidacy-threshold / +// auto-promotion check — are codified here instead of being scattered +// through the versioning code. Given the priors (what triggered the change, +// how the entry enters the tier arrays, the entry's previous state, the +// track configuration), the gate returns the entry's new tier and status. +// ============================================================================= + +// Track-entry workflow statuses. 'modified-in-place' marks entries whose +// pinned revision was changed by an in-place PUT: the content changed, but +// because in-place edits carry no revision history the track cannot say +// *what* changed — only that a re-review is required. +const TRACK_ENTRY_STATUSES = [ + 'modified-in-place', + 'work-in-progress', + 'awaiting-review', + 'reviewed', +]; + +// 'modified-in-place' ranks with 'work-in-progress': both mean "not reviewed +// in its current state". A permissive track (candidacy_threshold +// 'work-in-progress') therefore stages modified-in-place entries too. +const STATUS_RANK = { + 'modified-in-place': 0, + 'work-in-progress': 0, + 'awaiting-review': 1, + reviewed: 2, +}; + +/** + * Check whether a track-entry status meets or exceeds the configured + * candidacy threshold. + * + * @param {string} status - The entry's workflow status + * @param {string} threshold - The configured candidacy threshold + * @returns {boolean} + */ +function meetsCandidacyThreshold(status, threshold) { + const statusRank = STATUS_RANK[status]; + const thresholdRank = STATUS_RANK[threshold]; + + if (statusRank === undefined || thresholdRank === undefined) { + return false; + } + + return statusRank >= thresholdRank; +} + +/** + * Decide the tier and status of a tracked object's entry after a change. + * + * @param {Object} priors + * @param {'new-revision'|'in-place-update'|'revocation'} priors.trigger - + * The operation that produced the change + * @param {'move-pin'|'queue'|'enroll'} priors.mode - How the entry enters + * the tier arrays: move-pin replaces the previous entry, queue adds a + * second entry alongside it, enroll creates the object's first entry + * @param {{tier: string, status: string}|null} priors.previousEntry - The + * entry being replaced (move-pin) or null + * @param {'reset'|'preserve'} priors.statusPolicy - member_sync supplant + * status policy + * @param {string} priors.candidacyThreshold - config.candidacy_threshold + * @param {boolean} priors.autoPromote - config.auto_promote + * @returns {{tier: 'candidates'|'staged', status: string}} + */ +function decidePlacement({ + trigger, + mode, + previousEntry, + statusPolicy, + candidacyThreshold, + autoPromote, +}) { + // --- Status --- + let status; + if (trigger === 'in-place-update') { + // The pinned content itself changed with no revision history to diff — + // mark the entry so reviewers know a re-review is required and why. + status = 'modified-in-place'; + } else if ( + statusPolicy === 'preserve' && + previousEntry?.status && + previousEntry.status !== 'modified-in-place' + ) { + // A new revision replacing a modified-in-place pin is a fresh explicit + // version — never carry the in-place marker onto it. + status = previousEntry.status; + } else { + status = 'work-in-progress'; + } + + // --- Tier --- + let tier; + if (mode === 'queue') { + // Queued entries always start in candidates; they reach staged through + // review / auto-promotion like any other candidate. + tier = 'candidates'; + } else if (autoPromote === true && meetsCandidacyThreshold(status, candidacyThreshold)) { + // Codified auto-promotion: place directly in staged instead of bouncing + // through candidates and a second snapshot. In a permissive track an + // in-place edit of a staged entry therefore keeps its staged tier + // (status still flips to modified-in-place). + tier = 'staged'; + } else if (trigger !== 'in-place-update' && statusPolicy === 'preserve' && previousEntry) { + // Preserve keeps the entry in the tier it already occupied. + tier = previousEntry.tier; + } else { + // Default: (back) to candidates for review. This is how an in-place + // edit demotes a staged entry in a strict track. + tier = 'candidates'; + } + + return { tier, status }; +} + +module.exports = { + TRACK_ENTRY_STATUSES, + STATUS_RANK, + meetsCandidacyThreshold, + decidePlacement, +}; diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 42e261ee..6a123f65 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -35,7 +35,7 @@ const stagedEntryDefinition = { object_modified: { type: Date, required: true }, object_status: { type: String, - enum: ['work-in-progress', 'awaiting-review', 'reviewed'], + enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'], required: true, }, object_staged_at: { type: Date, required: true }, @@ -52,7 +52,7 @@ const candidateEntryDefinition = { object_modified: { type: Date, required: true }, object_status: { type: String, - enum: ['work-in-progress', 'awaiting-review', 'reviewed'], + enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'], required: true, }, object_added_at: { type: Date, required: true }, diff --git a/app/models/subschemas/workspace.js b/app/models/subschemas/workspace.js index 0190e8c0..4aad990f 100644 --- a/app/models/subschemas/workspace.js +++ b/app/models/subschemas/workspace.js @@ -40,10 +40,12 @@ const releaseTrackRef = { required: true, }, // Track-scoped workflow status. Members are inherently 'reviewed'; - // quarantined entries (virtual tracks) carry no status. + // quarantined entries (virtual tracks) carry no status; + // 'modified-in-place' marks entries whose pinned revision was changed by + // an in-place PUT and needs re-review. status: { type: String, - enum: ['work-in-progress', 'awaiting-review', 'reviewed'], + enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'], }, }; const releaseTrackRefSchema = new mongoose.Schema(releaseTrackRef, { _id: false }); diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index 12d302c3..9e301038 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -34,7 +34,7 @@ const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const snapshotService = require('./snapshot-service'); -const workflowService = require('./workflow-service'); +const workflowGate = require('../../lib/release-tracks/workflow-gate'); const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); @@ -57,7 +57,7 @@ const EventConstants = require('../../lib/event-constants'); * @returns {Promise} Array of affected release track snapshots */ exports.handleObjectModified = async function handleObjectModified(event) { - const { objectRef, newModified, modifiedBy } = event; + const { objectRef, newModified, modifiedBy, trigger } = event; // 1. Find all release tracks that reference this object (members, // candidates, or staged) @@ -78,6 +78,7 @@ exports.handleObjectModified = async function handleObjectModified(event) { objectRef, newModified, modifiedBy, + trigger, isMember: trackInfo.isMember, }); if (result) results.push(result); @@ -152,10 +153,11 @@ async function findTracksReferencingObject(objectRef) { * @param {string} event.objectRef - STIX ID of the modified object * @param {Date|string} event.newModified - New modified timestamp * @param {string} [event.modifiedBy] - User who made the modification + * @param {string} [event.trigger] - 'new-revision' | 'in-place-update' | 'revocation' * @returns {Promise} New snapshot if changes made, null otherwise */ async function processMemberSync(trackId, snapshot, event) { - const { objectRef, newModified, modifiedBy, isMember } = event; + const { objectRef, newModified, modifiedBy, isMember, trigger = 'new-revision' } = event; // Get member sync config with defaults const config = getMemberSyncConfig(snapshot); @@ -173,29 +175,34 @@ async function processMemberSync(trackId, snapshot, event) { const existingEntry = existingInStaged || existingInCandidates; const existingTier = existingInStaged ? 'staged' : existingInCandidates ? 'candidates' : null; - // Determine action based on supplant.behavior - let action = null; - if (!existingEntry) { + // Determine how the entry enters the tier arrays + let mode; + if (trigger === 'in-place-update') { + // In-place edits mutate the pinned content itself; supplant behavior + // (which governs how *new revisions* relate to existing pins) does not + // apply — the pinned entry is always re-marked, even under queue. + if (!existingEntry) { + // The edited revision is not pinned by this track (e.g. an unpinned + // older revision of a member object) — nothing the track ships changed. + return null; + } + mode = 'move-pin'; + } else if (!existingEntry) { // No candidate/staged entry. Only members enroll new revisions from // scratch; a non-member object can only be here via a pin that has // since disappeared (snapshot changed between discovery and processing). if (!isMember) return null; - action = { type: 'enroll', tier: 'candidates' }; + mode = 'enroll'; } else { - // Existing entry → apply supplant behavior switch (config.supplant.behavior) { case 'replace': - action = { - type: 'replace', - removeTier: existingTier, - removeEntry: existingEntry, - targetTier: config.supplant.status_policy === 'preserve' ? existingTier : 'candidates', - }; + mode = 'move-pin'; break; case 'queue': - action = { type: 'enroll', tier: 'candidates' }; + mode = 'queue'; break; case 'ignore': + default: logger.debug( `[member-sync] Track ${trackId}: ignoring ${objectRef} (existing entry in ${existingTier})`, ); @@ -203,11 +210,24 @@ async function processMemberSync(trackId, snapshot, event) { } } - if (!action) return null; + // Workflow gate: the single decision point for the entry's tier and + // status given all priors — including the candidacy threshold, so + // auto-promotion is decided here in one step instead of bouncing the + // entry through candidates and a second snapshot. + const placement = workflowGate.decidePlacement({ + trigger, + mode, + previousEntry: existingEntry + ? { tier: existingTier, status: existingEntry.object_status } + : null, + statusPolicy: config.supplant.status_policy, + candidacyThreshold: snapshot.config?.candidacy_threshold || 'reviewed', + autoPromote: snapshot.config?.auto_promote === true, + }); const incomingTime = new Date(newModified).getTime(); - if (action.type === 'enroll') { + if (mode === 'enroll' || mode === 'queue') { // Skip if this exact revision is already pinned in any tier — enrolling // it again would create a duplicate cross-tier reference (e.g. a // re-import announcing an already-released revision). @@ -225,82 +245,57 @@ async function processMemberSync(trackId, snapshot, event) { } } - if (action.type === 'replace') { - // An in-place edit of the pinned revision arrives with the same - // object_modified: the pin key does not change. Act only when the - // outcome differs — a reviewed entry resets for re-review, a staged - // entry demotes back to candidates — otherwise skip instead of cloning - // a no-op snapshot. - const existingTime = new Date(action.removeEntry.object_modified).getTime(); - const currentStatus = action.removeEntry.object_status || 'work-in-progress'; - const resultingStatus = - config.supplant.status_policy === 'preserve' ? currentStatus : 'work-in-progress'; + if (mode === 'move-pin') { + // Skip no-op moves: same pin key, same tier, same status (e.g. a second + // in-place edit of an entry already marked modified-in-place). + const existingTime = new Date(existingEntry.object_modified).getTime(); + const currentStatus = existingEntry.object_status || 'work-in-progress'; if ( existingTime === incomingTime && - action.targetTier === existingTier && - resultingStatus === currentStatus + placement.tier === existingTier && + placement.status === currentStatus ) { logger.debug( - `[member-sync] Track ${trackId}: in-place update of ${objectRef} leaves the ` + - `pinned entry unchanged, skipping`, + `[member-sync] Track ${trackId}: change to ${objectRef} leaves the pinned entry ` + + `unchanged, skipping`, ); return null; } } - // Build the new candidate/staged entry + // Build the new tier entry const now = new Date(); const newEntry = { object_ref: objectRef, object_modified: new Date(newModified), - object_added_at: now, - object_added_by: modifiedBy || 'system', + object_status: placement.status, }; - - // Determine status and tier placement - const targetTier = action.targetTier || action.tier; - - if (action.type === 'replace' && config.supplant.status_policy === 'preserve') { - // Preserve status from old entry - newEntry.object_status = action.removeEntry.object_status; - if (targetTier === 'staged') { - newEntry.object_staged_at = now; - newEntry.object_staged_by = modifiedBy || 'system'; - } + if (placement.tier === 'staged') { + newEntry.object_staged_at = now; + newEntry.object_staged_by = modifiedBy || 'system'; } else { - // Reset status to work-in-progress - newEntry.object_status = 'work-in-progress'; + newEntry.object_added_at = now; + newEntry.object_added_by = modifiedBy || 'system'; } // Build updated tier arrays let newCandidates = [...(snapshot.candidates || [])]; let newStaged = [...(snapshot.staged || [])]; - // Remove old entry if replacing - if (action.type === 'replace') { - if (action.removeTier === 'candidates') { - newCandidates = newCandidates.filter( - (c) => - !( - c.object_ref === objectRef && - new Date(c.object_modified).getTime() === - new Date(action.removeEntry.object_modified).getTime() - ), - ); - } else if (action.removeTier === 'staged') { - newStaged = newStaged.filter( - (s) => - !( - s.object_ref === objectRef && - new Date(s.object_modified).getTime() === - new Date(action.removeEntry.object_modified).getTime() - ), - ); + // Remove the previous entry when moving the pin + if (mode === 'move-pin') { + const removeTime = new Date(existingEntry.object_modified).getTime(); + const keep = (e) => + !(e.object_ref === objectRef && new Date(e.object_modified).getTime() === removeTime); + if (existingTier === 'candidates') { + newCandidates = newCandidates.filter(keep); + } else { + newStaged = newStaged.filter(keep); } } - // Add new entry to target tier - if (targetTier === 'staged') { + // Add the new entry to the tier the gate selected + if (placement.tier === 'staged') { newStaged.push(newEntry); } else { newCandidates.push(newEntry); @@ -312,16 +307,10 @@ async function processMemberSync(trackId, snapshot, event) { staged: newStaged, }); - logger.info(`[member-sync] Track ${trackId}: ${action.type} ${objectRef} → ${targetTier}`); - - // Check if auto-promotion should occur (new entry in candidates that meets threshold) - if (targetTier === 'candidates' && snapshot.config?.auto_promote) { - const promoted = await workflowService.evaluateAutoPromotion(trackId, newSnapshot); - if (promoted) { - logger.info(`[member-sync] Track ${trackId}: auto-promoted ${objectRef} to staged`); - return promoted; - } - } + logger.info( + `[member-sync] Track ${trackId}: ${trigger} (${mode}) ${objectRef} → ` + + `${placement.tier}/${placement.status}`, + ); return newSnapshot; } @@ -405,11 +394,14 @@ const STIX_OBJECT_EVENTS = [ async function handleStixObjectEvent(payload) { const { stixId, document, previousDocument, options } = payload; - // Transform to member sync event format + // Transform to member sync event format. PUT revision identity is + // immutable, so an updated event (previousDocument present) is always an + // in-place edit of the same revision; a created event is a new revision. const event = { objectRef: stixId, newModified: document.stix?.modified, oldModified: previousDocument?.stix?.modified, + trigger: previousDocument ? 'in-place-update' : 'new-revision', // Try to get user from options (create) or from document workflow metadata modifiedBy: options?.userAccountId || document.workspace?.workflow?.created_by_user_account || 'system', @@ -461,6 +453,7 @@ async function handleStixObjectRevokedEvent(payload) { const event = { objectRef: stixId, newModified: revokedDocument?.stix?.modified, + trigger: 'revocation', modifiedBy: options?.userAccountId || revokedDocument?.workspace?.workflow?.created_by_user_account || diff --git a/app/services/release-tracks/workflow-service.js b/app/services/release-tracks/workflow-service.js index 8a28884b..dfe8085c 100644 --- a/app/services/release-tracks/workflow-service.js +++ b/app/services/release-tracks/workflow-service.js @@ -19,34 +19,25 @@ const snapshotService = require('./snapshot-service'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); +const workflowGate = require('../../lib/release-tracks/workflow-gate'); const logger = require('../../lib/logger'); // ============================================================================= // Status ranking and threshold evaluation // ============================================================================= -const STATUS_RANK = { - 'work-in-progress': 0, - 'awaiting-review': 1, - reviewed: 2, -}; - /** * Check if a candidate's status meets or exceeds the configured threshold. + * Ranking is owned by the workflow gate so every placement decision uses + * the same order (including 'modified-in-place', which ranks with + * 'work-in-progress'). * * @param {string} candidateStatus - The candidate's current status * @param {string} threshold - The configured candidacy threshold * @returns {boolean} True if the candidate meets the threshold */ exports.meetsThreshold = function meetsThreshold(candidateStatus, threshold) { - const candidateRank = STATUS_RANK[candidateStatus]; - const thresholdRank = STATUS_RANK[threshold]; - - if (candidateRank === undefined || thresholdRank === undefined) { - return false; - } - - return candidateRank >= thresholdRank; + return workflowGate.meetsCandidacyThreshold(candidateStatus, threshold); }; // ============================================================================= diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 4f205141..92a5ee53 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -598,10 +598,13 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); + // The in-place PUT is captured by revision sync: the entry keeps its + // pin but is marked modified-in-place; the fake client-supplied entry + // is discarded expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, tier: 'candidates', - status: 'work-in-progress', + status: 'modified-in-place', }); expect(trackEntries(res.body)).toHaveLength(1); }); diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index 3ba9914a..6dc8902a 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -184,7 +184,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { }); describe('in-place edits of candidate/staged-pinned revisions', function () { - it('resets a reviewed candidate entry to work-in-progress on in-place PUT', async function () { + it('marks a reviewed candidate entry modified-in-place on in-place PUT', async function () { const technique = await postObject('/api/techniques', buildTechnique('Capture Candidate')); const trackId = await createTrack('Capture Candidate Track'); await addCandidate(trackId, technique); @@ -199,17 +199,26 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { buildUpdateBody(technique, 'Capture Candidate (edited)'), ).expect(200); - // The PUT response reflects the reset (read-your-own-writes) + // The PUT response reflects the marker (read-your-own-writes) expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, tier: 'candidates', - status: 'work-in-progress', + status: 'modified-in-place', }); const { candidates } = await getJson(`/api/release-tracks/${trackId}/candidates`); expect(candidates).toHaveLength(1); - expect(candidates[0].object_status).toBe('work-in-progress'); + expect(candidates[0].object_status).toBe('modified-in-place'); expect(new Date(candidates[0].object_modified).toISOString()).toBe(technique.stix.modified); + + // The marker is reviewable: modified-in-place → awaiting-review + await postObject( + `/api/release-tracks/${trackId}/candidates/review`, + { from: 'modified-in-place', to: 'awaiting-review' }, + 200, + ); + const after = await getJson(`/api/release-tracks/${trackId}/candidates`); + expect(after.candidates[0].object_status).toBe('awaiting-review'); }); it('demotes a staged entry back to candidates on in-place PUT', async function () { @@ -230,7 +239,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, tier: 'candidates', - status: 'work-in-progress', + status: 'modified-in-place', }); const snapshot = await getJson(`/api/release-tracks/${trackId}`); @@ -238,6 +247,40 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(snapshot.candidates).toHaveLength(1); }); + it('keeps a staged entry staged in a permissive track (candidacy threshold codified)', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Permissive')); + const trackId = await createTrack('Capture Permissive Track'); + await request(app) + .put(`/api/release-tracks/${trackId}/config`) + .send({ candidacy_threshold: 'work-in-progress', auto_promote: true }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + // In a permissive track the fresh candidate auto-promotes immediately + await addCandidate(trackId, technique); + let snapshot = await getJson(`/api/release-tracks/${trackId}`); + expect(snapshot.staged).toHaveLength(1); + + // An in-place edit is marked, but the tier is decided by the workflow + // gate: modified-in-place meets the work-in-progress threshold, so the + // entry stays staged instead of being demoted + const res = await putTechnique( + technique, + buildUpdateBody(technique, 'Capture Permissive (edited)'), + ).expect(200); + + expect(entryForTrack(res.body, trackId)).toEqual({ + id: trackId, + tier: 'staged', + status: 'modified-in-place', + }); + snapshot = await getJson(`/api/release-tracks/${trackId}`); + expect(snapshot.staged).toHaveLength(1); + expect(snapshot.staged[0].object_status).toBe('modified-in-place'); + expect(snapshot.candidates).toHaveLength(0); + }); + it('captures in-place deprecation of a reviewed candidate', async function () { const technique = await postObject('/api/techniques', buildTechnique('Capture Deprecate')); const trackId = await createTrack('Capture Deprecate Track'); @@ -253,26 +296,31 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { const res = await putTechnique(technique, update).expect(200); expect(res.body.stix.x_mitre_deprecated).toBe(true); - // The track saw the deprecation: the entry is back to work-in-progress + // The track saw the deprecation: the entry is marked for re-review expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, tier: 'candidates', - status: 'work-in-progress', + status: 'modified-in-place', }); }); - it('does not clone a snapshot when the in-place PUT changes nothing track-visible', async function () { + it('does not clone a snapshot when a repeat in-place PUT changes nothing track-visible', async function () { const technique = await postObject('/api/techniques', buildTechnique('Capture Noop')); const trackId = await createTrack('Capture Noop Track'); await addCandidate(trackId, technique); - const before = await latestSnapshotModified(trackId); - - // Candidate is already work-in-progress — the reset is a no-op, so no - // new snapshot should be created + // First in-place PUT marks the entry modified-in-place (new snapshot) await putTechnique(technique, buildUpdateBody(technique, 'Capture Noop (edited)')).expect( 200, ); + const before = await latestSnapshotModified(trackId); + + // Second in-place PUT: the entry is already modified-in-place in the + // same tier — no new snapshot should be created + await putTechnique( + technique, + buildUpdateBody(technique, 'Capture Noop (edited again)'), + ).expect(200); const after = await latestSnapshotModified(trackId); expect(after).toBe(before); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 94d55b1a..b19c23c4 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -144,7 +144,7 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho - [x] **Reject revision re-keying on PUT.** `updateFull` merged body `stix.id`/`stix.modified` over the stored document, so a PUT could silently re-key a revision and strand any track pins. Now returns 400 when the body identity fields differ from the path parameters. Re-keying must go through POST (a new revision), which member sync captures. Tests: `app/tests/api/base-services/update-identity-guard.spec.js`. -- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the existing `::updated` → member-sync path: under `replace` the entry resets to `work-in-progress` (staged demotes to candidates) so the change is re-reviewed, covering in-place deprecation (`x_mitre_deprecated` via PUT). No new `modified-in-place` status was introduced — reset suffices and avoids rippling a new enum through workflow ordering, candidacy thresholds, and the frontend. The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. +- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the `::updated` → revision-sync path and are marked with the server-assigned **`modified-in-place`** status (content changed with no revision history to diff — reviewers are told *that* something changed, not *what*; the marker is cleared via the review endpoint). Placement is centralized in the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`): tier is decided against `candidacy_threshold`/`auto_promote` (`modified-in-place` ranks with `work-in-progress`), so permissive tracks keep in-place-edited staged entries staged while strict tracks demote them for re-review — and threshold-qualifying placements land directly in `staged` in a single snapshot (no more candidates bounce). Covers in-place deprecation (`x_mitre_deprecated` via PUT). The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Future: an in-document changelog of in-place modifications would let the marker say *what* changed. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. - [x] **DELETE of tracked objects.** Implemented 2026-07-13 with a simplified decision: DELETE (single version or all versions) is *rejected* (409) when a revision is `members`-pinned, with guidance to retire the object via a new `x_mitre_deprecated` revision instead — members-pinned revisions are immutable and must never be deleted. (The earlier auto-convert-to-deprecation idea was dropped in favor of explicit rejection.) `candidates`/`staged`-pinned deletes remain allowed (the reconciler self-heals the dangling pin). Note: `CollectionsService` overrides `deleteVersionById`, so collections are not covered by the guard. Legacy delete controllers were migrated to the service-exception middleware (`next(err)`) so the 409 maps correctly. diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index dbe9031e..4b09dbbc 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -85,20 +85,31 @@ Member sync logic is triggered by **object modification events**. Specifically, > Relationships are deliberately excluded from sync — bundle export pulls > active relationships dynamically. > -> **Behavior evolution (2026-07-13):** three further change-capture rules: +> **Behavior evolution (2026-07-13):** further change-capture rules, all +> placement decisions now centralized in the **workflow gate** +> (`app/lib/release-tracks/workflow-gate.js`): > > - Sync also fires on the per-type `::revoked` events. The revoke workflow > saves the revoked revision directly via the repository (no > `::created`/`::updated` fires), so without this a track silently kept > exporting the pre-revoke revision. > - In-place `PUT`s of a pinned revision arrive as `::updated` with an -> unchanged `(stix.id, modified)` key. Under `replace`, the entry resets -> for re-review (staged demotes to candidates); if the outcome would be -> identical (entry already `work-in-progress` in the same tier), the sync -> skips instead of cloning a no-op snapshot. Enrollment is also skipped -> when the exact revision is already pinned in some tier (e.g. a re-import -> announcing an already-released revision) — previously this created a -> duplicate cross-tier reference. +> unchanged `(stix.id, modified)` key. The entry is marked with the +> server-assigned **`modified-in-place`** status — the content changed, +> but with no revision history to diff the track can only signal that a +> re-review is required. The marker ranks with `work-in-progress` in the +> candidacy-threshold order and is cleared through the normal review +> endpoint (`from: "modified-in-place"`). +> - The gate codifies the candidacy threshold into placement itself: an +> entry whose resulting status meets `candidacy_threshold` (with +> `auto_promote`) is placed directly in `staged` — one snapshot instead of +> bouncing through candidates and a post-hoc auto-promotion pass. In a +> permissive track (threshold `work-in-progress`) an in-place edit of a +> staged entry therefore keeps its staged tier; in a strict track it +> demotes to candidates. +> - Repeat no-op changes (entry already in the gate-decided tier/status) and +> enrollment of already-pinned revisions (e.g. a re-import announcing an +> already-released revision) skip snapshot creation. > - `members`-pinned revisions never reach the in-place path at all: > `BaseService` rejects `PUT`/`DELETE` of a members-pinned revision with > 409 (`MemberPinnedRevisionError`) — released content is immutable in diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 1ca81d59..29511a1d 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -27,7 +27,7 @@ scanning tracks. |-------|--------|---------| | `id` | `release-track--` | The referencing release track | | `tier` | `members`, `staged`, `candidates`, `quarantine` | Which tier of the track references this revision; values match the snapshot tier array names | -| `status` | `work-in-progress`, `awaiting-review`, `reviewed` | Track-scoped workflow status | +| `status` | `modified-in-place`, `work-in-progress`, `awaiting-review`, `reviewed` | Track-scoped workflow status (`modified-in-place` is server-assigned when the pinned revision is edited via an in-place PUT) | An object referenced by multiple tracks carries one entry per track. @@ -78,12 +78,18 @@ Release tracks are never blind to changes in the objects they pin: captures either one. - **Candidate/staged-pinned revisions can be edited in place, but the track sees it.** An in-place `PUT` (including one that only sets - `x_mitre_deprecated`) resets the pinned entry for re-review: a reviewed or - awaiting-review candidate drops back to `work-in-progress`, and a staged - entry is demoted back to `candidates` (per the track's member-sync - supplant config; `manual`-strategy tracks opt out). If the edit changes - nothing the track cares about (the entry was already `work-in-progress`), - no new snapshot is created. + `x_mitre_deprecated`) marks the pinned entry `modified-in-place`: the + content changed, but because in-place edits carry no revision history the + track cannot say *what* changed — only that a re-review is required. The + entry's tier is decided by the workflow gate against the track's candidacy + threshold: in a strict track (threshold `reviewed`, the default) a staged + entry demotes back to `candidates`; in a permissive track (threshold + `work-in-progress` with `auto_promote`) the entry stays staged, since + `modified-in-place` ranks with `work-in-progress`. `manual`-strategy + tracks opt out entirely. Repeat edits of an entry already marked + `modified-in-place` do not create additional snapshots. Reviewers clear + the marker through the normal review endpoint + (`from: "modified-in-place"`). - **Revoking a tracked object queues the revoked revision.** The revoke workflow creates one new revision of the revoked object (`revoked: true`); revision sync enrolls it as a candidate in tracks where diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 0e5fc952..3855076a 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -80,7 +80,9 @@ Release tracks can be configured with different thresholds for what workflow sta Typical release tracks will use the default candidacy threshold setting of `reviewed`, which requires that the object(s) status be `reviewed` in order for the object to become staged. -However, smaller teams operating in purely developmenet or research capacities may prefer a more permissive model. Perhaps they simply want all objects to be included in the release irrespective of object status. In such situations, the candidacy threshold can be lowered to `awaiting-review` or `work-in-progress`. +However, smaller teams operating in purely development or research capacities may prefer a more permissive model. Perhaps they simply want all objects to be included in the release irrespective of object status. In such situations, the candidacy threshold can be lowered to `awaiting-review` or `work-in-progress`. + +The threshold is enforced by the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`), the single decision point that places tracked objects into tiers whenever revision sync reacts to a change (new revision, in-place edit, revocation). The server-assigned `modified-in-place` status ranks with `work-in-progress` in the threshold order — so in a permissive track, an in-place edit of a staged object keeps it staged (marked for re-review), while in a strict track it demotes back to candidates. ### Option 1: Include Only Reviewed (Default) ```javascript From 6ea260660f0c4a38435aec991aeee8ddca76d40a Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:11:39 -0400 Subject: [PATCH 13/55] fix(release-tracks): capture technique conversions in revision sync Technique/subtechnique conversions save the new revision directly via the repository, so no created/updated event fired and a track pinning the converted object kept pinning the pre-conversion revision. The conversion events now carry the converted revision and acting user, and member sync subscribes via an adapter (same pattern as the revoked events), treating the conversion as a new-revision trigger through the workflow gate: candidate/staged pins move to the converted revision and member tracks enroll it as a candidate. Conversion responses refresh workspace.release_tracks after event processing so the result carries the re-pinned backrefs. --- .../release-tracks/member-sync-service.js | 60 ++++++++++++++- app/services/stix/techniques-service.js | 19 ++++- .../release-tracks-backrefs.spec.js | 16 ++-- .../release-tracks-change-capture.spec.js | 74 +++++++++++++++++++ docs/developer/TODO.md | 2 +- .../release-tracks/member-sync-strategies.md | 10 ++- 6 files changed, 166 insertions(+), 15 deletions(-) diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index 9e301038..a5c9800b 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -467,11 +467,56 @@ async function handleStixObjectRevokedEvent(payload) { } } +/** + * Technique/subtechnique conversion events. Conversions save the new + * revision directly via the repository (no ::created/::updated fires), so + * without this subscription a track pinning the converted object would keep + * pinning the pre-conversion revision with no capture. + */ +const STIX_OBJECT_CONVERTED_EVENTS = [ + EventConstants.TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE, + EventConstants.SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE, +]; + +/** + * Handle a technique/subtechnique conversion event. + * + * The conversion produces a new revision (payload.document) — treat it like + * any other new revision: enroll it in member tracks, move candidate/staged + * pins per the supplant config. + * + * @param {Object} payload - Event payload from TechniquesService + * @param {string} payload.stixId - The STIX ID of the converted object + * @param {Object} payload.document - The new converted revision + * @param {string} [payload.userAccountId] - The acting user + */ +async function handleStixObjectConvertedEvent(payload) { + const { stixId, document, userAccountId } = payload; + + if (!document?.stix?.modified) { + logger.warn(`[member-sync] Conversion event for ${stixId} carried no document, skipping`); + return; + } + + const event = { + objectRef: stixId, + newModified: document.stix.modified, + trigger: 'new-revision', + modifiedBy: userAccountId || document.workspace?.workflow?.created_by_user_account || 'system', + }; + + try { + await exports.handleObjectModified(event); + } catch (err) { + logger.error(`[member-sync] Error handling object conversion: ${err.message}`, err); + } +} + /** * Initialize event listeners for member sync. * - * Subscribes to all STIX object created/updated/revoked events via the - * EventBus. Called automatically when this module is loaded. + * Subscribes to all STIX object created/updated/revoked/converted events via + * the EventBus. Called automatically when this module is loaded. */ function initializeEventListeners() { for (const eventName of STIX_OBJECT_EVENTS) { @@ -480,10 +525,17 @@ function initializeEventListeners() { for (const eventName of STIX_OBJECT_REVOKED_EVENTS) { EventBus.on(eventName, handleStixObjectRevokedEvent); } + for (const eventName of STIX_OBJECT_CONVERTED_EVENTS) { + EventBus.on(eventName, handleStixObjectConvertedEvent); + } logger.info( `[member-sync] Member sync service initialized, listening to ` + - `${STIX_OBJECT_EVENTS.length + STIX_OBJECT_REVOKED_EVENTS.length} event types`, + `${ + STIX_OBJECT_EVENTS.length + + STIX_OBJECT_REVOKED_EVENTS.length + + STIX_OBJECT_CONVERTED_EVENTS.length + } event types`, ); } @@ -501,6 +553,8 @@ exports._internal = { getMemberSyncConfig, handleStixObjectEvent, handleStixObjectRevokedEvent, + handleStixObjectConvertedEvent, STIX_OBJECT_EVENTS, STIX_OBJECT_REVOKED_EVENTS, + STIX_OBJECT_CONVERTED_EVENTS, }; diff --git a/app/services/stix/techniques-service.js b/app/services/stix/techniques-service.js index 2c23cfad..ccc8f9c2 100644 --- a/app/services/stix/techniques-service.js +++ b/app/services/stix/techniques-service.js @@ -360,14 +360,21 @@ class TechniquesService extends BaseService { const result = new WorkflowResult('convert-to-subtechnique'); result.setPrimary(savedDocument); - // Emit domain event — RelationshipsService listens to create the subtechnique-of SRO + // Emit domain event — RelationshipsService listens to create the + // subtechnique-of SRO; member sync re-pins/enrolls the converted revision + // in referencing release tracks const eventResults = await EventBus.emit(EventConstants.TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE, { stixId /** STIX ID of the converted subtechnique */, parentStixId: parentTechnique.stix.id /** STIX ID of the parent technique */, + document: savedDocument.toObject ? savedDocument.toObject() : savedDocument, userAccountId: options.userAccountId, }); result.mergeEventResults(eventResults); + // Revision sync may have re-pinned a track to the converted revision — + // refresh so the response carries the resulting backrefs + await this._refreshReleaseTrackBackrefs(savedDocument); + return result.toJSON(); } @@ -445,12 +452,20 @@ class TechniquesService extends BaseService { const result = new WorkflowResult('convert-to-technique'); result.setPrimary(savedDocument); - // Emit domain event — RelationshipsService listens to deprecate subtechnique-of SROs + // Emit domain event — RelationshipsService listens to deprecate + // subtechnique-of SROs; member sync re-pins/enrolls the converted + // revision in referencing release tracks const eventResults = await EventBus.emit(EventConstants.SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE, { stixId /** STIX ID of the converted subtechnique */, + document: savedDocument.toObject ? savedDocument.toObject() : savedDocument, + userAccountId: options.userAccountId, }); result.mergeEventResults(eventResults); + // Revision sync may have re-pinned a track to the converted revision — + // refresh so the response carries the resulting backrefs + await this._refreshReleaseTrackBackrefs(savedDocument); + return result.toJSON(); } diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 92a5ee53..dd260e43 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -694,13 +694,19 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); - // The converted revision is a new version — no inherited backrefs + // The converted revision carries a backref only via revision sync (the + // candidate pin moved to it) — never via clone-copying: the entry is + // the re-pinned candidate, not a fake copied entry expect(res.body.primary.stix.x_mitre_is_subtechnique).toBe(true); - expect(res.body.primary.workspace.release_tracks).toBeUndefined(); + expect(entryForTrack(res.body.primary, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); - // The pinned revision keeps its backref - const pinned = await getTechniqueVersion(technique); - expect(entryForTrack(pinned, trackId)).toMatchObject({ tier: 'candidates' }); + // The pre-conversion revision no longer carries the entry + const oldRevision = await getTechniqueVersion(technique); + expect(entryForTrack(oldRevision, trackId)).toBeUndefined(); }); }); diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index 6dc8902a..a74b38ed 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -401,6 +401,80 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { }); }); + describe('technique conversion reaches the release track', function () { + async function convert(stixId, path, body) { + const res = await request(app) + .post(`/api/techniques/${stixId}/${path}`) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return res.body; + } + + it('moves a candidate pin to the converted revision (convert-to-subtechnique)', async function () { + const parent = await postObject('/api/techniques', buildTechnique('Capture Conv Parent')); + const technique = await postObject('/api/techniques', buildTechnique('Capture Conv Child')); + const trackId = await createTrack('Capture Conv Candidate Track'); + await addCandidate(trackId, technique); + + const result = await convert(technique.stix.id, 'convert-to-subtechnique', { + parentTechniqueAttackId: parent.workspace.attack_id, + }); + + // The conversion response carries the re-pinned backref + expect(result.primary.stix.x_mitre_is_subtechnique).toBe(true); + expect(entryForTrack(result.primary, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + + // The pin moved to the converted revision + const oldRevision = await getTechniqueVersion(technique.stix.id, technique.stix.modified); + expect(entryForTrack(oldRevision, trackId)).toBeUndefined(); + const { candidates } = await getJson(`/api/release-tracks/${trackId}/candidates`); + expect(candidates).toHaveLength(1); + expect(new Date(candidates[0].object_modified).toISOString()).toBe( + result.primary.stix.modified, + ); + }); + + it('enrolls the converted revision as a candidate in member tracks (convert-to-technique)', async function () { + const parent = await postObject('/api/techniques', buildTechnique('Capture Conv2 Parent')); + const technique = await postObject('/api/techniques', buildTechnique('Capture Conv2 Child')); + + // Make it a subtechnique first (untracked at this point — no sync) + const subtechniqueResult = await convert(technique.stix.id, 'convert-to-subtechnique', { + parentTechniqueAttackId: parent.workspace.attack_id, + }); + const subtechniqueRevision = subtechniqueResult.primary; + + const trackId = await createTrack('Capture Conv Member Track'); + await setMembers(trackId, subtechniqueRevision); + + const result = await convert(technique.stix.id, 'convert-to-technique', {}); + + // The converted revision is enrolled as a candidate; the member pin + // stays on the pre-conversion revision + expect(result.primary.stix.x_mitre_is_subtechnique).toBe(false); + expect(entryForTrack(result.primary, trackId)).toEqual({ + id: trackId, + tier: 'candidates', + status: 'work-in-progress', + }); + const memberRevision = await getTechniqueVersion( + subtechniqueRevision.stix.id, + subtechniqueRevision.stix.modified, + ); + expect(entryForTrack(memberRevision, trackId)).toEqual({ + id: trackId, + tier: 'members', + status: 'reviewed', + }); + }); + }); + after(async function () { await database.closeConnection(); }); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index b19c23c4..f9986708 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -150,7 +150,7 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho - [x] **Revoke must reach member sync.** Implemented 2026-07-13: member sync subscribes to the 11 per-type `::revoked` events via a payload adapter (`handleStixObjectRevokedEvent`), so the revoked revision (`revoked: true`) is enrolled as a candidate in member tracks and candidate/staged pins move to it — treated exactly like any new revision. The revoke response's primary document carries the resulting backrefs. As decided, member sync is NOT extended to relationships: the revoked-by SRO and deprecation clones are pulled in dynamically at bundle export. -- [ ] **Technique conversion should reach revision sync.** Same class as the (fixed) revoke gap: `convertToSubtechnique`/`convertToTechnique` save the new revision via `repository.save` directly — no `::created`/`::updated` fires, so a track pinning the converted object keeps pinning the pre-conversion revision with no capture. Fix candidates: emit the created/updated event from the conversion paths, or subscribe member sync to the `TECHNIQUE_CONVERTED_*` events with a payload adapter (same pattern as `handleStixObjectRevokedEvent`). +- [x] **Technique conversion should reach revision sync.** Implemented 2026-07-13 with the adapter approach (same pattern as `handleStixObjectRevokedEvent`): the `TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE` / `SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE` event payloads now carry the converted revision (`document`) and acting user, and member sync subscribes via `handleStixObjectConvertedEvent`, treating the conversion as a `new-revision` trigger through the workflow gate — candidate/staged pins move to the converted revision, member tracks enroll it as a candidate. The conversion responses refresh `workspace.release_tracks` after event processing (read-your-own-writes). Tests: conversion cases in `release-tracks-change-capture.spec.js` and the updated clone-strip test in `release-tracks-backrefs.spec.js`. ## Diffing Endpoint diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index 4b09dbbc..d6e78427 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -89,10 +89,12 @@ Member sync logic is triggered by **object modification events**. Specifically, > placement decisions now centralized in the **workflow gate** > (`app/lib/release-tracks/workflow-gate.js`): > -> - Sync also fires on the per-type `::revoked` events. The revoke workflow -> saves the revoked revision directly via the repository (no -> `::created`/`::updated` fires), so without this a track silently kept -> exporting the pre-revoke revision. +> - Sync also fires on the per-type `::revoked` events and on the +> technique/subtechnique conversion events +> (`attack-pattern::converted-to-subtechnique` / `::converted-to-technique`). +> Both workflows save the new revision directly via the repository (no +> `::created`/`::updated` fires), so without these subscriptions a track +> silently kept exporting the pre-revoke / pre-conversion revision. > - In-place `PUT`s of a pinned revision arrive as `::updated` with an > unchanged `(stix.id, modified)` key. The entry is marked with the > server-assigned **`modified-in-place`** status — the content changed, From 446b0170ef67081be7dc79d92b8d703b42699049 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:16:20 -0400 Subject: [PATCH 14/55] feat(release-tracks): add track type to workspace.release_tracks backrefs Backref entries now carry the referencing track's type (standard or virtual), so consumers can distinguish a virtual release's reference from a standard track's without fetching the track. Entries written before the field existed are backfilled on the track's next contents change. Adds the first regression coverage of virtual-track backrefs (composition over a tagged component track). --- app/api/definitions/components/workspace.yml | 4 ++ app/lib/release-tracks/backref-reconciler.js | 23 +++++-- app/models/subschemas/workspace.js | 7 +++ .../release-tracks-backrefs.spec.js | 61 +++++++++++++++++++ .../release-tracks-change-capture.spec.js | 12 ++++ docs/developer/TODO.md | 4 ++ .../release-tracks/backref-reconciliation.md | 4 +- docs/developer/release-tracks/entities.md | 4 +- docs/user/release-tracks/object-backrefs.md | 4 ++ 9 files changed, 117 insertions(+), 6 deletions(-) diff --git a/app/api/definitions/components/workspace.yml b/app/api/definitions/components/workspace.yml index a3c86367..cdf8cbb2 100644 --- a/app/api/definitions/components/workspace.yml +++ b/app/api/definitions/components/workspace.yml @@ -28,6 +28,10 @@ components: id: type: string description: 'The release track ID (release-track--)' + type: + type: string + enum: ['standard', 'virtual'] + description: 'The type of the referencing release track' tier: type: string enum: ['members', 'staged', 'candidates', 'quarantine'] diff --git a/app/lib/release-tracks/backref-reconciler.js b/app/lib/release-tracks/backref-reconciler.js index b6f9b49c..44995d6f 100644 --- a/app/lib/release-tracks/backref-reconciler.js +++ b/app/lib/release-tracks/backref-reconciler.js @@ -9,8 +9,9 @@ // // { // id: 'release-track--', +// type: 'standard'|'virtual', // tier: 'members'|'staged'|'candidates'|'quarantine', -// status: 'work-in-progress'|'awaiting-review'|'reviewed' +// status: 'modified-in-place'|'work-in-progress'|'awaiting-review'|'reviewed' // } // // Backrefs are pinned to specific object revisions: the entry lives on the @@ -125,11 +126,21 @@ async function reconcile(repository, trackId, snapshot, includeRef) { satisfied.add(key); const existing = (document.workspace.release_tracks || []).find((e) => e.id === trackId); - if (existing && existing.tier === want.tier && (existing.status || undefined) === want.status) { + if ( + existing && + existing.tier === want.tier && + (existing.status || undefined) === want.status && + existing.type === snapshot.type + ) { continue; // already correct } - const update = { $set: { 'workspace.release_tracks.$.tier': want.tier } }; + const update = { + $set: { + 'workspace.release_tracks.$.tier': want.tier, + 'workspace.release_tracks.$.type': snapshot.type, + }, + }; if (want.status === undefined) { update.$unset = { 'workspace.release_tracks.$.status': '' }; } else { @@ -166,7 +177,11 @@ async function reconcile(repository, trackId, snapshot, includeRef) { continue; } - const entry = { id: trackId, tier: want.tier }; + const entry = { + id: trackId, + type: snapshot.type, + tier: want.tier, + }; if (want.status !== undefined) { entry.status = want.status; } diff --git a/app/models/subschemas/workspace.js b/app/models/subschemas/workspace.js index 4aad990f..833bab58 100644 --- a/app/models/subschemas/workspace.js +++ b/app/models/subschemas/workspace.js @@ -32,6 +32,13 @@ const validationIssueSchema = new mongoose.Schema(validationIssue, { _id: false const releaseTrackRef = { id: { type: String, required: true }, + // The type of the referencing release track. Optional in the schema to + // tolerate entries written before the field existed (the reconciler + // backfills on the track's next contents change) but always set on write. + type: { + type: String, + enum: ['standard', 'virtual'], + }, // Which tier of the track references this revision; values match the // snapshot tier array names. tier: { diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index dd260e43..349d7e47 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -112,6 +112,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -127,6 +128,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'awaiting-review', }); @@ -142,6 +144,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'staged', status: 'awaiting-review', }); @@ -157,6 +160,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'awaiting-review', }); @@ -173,6 +177,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'members', status: 'reviewed', }); @@ -189,6 +194,49 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { }); }); + describe('track type on backrefs', function () { + it('marks entries from virtual tracks with type virtual', async function () { + // Build a standard component track with one tagged member + const technique = await postObject('/api/techniques', buildTechnique('Backref Virtual')); + const componentTrackId = await createTrack('Backref Virtual Component Track'); + await addCandidates(componentTrackId, [technique]); + await postObject( + `/api/release-tracks/${componentTrackId}/candidates/promote`, + { object_refs: [technique.stix.id] }, + 200, + ); + await postObject(`/api/release-tracks/${componentTrackId}/bump`, { type: 'minor' }, 200); + + // Compose a virtual track over it and create a snapshot + const virtual = await postObject('/api/release-tracks/new', { + name: 'Backref Virtual Track', + type: 'virtual', + }); + await request(app) + .put(`/api/release-tracks/${virtual.id}/composition`) + .send({ + component_tracks: [ + { track_id: componentTrackId, resolution_strategy: 'latest_tagged', priority: 0 }, + ], + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + await postObject(`/api/release-tracks/${virtual.id}/snapshots/create`, {}, 201); + + // The object now carries one entry per referencing track, with types + const retrieved = await getTechniqueVersion(technique); + expect(entryForTrack(retrieved, componentTrackId)).toMatchObject({ + type: 'standard', + tier: 'members', + }); + expect(entryForTrack(retrieved, virtual.id)).toMatchObject({ + type: 'virtual', + tier: 'members', + }); + }); + }); + describe('candidate removal and version pins', function () { it('removing a candidate removes the backref', async function () { const technique = await postObject('/api/techniques', buildTechnique('Backref Removal')); @@ -230,6 +278,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { expect(entryForTrack(retrievedA, trackId)).toBeUndefined(); expect(entryForTrack(retrievedB, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -252,6 +301,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { let retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'members', status: 'reviewed', }); @@ -325,11 +375,13 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const retrievedB = await getTechniqueVersion(revisionB); expect(entryForTrack(retrievedA, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'members', status: 'reviewed', }); expect(entryForTrack(retrievedB, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -355,6 +407,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { // that re-pin the track are awaited before the response is composed expect(entryForTrack(revisionB, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -364,6 +417,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { expect(entryForTrack(retrievedA, trackId)).toBeUndefined(); expect(entryForTrack(retrievedB, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -392,6 +446,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { expect(entryForTrack(retrievedA, trackId)).toBeUndefined(); expect(entryForTrack(retrievedB, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -420,6 +475,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const retrievedB = await getTechniqueVersion(revisionB); expect(entryForTrack(retrievedA, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -464,6 +520,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { ); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -526,6 +583,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { expect(entryForTrack(await getTechniqueVersion(revisionA), trackId)).toBeUndefined(); expect(entryForTrack(await getTechniqueVersion(revisionB), trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -603,6 +661,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { // is discarded expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'modified-in-place', }); @@ -656,6 +715,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { expect(res.body.primary.stix.revoked).toBe(true); expect(entryForTrack(res.body.primary, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -700,6 +760,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { expect(res.body.primary.stix.x_mitre_is_subtechnique).toBe(true); expect(entryForTrack(res.body.primary, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index a74b38ed..292af399 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -147,6 +147,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(retrieved.stix.name).toBe('Capture Member'); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'members', status: 'reviewed', }); @@ -202,6 +203,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { // The PUT response reflects the marker (read-your-own-writes) expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'modified-in-place', }); @@ -238,6 +240,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'modified-in-place', }); @@ -272,6 +275,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'staged', status: 'modified-in-place', }); @@ -299,6 +303,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { // The track saw the deprecation: the entry is marked for re-review expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'modified-in-place', }); @@ -361,6 +366,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(result.primary.stix.revoked).toBe(true); expect(entryForTrack(result.primary, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -369,6 +375,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { const memberRevision = await getTechniqueVersion(technique.stix.id, technique.stix.modified); expect(entryForTrack(memberRevision, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'members', status: 'reviewed', }); @@ -378,6 +385,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { ); expect(entryForTrack(revokedRevision, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -395,6 +403,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(entryForTrack(oldRevision, trackId)).toBeUndefined(); expect(entryForTrack(result.primary, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -426,6 +435,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(result.primary.stix.x_mitre_is_subtechnique).toBe(true); expect(entryForTrack(result.primary, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -460,6 +470,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { expect(result.primary.stix.x_mitre_is_subtechnique).toBe(false); expect(entryForTrack(result.primary, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'candidates', status: 'work-in-progress', }); @@ -469,6 +480,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { ); expect(entryForTrack(memberRevision, trackId)).toEqual({ id: trackId, + type: 'standard', tier: 'members', status: 'reviewed', }); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index f9986708..8ce78711 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -152,6 +152,10 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho - [x] **Technique conversion should reach revision sync.** Implemented 2026-07-13 with the adapter approach (same pattern as `handleStixObjectRevokedEvent`): the `TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE` / `SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE` event payloads now carry the converted revision (`document`) and acting user, and member sync subscribes via `handleStixObjectConvertedEvent`, treating the conversion as a `new-revision` trigger through the workflow gate — candidate/staged pins move to the converted revision, member tracks enroll it as a candidate. The conversion responses refresh `workspace.release_tracks` after event processing (read-your-own-writes). Tests: conversion cases in `release-tracks-change-capture.spec.js` and the updated clone-strip test in `release-tracks-backrefs.spec.js`. +## Small Fixes + +- [ ] **Composition schema mismatch: `priority`.** `PUT /api/release-tracks/:id/composition` — the Zod schema (`componentTrackSchema`) marks `priority` optional, but the mongoose snapshot schema requires it, so omitting it passes validation and then fails the save with a 500 (`DatabaseError`) instead of a 400. Align the schemas (either default `priority` or make it required in Zod). Found 2026-07-15 while testing virtual-track backrefs. + ## Diffing Endpoint - [ ] Implement object diffing endpoints for snapshots. Users should be able to effectively preview changes to objects before tier transitions (candidates, staged, members). diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index d553eaff..d8ece526 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -79,7 +79,9 @@ For one `(repository, trackId, snapshot, includeRef)`: supported by a sparse multikey index on both collections. 3. **Diff → bulkWrite** (batched, unordered): - current but not desired → `$pull` the track's entry; - - both, but phase/status differ → positional `$set`/`$unset`; + - both, but tier/status/type differ → positional `$set`/`$unset` (the + `type` comparison also backfills entries written before the field + existed); - desired but not current → resolve the pinned revision to its `_id` (batched `$or` on the `stix.id + stix.modified` index) and `$push` the entry. Pins whose revision document doesn't exist (dangling pin, or a diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index f90895bb..fff65036 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -206,11 +206,13 @@ field semantics): release_tracks: [ { id: "release-track--123", + type: "standard", // "standard" | "virtual" tier: "members", // "members" | "staged" | "candidates" | "quarantine" - status: "reviewed" // "work-in-progress" | "awaiting-review" | "reviewed" + status: "reviewed" // "modified-in-place" | "work-in-progress" | "awaiting-review" | "reviewed" }, { id: "release-track--456", + type: "standard", tier: "candidates", status: "work-in-progress" } diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 29511a1d..aeb10d8d 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -14,6 +14,7 @@ scanning tracks. "release_tracks": [ { "id": "release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d", + "type": "standard", "tier": "candidates", "status": "work-in-progress" } @@ -26,6 +27,7 @@ scanning tracks. | Field | Values | Meaning | |-------|--------|---------| | `id` | `release-track--` | The referencing release track | +| `type` | `standard`, `virtual` | The type of the referencing release track | | `tier` | `members`, `staged`, `candidates`, `quarantine` | Which tier of the track references this revision; values match the snapshot tier array names | | `status` | `modified-in-place`, `work-in-progress`, `awaiting-review`, `reviewed` | Track-scoped workflow status (`modified-in-place` is server-assigned when the pinned revision is edited via an in-place PUT) | @@ -53,6 +55,8 @@ An object referenced by multiple tracks carries one entry per track. - **Reflects the latest snapshot.** Backrefs mirror the track's *current* (most recent) snapshot. Deleting the latest snapshot reverts backrefs to the previous snapshot's membership; deleting a track removes all of its entries. + Entries written before the `type` field existed are backfilled + automatically on the track's next contents change. - **Status mapping.** Candidates and staged entries carry their track-scoped workflow status. Members are always `reviewed` (promotion to member implies review). Quarantined entries (virtual tracks) have no workflow status, so From 6dc7cea87b12b81db6441b777102a13e09992585 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:36:52 -0400 Subject: [PATCH 15/55] feat(release-tracks): add releases-by-object lookup Maintain tagged snapshot references in the release-track registry and query historical membership across per-track collections with bounded concurrency. Backfill existing tracks, protect tagged snapshots, and document and test the new endpoint. --- .../definitions/components/release-tracks.yml | 51 ++++ app/api/definitions/openapi.yml | 3 + .../paths/release-tracks-paths.yml | 77 ++++- app/controllers/release-tracks-controller.js | 33 ++ app/exceptions/index.js | 7 + app/lib/error-handler.js | 2 + .../release-tracks/release-track-schemas.js | 9 + .../release-track-registry-model.js | 13 + .../release-track-snapshot-schema.js | 11 + .../release-track-dynamic.repository.js | 43 +++ .../release-track-registry.repository.js | 39 +++ app/routes/release-tracks-routes.js | 8 + .../release-tracks/release-history-service.js | 133 +++++++++ .../release-tracks/release-tracks-service.js | 5 + .../release-tracks/snapshot-service.js | 20 +- .../release-tracks/versioning-service.js | 24 +- .../release-tracks/releases-by-object.spec.js | 281 ++++++++++++++++++ docs/README.md | 2 + docs/developer/TODO.md | 226 +++++++++++++- docs/developer/release-tracks/entities.md | 62 +++- .../release-tracks/error-handling.md | 16 +- .../release-tracks/releases-by-object.md | 108 +++++++ docs/user/release-tracks/api-reference.md | 160 +++++++--- .../user/release-tracks/releases-by-object.md | 67 +++++ ...-backfill-release-track-tagged-releases.js | 130 ++++++++ 25 files changed, 1471 insertions(+), 59 deletions(-) create mode 100644 app/services/release-tracks/release-history-service.js create mode 100644 app/tests/api/release-tracks/releases-by-object.spec.js create mode 100644 docs/developer/release-tracks/releases-by-object.md create mode 100644 docs/user/release-tracks/releases-by-object.md create mode 100644 migrations/20260716000000-backfill-release-track-tagged-releases.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 254baec9..4850f618 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -339,6 +339,11 @@ components: tagged_release_count: type: number description: 'Number of tagged releases' + tagged_releases: + type: array + description: 'Compact references to every tagged snapshot in this track' + items: + $ref: '#/components/schemas/tagged-release-reference' summary: type: object description: 'Counts of objects in each release track tier for the latest snapshot' @@ -365,6 +370,52 @@ components: description: 'Automated snapshot schedule (virtual tracks only)' $ref: '#/components/schemas/snapshot-schedule' + tagged-release-reference: + type: object + description: 'Registry reference to a tagged snapshot' + properties: + snapshot_modified: + type: string + format: date-time + description: 'The tagged snapshot modified timestamp' + version: + type: string + description: 'The tagged MAJOR.MINOR version' + tagged_at: + type: string + format: date-time + description: 'When the snapshot was tagged' + tagged_by: + type: string + description: 'User account that tagged the snapshot' + + release-by-object-entry: + type: object + description: 'One tagged release that directly contains the requested STIX object' + properties: + track_id: + type: string + track_type: + type: string + enum: + - standard + - virtual + track_name: + type: string + version: + type: string + snapshot_modified: + type: string + format: date-time + tagged_at: + type: string + format: date-time + tagged_by: + type: string + object_modified: + type: string + format: date-time + snapshot-schedule: type: object description: 'Schedule for automated virtual track snapshot creation' diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 4b1b041b..722384b8 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -334,6 +334,9 @@ paths: /api/release-tracks: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks' + /api/release-tracks/objects/{objectRef}/releases: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1objects~1{objectRef}~1releases' + /api/release-tracks/new: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1new' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 512bcc62..39d5bb5c 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -142,6 +142,81 @@ paths: offset: type: number + /api/release-tracks/objects/{objectRef}/releases: + get: + summary: 'List tagged releases containing a STIX object' + operationId: 'release-tracks-releases-by-object' + description: | + Return every tagged release-track snapshot whose members tier directly + contains the supplied STIX ID. The result spans all object revisions. + Drafts, non-member tiers, and secondary bundle-export objects are excluded. + tags: + - 'Release Tracks' + parameters: + - name: objectRef + in: path + required: true + description: 'STIX ID to locate across tagged releases' + schema: + type: string + - name: type + in: query + description: 'Restrict results to one release-track type' + schema: + type: string + enum: + - standard + - virtual + - name: order + in: query + description: 'Snapshot chronology order' + schema: + type: string + enum: + - asc + - desc + default: asc + - name: limit + in: query + description: 'Maximum number of results to return' + schema: + type: integer + minimum: 1 + maximum: 200 + default: 50 + - name: offset + in: query + description: 'Number of matching releases to skip' + schema: + type: integer + minimum: 0 + default: 0 + responses: + '200': + description: 'Tagged releases containing the object' + content: + application/json: + schema: + type: object + properties: + object_ref: + type: string + data: + type: array + items: + $ref: '../components/release-tracks.yml#/components/schemas/release-by-object-entry' + pagination: + type: object + properties: + total: + type: integer + limit: + type: integer + offset: + type: integer + '400': + description: 'Malformed STIX ID or invalid query parameter' + /api/release-tracks/new: post: summary: 'Create a new release track' @@ -901,7 +976,7 @@ paths: responses: '204': description: 'Snapshot deleted successfully' - '400': + '409': description: 'Cannot delete tagged snapshot' '404': description: 'Snapshot not found' diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index db26435c..0d1dd346 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -28,6 +28,10 @@ const { stixVersionQuerySchema, booleanQuerySchema, trackTypeQuerySchema, + releaseOrderQuerySchema, + releaseLimitQuerySchema, + releaseOffsetQuerySchema, + stixIdentifierSchema, trackEntryStatusSchema, createTrackBodySchema, createFromBundleBodySchema, @@ -229,6 +233,35 @@ exports.listReleaseTracks = async function listReleaseTracks(req, res, next) { } }; +/** GET /api/release-tracks/objects/:objectRef/releases */ +exports.getReleasesByObject = async function getReleasesByObject(req, res, next) { + try { + const objectRefResult = stixIdentifierSchema.safeParse(req.params.objectRef); + if (!objectRefResult.success) { + return next( + new BadRequestError({ + message: 'Invalid STIX object reference', + details: objectRefResult.error.errors, + }), + ); + } + + const options = { + type: parseOptionalQueryStrict(req.query.type, trackTypeQuerySchema, undefined, 'type'), + order: parseOptionalQueryStrict(req.query.order, releaseOrderQuerySchema, 'asc', 'order'), + limit: parseOptionalQueryStrict(req.query.limit, releaseLimitQuerySchema, 50, 'limit'), + offset: parseOptionalQueryStrict(req.query.offset, releaseOffsetQuerySchema, 0, 'offset'), + }; + + const result = await releaseTracksService.getReleasesByObject(objectRefResult.data, options); + logger.debug(`Success: Retrieved tagged releases for object ${objectRefResult.data}`); + return res.status(200).send(result); + } catch (err) { + logger.error('Failed to retrieve tagged releases by object: ' + err); + return next(err); + } +}; + /** POST /api/release-tracks/new */ exports.createReleaseTrack = async function createReleaseTrack(req, res, next) { try { diff --git a/app/exceptions/index.js b/app/exceptions/index.js index c84af8ad..d4c74479 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -297,6 +297,12 @@ class AlreadyReleasedError extends CustomError { } } +class TaggedSnapshotDeletionError extends CustomError { + constructor(version, options) { + super(`Tagged snapshot version ${version} cannot be deleted`, options); + } +} + class MemberPinnedRevisionError extends CustomError { constructor(options) { super( @@ -371,6 +377,7 @@ module.exports = { //** Version control errors */ AlreadyReleasedError, + TaggedSnapshotDeletionError, InvalidVersionError, //** Release track errors */ diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 78633c7e..248d6886 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -36,6 +36,7 @@ const { AlreadyRevokedError, SelfRevocationError, AlreadyReleasedError, + TaggedSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, NoTaggedSnapshotsError, @@ -130,6 +131,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof DuplicateNameError || err instanceof AlreadyRevokedError || err instanceof AlreadyReleasedError || + err instanceof TaggedSnapshotDeletionError || err instanceof ReleaseConflictError || err instanceof MemberPinnedRevisionError || err instanceof ObjectHasValidationIssuesError || diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 526c955e..d8e3143c 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -191,6 +191,12 @@ const booleanQuerySchema = z.union([z.boolean(), z.stringbool()]); const trackTypeQuerySchema = z.enum(['standard', 'virtual']); +const releaseOrderQuerySchema = z.enum(['asc', 'desc']); + +const releaseLimitQuerySchema = z.coerce.number().int().min(1).max(200); + +const releaseOffsetQuerySchema = z.coerce.number().int().min(0); + const bumpTypeSchema = z.enum(['major', 'minor']); const workflowStatusSchema = z.enum(['work-in-progress', 'awaiting-review', 'reviewed']); @@ -428,6 +434,9 @@ module.exports = { stixVersionQuerySchema, booleanQuerySchema, trackTypeQuerySchema, + releaseOrderQuerySchema, + releaseLimitQuerySchema, + releaseOffsetQuerySchema, bumpTypeSchema, workflowStatusSchema, trackEntryStatusSchema, diff --git a/app/models/release-tracks/release-track-registry-model.js b/app/models/release-tracks/release-track-registry-model.js index 3bf2ef4b..4c0c5e2c 100644 --- a/app/models/release-tracks/release-track-registry-model.js +++ b/app/models/release-tracks/release-track-registry-model.js @@ -24,6 +24,18 @@ const snapshotScheduleDefinition = { }; const snapshotScheduleSchema = new mongoose.Schema(snapshotScheduleDefinition, { _id: false }); +const taggedReleaseDefinition = { + snapshot_modified: { type: Date, required: true }, + version: { + type: String, + required: true, + validate: validateVersion, + }, + tagged_at: { type: Date, required: true }, + tagged_by: { type: String, required: true }, +}; +const taggedReleaseSchema = new mongoose.Schema(taggedReleaseDefinition, { _id: false }); + // --- Registry document definition --- const releaseTrackRegistryDefinition = { @@ -54,6 +66,7 @@ const releaseTrackRegistryDefinition = { }, snapshot_count: { type: Number, default: 0 }, tagged_release_count: { type: Number, default: 0 }, + tagged_releases: { type: [taggedReleaseSchema], default: [] }, // Virtual tracks only snapshot_schedule: { type: snapshotScheduleSchema, default: undefined }, diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 6a123f65..c2630d4e 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -353,6 +353,17 @@ releaseTrackSnapshotSchema.index({ id: 1, modified: -1 }, { unique: true }); // Find the latest tagged version releaseTrackSnapshotSchema.index({ id: 1, version: 1 }); +// Historical releases-by-object lookup. Draft snapshots are deliberately +// excluded because they are numerous, mutable through cloning, and never +// eligible for the endpoint. +releaseTrackSnapshotSchema.index( + { 'members.object_ref': 1, modified: -1 }, + { + name: 'tagged_members_object_ref', + partialFilterExpression: { version: { $type: 'string' } }, + }, +); + // ============================================================================= // Exports // ============================================================================= diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 9a2475d1..9731f3f1 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -90,6 +90,49 @@ class ReleaseTrackDynamicRepository { } } + async getTaggedSnapshotMetadata(trackId) { + try { + const Model = this._getModel(trackId); + return await Model.find({ id: trackId, version: { $type: 'string' } }) + .select('modified version version_history') + .sort({ modified: 1 }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async findTaggedSnapshotsContainingObject(trackId, snapshotModifiedValues, objectRef) { + if (!snapshotModifiedValues || snapshotModifiedValues.length === 0) { + return []; + } + + try { + const Model = this._getModel(trackId); + return await Model.find( + { + id: trackId, + modified: { $in: snapshotModifiedValues }, + version: { $type: 'string' }, + 'members.object_ref': objectRef, + }, + { + id: 1, + type: 1, + name: 1, + modified: 1, + version: 1, + members: { $elemMatch: { object_ref: objectRef } }, + }, + ) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async getAllSnapshots(trackId, options = {}) { try { const Model = this._getModel(trackId); diff --git a/app/repository/release-tracks/release-track-registry.repository.js b/app/repository/release-tracks/release-track-registry.repository.js index de21751e..d3520cd4 100644 --- a/app/repository/release-tracks/release-track-registry.repository.js +++ b/app/repository/release-tracks/release-track-registry.repository.js @@ -86,6 +86,45 @@ class ReleaseTrackRegistryRepository { } } + async findWithTaggedReleases(options = {}) { + try { + const query = { 'tagged_releases.0': { $exists: true } }; + if (options.type) { + query.type = options.type; + } + + return await this.model + .find(query) + .select('track_id type name tagged_releases') + .sort({ track_id: 1 }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async replaceTaggedReleases(trackId, taggedReleases, latestTaggedVersion) { + try { + return await this.model + .findOneAndUpdate( + { track_id: trackId }, + { + $set: { + tagged_releases: taggedReleases, + tagged_release_count: taggedReleases.length, + latest_tagged_version: latestTaggedVersion, + updated_at: new Date(), + }, + }, + { new: true, runValidators: true, lean: true }, + ) + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async updateByTrackId(trackId, updates) { try { const result = await this.model diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 17209fb2..d7c16ce6 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -32,6 +32,14 @@ router releaseTracksController.listReleaseTracks, ); +router + .route('/release-tracks/objects/:objectRef/releases') + .get( + authn.authenticate, + authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + releaseTracksController.getReleasesByObject, + ); + router .route('/release-tracks/new') .post( diff --git a/app/services/release-tracks/release-history-service.js b/app/services/release-tracks/release-history-service.js new file mode 100644 index 00000000..f920ba7b --- /dev/null +++ b/app/services/release-tracks/release-history-service.js @@ -0,0 +1,133 @@ +'use strict'; + +// ============================================================================= +// Release History Service +// +// Maintains the compact tagged-release catalogue in releaseTrackRegistry and +// answers global object -> tagged release queries by bounded fan-out across +// the per-track snapshot collections. +// ============================================================================= + +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); +const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const versionUtils = require('../../lib/release-tracks/version-utils'); + +const QUERY_CONCURRENCY = 12; + +function sameInstant(left, right) { + return new Date(left).getTime() === new Date(right).getTime(); +} + +function tagMetadataForSnapshot(snapshot) { + const historyEntry = (snapshot.version_history || []).find( + (entry) => + entry.version === snapshot.version && sameInstant(entry.snapshot_id, snapshot.modified), + ); + + return { + snapshot_modified: snapshot.modified, + version: snapshot.version, + tagged_at: historyEntry?.tagged_at || snapshot.modified, + tagged_by: historyEntry?.tagged_by || 'system', + }; +} + +function highestVersion(taggedReleases) { + let highest = null; + for (const release of taggedReleases) { + if (!highest || versionUtils.compareVersions(release.version, highest) > 0) { + highest = release.version; + } + } + return highest; +} + +async function mapWithConcurrency(items, concurrency, mapper) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + results[index] = await mapper(items[index], index); + } + } + + const workerCount = Math.min(concurrency, items.length); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return results; +} + +exports.getTrackWideVersionHistory = async function getTrackWideVersionHistory(trackId) { + const snapshots = await dynamicRepo.getTaggedSnapshotMetadata(trackId); + return snapshots.map((snapshot) => ({ version: snapshot.version })); +}; + +exports.reconcileTaggedReleases = async function reconcileTaggedReleases(trackId) { + const snapshots = await dynamicRepo.getTaggedSnapshotMetadata(trackId); + const taggedReleases = snapshots.map(tagMetadataForSnapshot); + await registryRepo.replaceTaggedReleases(trackId, taggedReleases, highestVersion(taggedReleases)); + return taggedReleases; +}; + +exports.getReleasesByObject = async function getReleasesByObject(objectRef, options = {}) { + const tracks = await registryRepo.findWithTaggedReleases({ type: options.type }); + + const matchesByTrack = await mapWithConcurrency(tracks, QUERY_CONCURRENCY, async (track) => { + const releaseByModified = new Map( + track.tagged_releases.map((release) => [ + new Date(release.snapshot_modified).toISOString(), + release, + ]), + ); + const snapshots = await dynamicRepo.findTaggedSnapshotsContainingObject( + track.track_id, + track.tagged_releases.map((release) => release.snapshot_modified), + objectRef, + ); + + return snapshots.map((snapshot) => { + const release = releaseByModified.get(new Date(snapshot.modified).toISOString()); + const member = snapshot.members[0]; + return { + track_id: track.track_id, + track_type: snapshot.type || track.type, + track_name: snapshot.name || track.name, + version: snapshot.version, + snapshot_modified: snapshot.modified, + tagged_at: release.tagged_at, + tagged_by: release.tagged_by, + object_modified: member.object_modified, + }; + }); + }); + + const direction = options.order === 'desc' ? -1 : 1; + const data = matchesByTrack.flat().sort((left, right) => { + const timeComparison = + new Date(left.snapshot_modified).getTime() - new Date(right.snapshot_modified).getTime(); + if (timeComparison !== 0) return timeComparison * direction; + const trackComparison = left.track_id.localeCompare(right.track_id); + if (trackComparison !== 0) return trackComparison; + return versionUtils.compareVersions(left.version, right.version) * direction; + }); + + const offset = options.offset || 0; + const limit = options.limit || 50; + + return { + object_ref: objectRef, + data: data.slice(offset, offset + limit), + pagination: { + total: data.length, + limit, + offset, + }, + }; +}; + +exports._private = { + highestVersion, + mapWithConcurrency, + tagMetadataForSnapshot, +}; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 502c2b8a..db8ed11b 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -23,6 +23,7 @@ const exportService = require('./export-service'); const ephemeralService = require('./ephemeral-service'); const bundleImportService = require('./bundle-import-service'); const memberSyncService = require('./member-sync-service'); +const releaseHistoryService = require('./release-history-service'); const attackObjectsService = require('../stix/attack-objects-service'); const userAccountsService = require('../system/user-accounts-service'); @@ -164,6 +165,10 @@ exports.listTracks = function listTracks(options) { return snapshotService.listTracks(options); }; +exports.getReleasesByObject = function getReleasesByObject(objectRef, options) { + return releaseHistoryService.getReleasesByObject(objectRef, options); +}; + exports.createTrack = function createTrack(data) { return snapshotService.createTrack(data); }; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index eaa62a6b..b54aa447 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -19,7 +19,12 @@ const modelFactory = require('../../models/release-tracks/model-factory'); const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); -const { TrackNotFoundError, NotFoundError } = require('../../exceptions'); +const versionUtils = require('../../lib/release-tracks/version-utils'); +const { + TrackNotFoundError, + NotFoundError, + TaggedSnapshotDeletionError, +} = require('../../exceptions'); // ============================================================================= // Internal helpers @@ -63,8 +68,13 @@ async function syncRegistryCounters(trackId) { // Latest snapshot is first (sorted desc by modified) const latestSnapshotModified = snapshots.length > 0 ? snapshots[0].modified : null; - // Latest tagged version: find the tagged snapshot with the highest modified - const latestTaggedVersion = tagged.length > 0 ? tagged[0].version : null; + const latestTaggedVersion = tagged.reduce( + (highest, snapshot) => + !highest || versionUtils.compareVersions(snapshot.version, highest) > 0 + ? snapshot.version + : highest, + null, + ); await registryRepo.updateByTrackId(trackId, { snapshot_count: snapshotCount, @@ -531,6 +541,10 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { }); } + if (snapshot.version != null) { + throw new TaggedSnapshotDeletionError(snapshot.version); + } + await dynamicRepo.deleteSnapshot(trackId, modified); await syncRegistryCounters(trackId); diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 8150b163..372cd8ed 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -16,9 +16,9 @@ const snapshotService = require('./snapshot-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); -const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); const versionUtils = require('../../lib/release-tracks/version-utils'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); +const releaseHistoryService = require('./release-history-service'); const logger = require('../../lib/logger'); const { AlreadyReleasedError } = require('../../exceptions'); @@ -37,10 +37,14 @@ const { AlreadyReleasedError } = require('../../exceptions'); async function _doBump(trackId, snapshot, options) { // Guard: cannot re-tag an already-tagged snapshot if (snapshot.version != null) { + await releaseHistoryService.reconcileTaggedReleases(trackId); throw new AlreadyReleasedError(snapshot.version); } - const versionHistory = snapshot.version_history || []; + // A historical draft's embedded version_history can predate newer tags. + // Read the track-wide tagged releases so retroactive tagging cannot reuse or + // regress a version. + const versionHistory = await releaseHistoryService.getTrackWideVersionHistory(trackId); // Calculate version const version = versionUtils.calculateNextVersion(versionHistory, options.type, options.version); @@ -122,15 +126,13 @@ async function _doBump(trackId, snapshot, options) { if (!tagged) { // Race condition: snapshot was already tagged between our read and update + await releaseHistoryService.reconcileTaggedReleases(trackId); throw new AlreadyReleasedError('(concurrent tag)'); } - // Update registry counters - await registryRepo.updateByTrackId(trackId, { - latest_tagged_version: version, - tagged_release_count: versionHistory.length + 1, - updated_at: now, - }); + // Rebuild the registry's compact tagged-release catalogue from the source + // snapshots. This is idempotent and repairs missed/partial prior updates. + await releaseHistoryService.reconcileTaggedReleases(trackId); // The staged → members promotion changed tier membership. Re-read the // latest snapshot rather than using `tagged` — bumpByModified may have @@ -195,7 +197,11 @@ exports.bumpByModified = async function bumpByModified(trackId, modified, option exports.previewBump = async function previewBump(trackId, _format) { const snapshot = await snapshotService.getLatestSnapshot(trackId); - const versionHistory = snapshot.version_history || []; + // The latest draft may have been cloned before a historical snapshot was + // retroactively tagged. Use the authoritative track-wide ledger here for + // the same reason _doBump does, otherwise preview can advertise a version + // that the subsequent bump rejects. + const versionHistory = await releaseHistoryService.getTrackWideVersionHistory(trackId); const staged = snapshot.staged || []; const existingMembers = snapshot.members || []; diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js new file mode 100644 index 00000000..0e9d3612 --- /dev/null +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -0,0 +1,281 @@ +const mongoose = require('mongoose'); +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); +const backfillMigration = require('../../../../migrations/20260716000000-backfill-release-track-tagged-releases'); + +const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name, identity = {}) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + ...identity, + created: identity.created || timestamp, + modified: identity.modified || timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('GET /api/release-tracks/objects/:objectRef/releases', function () { + let app; + let passportCookie; + let objectRevisionA; + let objectRevisionB; + let otherObject; + let trackA; + let trackATaggedSnapshot; + let trackB; + let virtualTrack; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + + objectRevisionA = await post('/api/techniques', buildTechnique('Release Lineage A'), 201); + objectRevisionB = await post( + '/api/techniques', + buildTechnique('Release Lineage B', { + id: objectRevisionA.stix.id, + created: objectRevisionA.stix.created, + modified: new Date(new Date(objectRevisionA.stix.modified).getTime() + 1000).toISOString(), + }), + 201, + ); + otherObject = await post('/api/techniques', buildTechnique('Other Release Object'), 201); + + const createdA = await createTrack('Releases By Object A'); + trackA = createdA.id; + const initialSnapshotModified = createdA.modified; + trackATaggedSnapshot = await setMembers(trackA, [objectRevisionA]); + await post(`/api/release-tracks/${trackA}/bump`, { type: 'minor' }, 200); + + // Remove the requested object from the latest state and tag that state. + // The earlier tagged release must remain discoverable despite its current + // backref disappearing. + await setMembers(trackA, [otherObject]); + await post(`/api/release-tracks/${trackA}/bump`, { type: 'minor' }, 200); + + // Retroactively tag the original empty draft. Its embedded history is + // stale, so the track-wide version ledger must produce 1.2 rather than 1.0. + await post( + `/api/release-tracks/${trackA}/snapshots/${initialSnapshotModified}/bump`, + { type: 'minor' }, + 200, + ); + + const createdB = await createTrack('Releases By Object B'); + trackB = createdB.id; + await setMembers(trackB, [objectRevisionB]); + await post(`/api/release-tracks/${trackB}/bump`, { type: 'minor' }, 200); + + // A tagged snapshot where the object is only a candidate must not match. + const candidateOnly = await createTrack('Releases Candidate Only'); + await post( + `/api/release-tracks/${candidateOnly.id}/candidates`, + { object_refs: [{ id: objectRevisionA.stix.id, modified: objectRevisionA.stix.modified }] }, + 200, + ); + await post(`/api/release-tracks/${candidateOnly.id}/bump`, { type: 'minor' }, 200); + + // Virtual tagged releases use the same direct-members semantics. + const virtual = await post( + '/api/release-tracks/new', + { name: 'Releases By Object Virtual', type: 'virtual' }, + 201, + ); + virtualTrack = virtual.id; + await put(`/api/release-tracks/${virtualTrack}/composition`, { + component_tracks: [{ track_id: trackB, resolution_strategy: 'latest_tagged', priority: 0 }], + }); + await post(`/api/release-tracks/${virtualTrack}/snapshots/create`, {}, 201); + await post(`/api/release-tracks/${virtualTrack}/bump`, { type: 'minor' }, 200); + }); + + async function post(path, body, status) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (response.status !== status) { + throw new Error( + `${path} expected ${status}, received ${response.status}: ${JSON.stringify(response.body)}`, + ); + } + return response.body; + } + + async function put(path, body, status = 200) { + const response = await request(app) + .put(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + async function get(path, status = 200) { + return request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + } + + async function createTrack(name) { + return post('/api/release-tracks/new', { name, type: 'standard' }, 201); + } + + async function setMembers(trackId, objects) { + return post( + `/api/release-tracks/${trackId}/contents`, + { + x_mitre_contents: objects.map((object) => ({ + obj_ref: object.stix.id, + obj_modified: object.stix.modified, + })), + }, + 200, + ); + } + + it('returns historical tagged member occurrences across tracks and revisions', async function () { + const response = await get(`/api/release-tracks/objects/${objectRevisionA.stix.id}/releases`); + + expect(response.body.object_ref).toBe(objectRevisionA.stix.id); + expect(response.body.pagination).toEqual({ total: 3, limit: 50, offset: 0 }); + expect(response.body.data).toHaveLength(3); + + const standardA = response.body.data.find((entry) => entry.track_id === trackA); + const standardB = response.body.data.find((entry) => entry.track_id === trackB); + const virtual = response.body.data.find((entry) => entry.track_id === virtualTrack); + + expect(standardA).toMatchObject({ + track_type: 'standard', + track_name: 'Releases By Object A', + version: '1.0', + object_modified: objectRevisionA.stix.modified, + }); + expect(standardB).toMatchObject({ + track_type: 'standard', + version: '1.0', + object_modified: objectRevisionB.stix.modified, + }); + expect(virtual).toMatchObject({ + track_type: 'virtual', + version: '1.0', + object_modified: objectRevisionB.stix.modified, + }); + expect(response.body.data.every((entry) => entry.tagged_at && entry.tagged_by)).toBe(true); + }); + + it('maintains a reconciled registry catalogue during normal and retroactive tagging', async function () { + const registry = await ReleaseTrackRegistry.findOne({ track_id: trackA }).lean().exec(); + expect(registry.tagged_release_count).toBe(3); + expect(registry.tagged_releases).toHaveLength(3); + expect(registry.tagged_releases.map((release) => release.version).sort()).toEqual([ + '1.0', + '1.1', + '1.2', + ]); + expect(registry.latest_tagged_version).toBe('1.2'); + }); + + it('previews the next version from the track-wide release ledger', async function () { + // Clone the latest snapshot after the retroactive 1.2 tag. The source + // snapshot predates that tag, so its embedded history does not contain it. + await post( + `/api/release-tracks/${trackA}/meta`, + { description: 'Draft created after a retroactive tag' }, + 200, + ); + + const preview = await get(`/api/release-tracks/${trackA}/bump/preview`); + expect(preview.body.next_version_minor).toBe('1.3'); + expect(preview.body.next_version_major).toBe('2.0'); + }); + + it('supports type filtering, ordering, and pagination', async function () { + const standard = await get( + `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=standard&order=desc&limit=1&offset=1`, + ); + expect(standard.body.pagination).toEqual({ total: 2, limit: 1, offset: 1 }); + expect(standard.body.data).toHaveLength(1); + expect(standard.body.data[0].track_type).toBe('standard'); + + const virtual = await get( + `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=virtual`, + ); + expect(virtual.body.pagination.total).toBe(1); + expect(virtual.body.data[0].track_id).toBe(virtualTrack); + }); + + it('returns an empty list for a valid STIX ID with no tagged membership', async function () { + const missing = 'attack-pattern--99999999-9999-4999-8999-999999999999'; + const response = await get(`/api/release-tracks/objects/${missing}/releases`); + expect(response.body).toEqual({ + object_ref: missing, + data: [], + pagination: { total: 0, limit: 50, offset: 0 }, + }); + }); + + it('rejects malformed STIX IDs and invalid query values', async function () { + await get('/api/release-tracks/objects/not-a-stix-id/releases', 400); + await get( + `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?order=sideways`, + 400, + ); + await get(`/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?limit=0`, 400); + }); + + it('rejects deletion of a tagged snapshot', async function () { + await request(app) + .delete(`/api/release-tracks/${trackA}/snapshots/${trackATaggedSnapshot.modified}`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + }); + + it('backfills missing registry refs from authoritative tagged snapshots', async function () { + await ReleaseTrackRegistry.updateOne( + { track_id: trackA }, + { + $set: { tagged_releases: [], tagged_release_count: 0, latest_tagged_version: null }, + }, + ); + + await backfillMigration.up(mongoose.connection.db); + + const registry = await ReleaseTrackRegistry.findOne({ track_id: trackA }).lean().exec(); + expect(registry.tagged_releases).toHaveLength(3); + expect(registry.tagged_release_count).toBe(3); + expect(registry.latest_tagged_version).toBe('1.2'); + + const response = await get( + `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=standard`, + ); + expect(response.body.pagination.total).toBe(2); + }); +}); diff --git a/docs/README.md b/docs/README.md index 8f956eb7..d3b4cfdc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ Guides for consumers of the REST API — endpoints, workflows, and terminology. - [Output Formats](user/release-tracks/output-formats.md): Output format specifications - [Workflow Examples](user/release-tracks/workflow-examples.md): End-to-end workflow examples - [Object Backrefs](user/release-tracks/object-backrefs.md): Release-track membership pointers on object documents (`workspace.release_tracks`) +- [Releases By Object](user/release-tracks/releases-by-object.md): Find tagged releases that directly contain a STIX object ## Developer Documentation @@ -44,6 +45,7 @@ Architecture, patterns, and implementation details for contributors. - [Member Sync Strategies](developer/release-tracks/member-sync-strategies.md): Automatic tracking of member object revisions - [Error Handling](developer/release-tracks/error-handling.md): Error handling patterns - [Implementation Notes](developer/release-tracks/implementation-notes.md): Implementation notes and decisions +- [Releases By Object](developer/release-tracks/releases-by-object.md): Registry catalogue, fan-out query, and indexing design ## Admin Documentation diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 8ce78711..6cdd7b65 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -152,10 +152,234 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho - [x] **Technique conversion should reach revision sync.** Implemented 2026-07-13 with the adapter approach (same pattern as `handleStixObjectRevokedEvent`): the `TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE` / `SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE` event payloads now carry the converted revision (`document`) and acting user, and member sync subscribes via `handleStixObjectConvertedEvent`, treating the conversion as a `new-revision` trigger through the workflow gate — candidate/staged pins move to the converted revision, member tracks enroll it as a candidate. The conversion responses refresh `workspace.release_tracks` after event processing (read-your-own-writes). Tests: conversion cases in `release-tracks-change-capture.spec.js` and the updated clone-strip test in `release-tracks-backrefs.spec.js`. +## Get Releases By Object + +- [ ] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a + caller can retrieve every tagged snapshot whose `members` tier directly + contains the supplied STIX ID, across all object revisions and release + tracks. + +### Design + +The existing `workspace.release_tracks` backrefs cannot answer this query: +they intentionally describe only each track's latest snapshot. A release that +historically contained an object must still be returned after a later snapshot +removes it. Conversely, copying all tagged snapshots into a new global MongoDB +collection would duplicate the existing per-track source data and undermine +the collection-per-track storage boundary. + +Use `releaseTrackRegistry` as a compact global forward catalogue instead. Its +single document per track gains a server-maintained `tagged_releases` array: + +```javascript +tagged_releases: [{ + snapshot_modified: Date, // (track_id, snapshot_modified) identifies the snapshot + version: String, + tagged_at: Date, + tagged_by: String +}] +``` + +`tagged_release_count` is derived from `tagged_releases.length`. The actual +snapshot — including the authoritative `members` pins — remains in the track's +dynamic collection. Tagging reconciles this registry projection from the +source snapshots rather than incrementally appending, so retries and +retroactive tagging are idempotent and self-healing. A migration backfills +existing tracks. + +The endpoint is stateless but necessarily fan-outs: read registry documents +with tagged releases, then issue one bounded-concurrency query per eligible +track using all of that track's tagged `snapshot_modified` values. Flatten, +sort deterministically, and paginate the matches. Registry references reduce +the search to tagged snapshots, but they are a forward index (track → release), +not an inverted object → release index; eliminating the per-track fan-out would +require a separate denormalized membership index and is deliberately out of +scope. + +Add a partial multikey index to every dynamic track collection for +`members.object_ref`, limited to snapshots whose `version` is a string. Drafts +therefore incur no index cost, and draft squashing does not affect the lookup. + +### Semantics + +- Match the STIX ID across all revisions; return the pinned `object_modified` + for each release. +- Include standard and virtual tracks by default; optional `type` filtering. +- Include only direct `members` entries from tagged snapshots. Do not include + candidates, staged/quarantined entries, or secondary objects added during + bundle export. +- Support `order=asc|desc` by `snapshot_modified`, plus `limit` and `offset`. +- Return 200 with an empty result for a valid STIX ID with no tagged releases; + malformed IDs return 400. +- Ascending order describes first *published/tagged* appearance, not the time + the object first entered an untagged draft. + +### Checklist + +- [x] Registry schema/repository: add `tagged_releases`, reconciliation, and + derived count/latest-version maintenance. +- [x] Dynamic snapshot schema/repository: add the tagged-member partial index + and a projected `findTaggedSnapshotsContainingObject` query. +- [x] Versioning: reconcile registry metadata after tagging and validate + version progression against track-wide tagged releases rather than a + potentially stale historical snapshot's embedded `version_history`. +- [x] API: route, controller Zod validation, facade/service orchestration, + deterministic pagination, and OpenAPI contract. +- [x] Migration: backfill registry tagged-release refs and ensure the new index + on all existing dynamic track collections. +- [x] Regression tests: multiple tracks/releases/revisions, removal after an + earlier release, retroactive tag, virtual track, draft/non-member exclusion, + filtering/order/pagination, empty/malformed input, and backfill behavior. +- [x] User/developer docs and Bruno request. +- [ ] Verification: targeted spec first, then the complete `npm test` suite. + - Targeted endpoint spec: 8 passing; release-track directory: 69 passing; + lint, OpenAPI validation, and middleware suite pass. + - `npm test` was attempted three times on 2026-07-16. Each API run reached + 861-880 passing but hit different roaming failures in unrelated legacy + specs (collection-bundle timeout, missing anonymous-session cookie, and + transient version lookups). Every failed file passed when rerun in + isolation. A clean full-suite run is still required before this task meets + the repository definition of done. + +## Snapshot Retention (Squash on Tag) + +- [ ] Implement draft-snapshot squashing so release cycles don't accumulate + unbounded snapshot storage. Design captured 2026-07-15; assessed as sound — + see analysis below. + +### Why + +Every mutation clones the full snapshot document (`cloneSnapshot` in +`snapshot-service.js`): metadata edits, config edits, tier operations, and — +critically — every member-sync enrollment. Each snapshot embeds the complete +`members`/`staged`/`candidates` arrays (~100–150 bytes BSON per pin entry). + +At ATT&CK scale (~10k–20k tracked objects), each snapshot document is +~1–3 MB. A release cycle where 10% of a 10k-object track is edited produces +~1,000 member-sync snapshots ≈ 1–3 GB of drafts per track per cycle — nearly +all of it intermediate states nobody will ever read again. Storage per cycle +is O(edits × track_size); the per-write clone is the root cause, but squashing +at the tag checkpoint caps the steady state without touching the write path. + +Mitigating facts (verified in code): + +- Bulk endpoints already exist: `addCandidates`, `promoteCandidates`, + `reviewCandidates`, `demoteStaged` all take arrays and produce **one** + snapshot per call. Initial population of a track is 3 snapshots (create → + bulk-add → bulk-promote), plus an in-place tag (tagging via + `tagSnapshotInPlace` creates **zero** snapshots). The N-snapshot trap is + calling the bulk endpoints once per object — document this loudly in user + docs, but no code change needed there. +- `::created` events for brand-new objects are no-ops for member sync + (`findTracksReferencingObject` only matches already-tracked `stix.id`s). + The O(N²) trap is bulk *re-imports/updates* of already-tracked objects + (e.g. re-importing a modified 20k-object bundle → 20k snapshots × MBs each). +- `version_history` is embedded in and carried forward by every clone, so the + release ledger survives squashing — tagged snapshots and the latest draft + always hold the full history. +- Backref reconciliation (`emitContentsChanged`) only ever reads the **latest** + snapshot; deleting non-latest drafts requires no backref work. + +### Semantics + +"Squash" = bulk-delete draft snapshots (`version == null`) older than a +boundary, preserving: all tagged snapshots, the boundary snapshot, and always +the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. + +1. **Squash-on-tag (opt-in):** `POST /api/release-tracks/:id/bump` (and + `.../snapshots/:modified/bump`) accept `squash: boolean` (default `false`). + After a successful tag of snapshot S, delete all snapshots matching + `{ id, version: null, modified: { $lt: S.modified } }`. Drafts newer than S + (work already underway toward the next release) survive. Response gains + `squashed_count`. +2. **Standalone maintenance endpoint** (recovery from bulk-operation + accidents, no tag required): `POST /api/release-tracks/:id/snapshots/squash` + with optional `before` (ISO timestamp; defaults to the latest tagged + snapshot's `modified`; if no tagged release exists and `before` is omitted, + 400). Same delete filter; never deletes the latest snapshot even if it is + an untagged draft and `before` post-dates it. +3. **Concurrency safety:** the filter can't race member sync — concurrent + clones get `modified = now`, which is always ≥ the boundary, so they are + never matched. Tag-then-squash need not be atomic: a crash between the two + just leaves drafts behind (retryable via the maintenance endpoint). +4. After deletion: one `syncRegistryCounters(trackId)` call; **no** + `emitContentsChanged` (latest snapshot unchanged by construction). Add a + repo-level `deleteDraftSnapshotsBefore(trackId, boundary)` (`deleteMany`) + rather than looping `deleteSnapshot` (which emits per-delete events). + +### Drawbacks accepted (documented trade-offs, not blockers) + +- **Provenance loss.** Intermediate drafts are the only record of the journey: + who added/staged what when (`object_added_by`, `object_staged_at`), status + transitions, `modified-in-place` markers that were later cleared. Promotion + strips staged metadata from member entries, so after squash only the final + state remains. This is exactly git-squash semantics and is why the flag is + opt-in, but teams that need review audit trails must not squash (or we later + add a roll-up audit record — see Future). +- **Retro-tagging is foreclosed.** `bumpByModified` can no longer tag a + squashed draft. Consistent by construction: squashing is the declaration + that intermediates don't matter. Note the "undo/move the tag" worry is + already moot — versions are immutable once set, re-tagging throws + `AlreadyReleasedError`, and no untag endpoint exists. The genuine loss is + forensic/DR, mitigated only by Mongo backups. +- **Virtual tracks: excluded from v1.** Their scheduled snapshots + (`snapshot_schedule`) exist precisely to build a periodic history; + squash-on-tag would destroy the thing the schedule creates. Reject + (or no-op with a warning) squash on virtual tracks until there's a + considered retention policy for them. + +### Alternatives considered + +- *Amend-in-place* (member sync mutates the latest draft instead of cloning): + attacks the root cause but breaks the "every modification is a new + snapshot" invariant, complicates concurrent reads, and silently degrades + the audit trail for everyone. Rejected for now. +- *Delta/structural-sharing storage*: large refactor of the snapshot store; + revisit only if squash proves insufficient. +- *TTL/retention config* (e.g. `config.retention.auto_squash_on_tag`, + max-draft-age): natural follow-on once manual squash exists. + +### Checklist + +- [ ] Repo: `deleteDraftSnapshotsBefore(trackId, boundary)` in + `release-track-dynamic.repository.js` (deleteMany on + `{ id, version: null, modified: { $lt: boundary } }`, excluding the latest + snapshot's `modified`). +- [ ] Service: squash logic in `versioning-service.js` (`squash` option on + `_doBump`) + standalone squash operation (probably `snapshot-service.js`); + reject for virtual tracks; return `squashed_count`. +- [ ] Controller/routes: `squash` in the Zod bump body schema; new + `POST /api/release-tracks/:id/snapshots/squash` route with Zod-validated + optional `before`. +- [ ] OpenAPI: bump request body + new squash path. +- [ ] Regression tests (`release-tracks-squash.spec.js`): squash-on-tag + deletes only pre-tag drafts; tagged snapshots survive; drafts newer than + the tagged snapshot survive; latest-draft never deleted by maintenance + squash; registry counters resync; backrefs untouched; virtual track + rejected; no-tagged-release + no `before` → 400; idempotent re-squash. +- [ ] Docs: `docs/user/release-tracks/versioning.md` (squash behavior + + the bulk-endpoints-vs-per-object-loop warning for initial population), + `docs/developer/release-tracks/` (why, trade-offs, provenance loss). +- [ ] Bruno: bump `.bru` gains `~squash` toggle; new squash request file. + +### Future (not in scope) + +- Roll-up audit record written at squash time (compact per-object journey + summary appended to the version_history entry or a side collection) to + soften the provenance loss. +- Retention config for auto-squash and for virtual-track snapshot history. +- Coalescing/debouncing member-sync snapshots during bulk update storms + (the re-import O(N²) trap) — e.g. a bulk-import context that suspends + per-object snapshotting and emits one consolidated snapshot at the end. + ## Small Fixes - [ ] **Composition schema mismatch: `priority`.** `PUT /api/release-tracks/:id/composition` — the Zod schema (`componentTrackSchema`) marks `priority` optional, but the mongoose snapshot schema requires it, so omitting it passes validation and then fails the save with a 500 (`DatabaseError`) instead of a 400. Align the schemas (either default `priority` or make it required in Zod). Found 2026-07-15 while testing virtual-track backrefs. +- [ ] **`deleteSnapshot` lacks a tagged-release guard.** `DELETE /api/release-tracks/:id/snapshots/:modified` (`snapshot-service.deleteSnapshot`) deletes any snapshot, including tagged releases — contradicting the "immutable once set" versioning rule. Should 409 on `version != null` (a squash implementation must also filter `version: null`; see Snapshot Retention section). Found 2026-07-15 while designing squash. + +- [ ] **`syncRegistryCounters` scales with snapshot count.** It fetches *all* snapshots (`getAllSnapshots` with projection) on every clone to recount — O(snapshot_count) reads per write, on the hottest path (member sync). Fine post-squash; consider a count query or incremental counters if draft accumulation between tags is large. + ## Diffing Endpoint - [ ] Implement object diffing endpoints for snapshots. Users should be able to effectively preview changes to objects before tier transitions (candidates, staged, members). @@ -269,4 +493,4 @@ Links/references between notes and snapshot objects will be one-to-many. A singl }, "stix": "StixObject", } -``` \ No newline at end of file +``` diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index fff65036..c049ef6f 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -9,31 +9,74 @@ This document tracks new database schemas, interfaces, etc.; as well as changes #### Naming Conventions **Release Track Names:** + - Must contain only alphanumeric characters and spaces: `[a-zA-Z0-9 ]` - No special characters allowed (no hyphens, underscores, or other punctuation) - Examples: `Enterprise`, `Groups Monthly`, `Techniques Quarterly` **Release Track IDs:** MongoDB Collections and release track IDs follow a simple naming convention: + ``` release-track--$uuid ``` Where: + - `release-track--` is a fixed prefix - `$uuid` is a dynamically generated UUIDv4 identifier (must be unique) **Example:** A user creates a release track named `Groups Monthly`: + 1. Name: `Groups Monthly` (user-specified, stored in the `name` field) 2. UUID: `8b0ff8f9-27fd-4d7e-bbc9-8fe9465342af` (generated) 3. Final ID: `release-track--8b0ff8f9-27fd-4d7e-bbc9-8fe9465342af` This ID is used for: + - MongoDB Collection name - The `id` field in release track snapshots - API endpoint references (`/api/release-tracks/:id`) +### Release Track Registry + +`releaseTrackRegistry` contains exactly one document per release track. It is +the global catalogue for discovering dynamic track collections and their +compact metadata; snapshot contents remain authoritative in the per-track +collections. + +```javascript +{ + track_id: "release-track--123", + type: "standard", + name: "ATT&CK Enterprise", + latest_snapshot_modified: "2024-02-01T10:00:00.000Z", + latest_tagged_version: "2.0", + snapshot_count: 47, + tagged_release_count: 2, + tagged_releases: [ + { + snapshot_modified: "2024-01-15T16:20:00.000Z", + version: "1.0", + tagged_at: "2024-01-15T17:00:00.000Z", + tagged_by: "user-id" + }, + { + snapshot_modified: "2024-02-01T10:00:00.000Z", + version: "2.0", + tagged_at: "2024-02-01T11:00:00.000Z", + tagged_by: "user-id" + } + ] +} +``` + +`tagged_release_count` is derived from `tagged_releases.length`, and +`latest_tagged_version` is the highest semantic MAJOR.MINOR version rather +than the tag on the chronologically newest snapshot. See +[releases-by-object.md](releases-by-object.md) for reconciliation and query +details. ### Release Track Types @@ -43,6 +86,7 @@ Release tracks can be one of two types: 2. **Virtual Release Tracks**: Computed aggregations of other release tracks, used to compose releases from multiple source tracks The type is identified by the `stix.type` field: + - Standard tracks: `stix.type` is omitted or set to `"standard"` - Virtual tracks: `stix.type = "virtual"` @@ -166,20 +210,21 @@ The `version_history` array tracks all tagged releases in reverse chronological ```javascript version_history: [ { - version: "2.0", // Version (MAJOR.MINOR) - tagged_at: "2024-02-01T...", // When the tagging occurred - tagged_by: "user@example.com", // Who performed the tagging - snapshot_id: "2024-02-01T10:00:00.000Z", // Which snapshot was tagged + version: '2.0', // Version (MAJOR.MINOR) + tagged_at: '2024-02-01T...', // When the tagging occurred + tagged_by: 'user@example.com', // Who performed the tagging + snapshot_id: '2024-02-01T10:00:00.000Z', // Which snapshot was tagged summary: { members_count: 3000, - promoted_count: 150 - } + promoted_count: 150, + }, }, // ... older versions -] +]; ``` This provides: + - Complete audit trail of tagged releases - Attribution for each tagged release - Chronological release history @@ -222,6 +267,7 @@ field semantics): ``` **Key Points:** + - `workspace.release_tracks` provides reverse lookup for queries like "show me all release tracks containing this object" - Entries reflect each track's **latest** snapshot and are pinned to the specific object revision the tier entry references - Same object version can have different statuses in different release tracks @@ -413,4 +459,4 @@ Virtual release tracks compute their contents by aggregating objects from compon - Snapshots are created **manually or on schedule** (never event-driven) - All snapshots start as **drafts** and must be explicitly tagged - Component tracks must exist and have at least one tagged release -- Each component track must have a unique **priority** value (no duplicates) \ No newline at end of file +- Each component track must have a unique **priority** value (no duplicates) diff --git a/docs/developer/release-tracks/error-handling.md b/docs/developer/release-tracks/error-handling.md index 36358d7e..d52f7470 100644 --- a/docs/developer/release-tracks/error-handling.md +++ b/docs/developer/release-tracks/error-handling.md @@ -7,6 +7,7 @@ **HTTP Status:** 409 Conflict **Example:** + ```json { "error": "This snapshot has already been tagged as version 1.0" @@ -18,6 +19,7 @@ ### InvalidVersionError **Thrown when:** + - Explicit version is not valid MAJOR.MINOR format - Explicit version is not greater than the previous highest version - Version bump would result in regression @@ -25,6 +27,7 @@ **HTTP Status:** 400 Bad Request **Examples:** + ```json { "error": "Version must be greater than current version 1.5" @@ -39,6 +42,16 @@ **Solution:** Provide a valid version that is greater than all previous versions. +### TaggedSnapshotDeletionError + +**Thrown when:** Attempting to delete a snapshot that has already been tagged. + +**HTTP Status:** 409 Conflict + +Tagged snapshots are immutable release records. Create or modify a draft +snapshot instead; deleting an entire release track remains a separate +track-level operation. + ### NotFoundError **Thrown when:** Collection with specified ID does not exist. @@ -46,8 +59,9 @@ **HTTP Status:** 404 Not Found **Example:** + ```json { "error": "Collection not found" } -``` \ No newline at end of file +``` diff --git a/docs/developer/release-tracks/releases-by-object.md b/docs/developer/release-tracks/releases-by-object.md new file mode 100644 index 00000000..14a58f5e --- /dev/null +++ b/docs/developer/release-tracks/releases-by-object.md @@ -0,0 +1,108 @@ +# Releases By Object: Design and Implementation + +## Problem + +`workspace.release_tracks` is a current-membership index. It mirrors the +latest snapshot of each track and therefore cannot answer which historical, +tagged releases contained a STIX object. Looking only at current backrefs +would miss a tagged release after a later snapshot removed the object. + +Release-track snapshots are also physically isolated: every track owns a +dynamic MongoDB collection. A correct global lookup must either fan out across +those collections or maintain an object-to-release inverted index. The first +implementation preserves the existing storage boundary and uses a bounded, +registry-driven fan-out. + +## Registry release catalogue + +`releaseTrackRegistry` remains the global indexing point and continues to +contain exactly one document per track. Each document carries a compact list +of tagged-release references: + +```javascript +{ + track_id: 'release-track--...', + type: 'standard', + name: 'Enterprise ATT&CK', + tagged_releases: [ + { + snapshot_modified: new Date('2026-07-13T15:52:58.508Z'), + version: '1.0', + tagged_at: new Date('2026-07-13T16:00:00.000Z'), + tagged_by: 'user-id' + } + ], + tagged_release_count: 1, + latest_tagged_version: '1.0' +} +``` + +There is no separate `snapshot_id`: a snapshot is identified by its track ID +and `modified` timestamp. `tagged_release_count` is derived from the array +length. The dynamic snapshot remains authoritative for its contents. + +### Reconciliation + +Tagging is already a two-document workflow: it mutates the snapshot in its +dynamic collection, then updates the registry. After a successful tag, the +service reads the track's tagged snapshot metadata and replaces the registry +projection. Reconciliation rather than `$push` makes the operation idempotent, +repairs missing entries, and handles retroactive tags. + +Existing deployments receive the same projection through an idempotent +database migration. Tagged snapshots are immutable and cannot be deleted; +deleting a whole track removes both its dynamic collection and registry +document. Draft-snapshot squashing is orthogonal because it only deletes +snapshots with `version == null`. + +Version calculation and monotonicity validation must use track-wide tagged +release metadata. An older draft's embedded `version_history` can predate +newer tags and is not a safe global ledger for retroactive tagging. + +## Query algorithm + +For `GET /api/release-tracks/objects/:objectRef/releases`: + +1. Read registry documents that have tagged releases, applying an optional + standard/virtual type filter. +2. For every eligible track, query its dynamic collection once with the full + set of referenced tagged snapshot timestamps and the requested + `members.object_ref`. +3. Project only snapshot metadata and the matching member entry. +4. Execute track queries through a small bounded-concurrency runner. +5. Flatten the matches, join tagging attribution from the registry, sort by + `snapshot_modified` with stable tie-breakers, then apply pagination. + +This is one database query per eligible track, not per tagged release. The +cost is still proportional to the number of tagged tracks and does not shrink +with response pagination because membership is unknown until each track is +searched. If measured production latency later makes that unacceptable, an +object-to-release inverted index is the appropriate follow-on; it is not part +of this design. + +## Per-track index + +Each dynamic collection receives a partial multikey index equivalent to: + +```javascript +{ + key: { 'members.object_ref': 1, modified: -1 }, + partialFilterExpression: { version: { $type: 'string' } } +} +``` + +Only tagged snapshots contribute index keys. This avoids amplifying the large +volume of intermediate drafts and makes the index naturally compatible with +draft squashing. + +## Response semantics + +- A match is a direct `members` entry in a tagged snapshot. +- The query is by STIX ID and spans all revisions; every row reports the exact + pinned `object_modified` revision. +- Candidates, staged entries, quarantine entries, and secondary objects added + only during bundle export are excluded. +- Standard and virtual tracks are included unless filtered. +- Ascending snapshot order exposes first tagged appearance. It is not an audit + record of when the object first entered a draft. +- A valid but unmatched STIX ID returns an empty 200 response. diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index b0c903ce..88a165b0 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -5,6 +5,7 @@ This document provides the complete API reference for Release Tracks V2 (formerly "Collections V2"). **Related Documentation:** + - [summary.md](./summary.md) - High-level design summary and problem statement - [terminology.md](./terminology.md) - Complete terminology guide - [versioning.md](./versioning.md) - Versioning and release process @@ -15,6 +16,7 @@ This document provides the complete API reference for Release Tracks V2 (formerl - [member-sync-strategies.md](../../developer/release-tracks/member-sync-strategies.md) - Automatic tracking of member object revisions **Quick Navigation:** + - [Ephemeral Release Tracks](#ephemeral-release-tracks) - [Release Track Management](#release-track-management) - [Snapshot-Specific Operations](#snapshot-specific-operations) @@ -28,17 +30,19 @@ This document provides the complete API reference for Release Tracks V2 (formerl - [Output Formats](#output-formats) - [Error Responses](#error-responses) - ## Complete Endpoint List ### Ephemeral Release Tracks + ``` GET /api/release-tracks/ephemeral/:domain ``` ### Release Track Management + ``` GET /api/release-tracks +GET /api/release-tracks/objects/:objectRef/releases POST /api/release-tracks/new POST /api/release-tracks/new-from-bundle POST /api/release-tracks/import @@ -51,6 +55,7 @@ DELETE /api/release-tracks/:id ``` ### Snapshot Operations + ``` GET /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/meta @@ -60,6 +65,7 @@ DELETE /api/release-tracks/:id/snapshots/:modified ``` ### Candidate Management + ``` POST /api/release-tracks/:id/candidates GET /api/release-tracks/:id/candidates @@ -70,28 +76,33 @@ POST /api/release-tracks/:id/candidates/:objectRef/update-version ``` ### Staged Objects + ``` GET /api/release-tracks/:id/staged POST /api/release-tracks/:id/staged/demote ``` ### Configuration + ``` GET /api/release-tracks/:id/config PUT /api/release-tracks/:id/config ``` ### Preview & Dry Run + ``` GET /api/release-tracks/:id/bump/preview ``` ### Version Management + ``` GET /api/release-tracks/:id/objects/:objectRef/versions ``` ### Virtual Release Tracks (Additional) + ``` PUT /api/release-tracks/:id/composition POST /api/release-tracks/:id/snapshots/create @@ -105,11 +116,12 @@ GET /api/release-tracks/:id/snapshots/preview "Ephemeral" release tracks refer to unmanaged, stateless release track snapshots. Upon request, a STIX bundle will be generated containing the latest copy of all objects contained within the respective domain as defined by the `:domain` path parameter. Three options are supported in the `:domain` path parameter: + - `enterprise` - `ics` - `mobile` -These refer to all objects delineated by ATT&CK domain membership as reflected by the objects' `x_mitre_domains` property. +These refer to all objects delineated by ATT&CK domain membership as reflected by the objects' `x_mitre_domains` property. ### Get Ephemeral Bundle @@ -125,18 +137,19 @@ identities and marking definitions are included so the bundle is self-contained. **Path Parameters:** + - `:domain` - `enterprise` | `ics` | `mobile` **Query Parameters:** -| Parameter | Values | Default | Description | -|-----------|--------|---------|-------------| -| `format` | `bundle` \| `workbench` \| `filesystemstore` | `bundle` | Output format (`filesystemstore` is not yet implemented) | -| `stixVersion` | `2.0` \| `2.1` | `2.1` | STIX version the emitted bundle conforms to (bundle format only) | -| `includeToc` | `true` \| `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) in the bundle. The TOC is generated with `x_mitre_version: "0.1"` (signifying an ephemeral, non-release-track collection), a `modified` of the current timestamp, and the deployment's default ATT&CK spec version. | -| `includeObjectsWithMissingAttackId` | `true` \| `false` | `false` | Include objects that should have an ATT&CK ID set but do not | -| `includeDeprecated` | `true` \| `false` | `false` | Include objects with `x_mitre_deprecated: true` (this also governs deprecated Data Sources) | -| `includeRevoked` | `true` \| `false` | `false` | Include objects with `revoked: true` | +| Parameter | Values | Default | Description | +| ----------------------------------- | -------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `format` | `bundle` \| `workbench` \| `filesystemstore` | `bundle` | Output format (`filesystemstore` is not yet implemented) | +| `stixVersion` | `2.0` \| `2.1` | `2.1` | STIX version the emitted bundle conforms to (bundle format only) | +| `includeToc` | `true` \| `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) in the bundle. The TOC is generated with `x_mitre_version: "0.1"` (signifying an ephemeral, non-release-track collection), a `modified` of the current timestamp, and the deployment's default ATT&CK spec version. | +| `includeObjectsWithMissingAttackId` | `true` \| `false` | `false` | Include objects that should have an ATT&CK ID set but do not | +| `includeDeprecated` | `true` \| `false` | `false` | Include objects with `x_mitre_deprecated: true` (this also governs deprecated Data Sources) | +| `includeRevoked` | `true` \| `false` | `false` | Include objects with `revoked: true` | > [!Note] > The ephemeral endpoint does not support the `include` or `state` tier @@ -156,12 +169,14 @@ GET /api/release-tracks ``` **Query Parameters:** + - `releases` - `only` (filter to show only release tracks that have at least one tagged release) - `type` - `standard` | `virtual` (filter by track type) - `limit` - Number of results (pagination) - `offset` - Pagination offset **Response Example:** + ```json { "release_tracks": [ @@ -174,6 +189,14 @@ GET /api/release-tracks "latest_modified": "2024-01-15T16:20:00Z", "snapshot_count": 47, "tagged_release_count": 12, + "tagged_releases": [ + { + "snapshot_modified": "2024-01-15T16:20:00Z", + "version": "14.1", + "tagged_at": "2024-01-15T17:00:00Z", + "tagged_by": "user-id" + } + ], "summary": { "members_count": 3247, "staged_count": 18, @@ -209,6 +232,7 @@ POST /api/release-tracks/new ``` **Request Body:** + ```json { "name": "Release Track Name", @@ -227,6 +251,7 @@ POST /api/release-tracks/new-from-bundle ``` **Request Body:** + ```json { "type": "bundle", @@ -247,6 +272,7 @@ POST /api/release-tracks/new-from-bundle ``` **Response:** + ```json { "release_track_id": "release-track--new-uuid", @@ -292,22 +318,22 @@ Workbench responses return the release-track snapshot shape. Entries in the `mem **Query Parameters:** -| Parameter | Values | Description | -|-----------|--------|-------------| -| `format` | `workbench` \| `bundle` \| `filesystemstore` | Output format (default: `workbench`; `filesystemstore` is not yet implemented) | -| `include` | `members` \| `staged` \| `candidates` \| `quarantine` \| `all` | Which tier arrays to include in `workbench` responses (default: all tiers) | -| `releases` | `only` | Return only the latest tagged release instead of latest snapshot | -| `version` | `X.Y` | Return specific version (e.g., `14.1`) | -| `versions` | `all` | List all snapshots with metadata | +| Parameter | Values | Description | +| ---------- | -------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `format` | `workbench` \| `bundle` \| `filesystemstore` | Output format (default: `workbench`; `filesystemstore` is not yet implemented) | +| `include` | `members` \| `staged` \| `candidates` \| `quarantine` \| `all` | Which tier arrays to include in `workbench` responses (default: all tiers) | +| `releases` | `only` | Return only the latest tagged release instead of latest snapshot | +| `version` | `X.Y` | Return specific version (e.g., `14.1`) | +| `versions` | `all` | List all snapshots with metadata | **Additional query parameters for `format=bundle`:** -| Parameter | Values | Description | -|-----------|--------|-------------| -| `include` | `staged` and/or `candidates` (comma-separated or repeated) | Additional tiers to include in the bundle alongside members. If omitted, only members are included. (Note the different semantics from `workbench` responses.) | -| `state` | `work-in-progress` and/or `awaiting-review` (comma-separated or repeated) | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included. Members are unaffected. | -| `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`) | -| `includeToc` | `true` \| `false` | Include a table-of-contents object (of type `x-mitre-collection`) derived from the release-track metadata (default: `true`) | +| Parameter | Values | Description | +| ------------- | ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `include` | `staged` and/or `candidates` (comma-separated or repeated) | Additional tiers to include in the bundle alongside members. If omitted, only members are included. (Note the different semantics from `workbench` responses.) | +| `state` | `work-in-progress` and/or `awaiting-review` (comma-separated or repeated) | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included. Members are unaffected. | +| `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`) | +| `includeToc` | `true` \| `false` | Include a table-of-contents object (of type `x-mitre-collection`) derived from the release-track metadata (default: `true`) | See [Output Formats](output-formats.md) for details on the bundle structure. @@ -342,8 +368,10 @@ GET /api/release-tracks/:id?versions=all ### Update Metadata A user or team may wish to: + - rename a release (e.g., fix a typo like `"Entrprise"` to `"Enterprise"`) or shift the scope/purpose of an existing release track without losing its history (though [cloning](#clone-latest-snapshot) is preferred in this scenario) - update metadata (which at present consists of a `description` field, `object_marking_references` (typically only includes the global marking definition) and the author (`created_by_ref`). + ``` POST /api/release-tracks/:id/meta ``` @@ -351,6 +379,7 @@ POST /api/release-tracks/:id/meta Creates new snapshot with updated metadata. **Request Body:** + ```json { "name": "Updated Name", @@ -366,9 +395,10 @@ Creates new snapshot with updated metadata. POST /api/release-tracks/:id/contents ``` -Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** The main workflow for enrolling new member objects into `x_mitre_contents` is through the candidate-staging promotion cycle described in [versioning.md](./versioning.md). +Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** The main workflow for enrolling new member objects into `x_mitre_contents` is through the candidate-staging promotion cycle described in [versioning.md](./versioning.md). **Request Body:** + ```json { "x_mitre_contents": ["attack-pattern--uuid1", "malware--uuid2"] @@ -389,6 +419,7 @@ POST /api/release-tracks/:id/bump ``` **Request Body (optional):** + ```json { "type": "major" | "minor", // Defaults to "minor" if omitted @@ -400,11 +431,13 @@ POST /api/release-tracks/:id/bump ### Clone Release Track From Latest Bootstraps a new `release-track` instance from an existing snapshot. + ``` POST /api/release-tracks/:id/clone ``` **Request Body:** + ```json { "name": "Cloned Release Track" // optional @@ -418,6 +451,7 @@ DELETE /api/release-tracks/:id ``` **Query Parameters:** + - `versions` - `latest` (delete only latest, default: all) --- @@ -435,9 +469,11 @@ GET /api/release-tracks/:id/snapshots/:modified ``` **Path Parameters:** + - `:modified` - ISO 8601 timestamp (e.g., `2024-01-15T16:20:00.000Z`) **Query Parameters:** + - `format` - `workbench` | `bundle` | `filesystemstore` (default: `workbench`; `filesystemstore` is not yet implemented) - `include` - `members` | `staged` | `candidates` | `quarantine` | `all` (default: all tiers) @@ -446,6 +482,7 @@ For `format=bundle`, the same additional parameters as semantics), `state`, `stixVersion`, and `includeToc`. **Example:** + ```bash # Get snapshot from January 15, 2024 for the Workbench UI GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z @@ -490,6 +527,7 @@ POST /api/release-tracks/:id/snapshots/:modified/bump ### Clone Specific Snapshot Bootstraps a new release track from the specified snapshot. + ``` POST /api/release-tracks/:id/snapshots/:modified/clone ``` @@ -497,6 +535,7 @@ POST /api/release-tracks/:id/snapshots/:modified/clone ### Delete Specific Snapshot **TODO**: further consideration needs to be given here. We need to be careful to avoid breaking contextual continuity between snapshots. + ``` DELETE /api/release-tracks/:id/snapshots/:modified ``` @@ -514,16 +553,18 @@ POST /api/release-tracks/:id/candidates ``` **Request Body:** + ```json { "object_refs": [ - {"id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z"}, // pinned to specific version - {"id": "malware--uuid"} // follows latest version while marked as candidate + { "id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z" }, // pinned to specific version + { "id": "malware--uuid" } // follows latest version while marked as candidate ] } ``` Simplified (uses latest versions): + ```json { "object_refs": ["attack-pattern--uuid", "malware--uuid"] @@ -539,9 +580,11 @@ GET /api/release-tracks/:id/candidates ``` **Query Parameters:** + - `status` - Filter by workflow status: `work-in-progress` | `awaiting-review` | `reviewed` **Response Example:** + ```json { "candidates": [ @@ -571,31 +614,33 @@ GET /api/release-tracks/:id/candidates ### Remove Candidate Remove an object from the latest snapshot's candidates list (`workspace.candidates`). + ``` DELETE /api/release-tracks/:id/candidates/:objectRef ``` ### Bulk Object Status Transition -Bulk transition candidate objects currently in the latest snapshot from workflow status `from` to workflow status `to`. -- Optionally target specific candidates using the `object_refs` filter. +Bulk transition candidate objects currently in the latest snapshot from workflow status `from` to workflow status `to`. + +- Optionally target specific candidates using the `object_refs` filter. - `object_refs` is optional; if omitted, transitions all matching `from` status. Bidirectional status transition is supported here. For example, objects can be transition from "reviewed" → "awaiting-review" or from "awaiting-review" → "work-in-progress". Notably, changes to an object's status (e.g., "work-in-progress" → "awaiting-review") will automatically update its release track membership standing (e.g., candidate, staged, member). In the most restrictive (typical) scenario, a candidate object transitioning to the "reviewed" state will trigger a new draft snapshot creation wherein the object is now staged. + ``` POST /api/release-tracks/:id/candidates/review ``` **Request Body:** + ```json { "from": "work-in-progress", "to": "awaiting-review", - "object_refs": [ - {"id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z"} - ] + "object_refs": [{ "id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z" }] } ``` @@ -612,6 +657,7 @@ GET /api/release-tracks/:id/staged ``` **Response Example:** + ```json { "staged": [ @@ -636,6 +682,7 @@ POST /api/release-tracks/:id/candidates/promote ``` **Request Body:** + ```json { "object_refs": ["attack-pattern--eee"] @@ -643,6 +690,7 @@ POST /api/release-tracks/:id/candidates/promote ``` **Response:** + ```json { "promoted": [ @@ -662,11 +710,10 @@ POST /api/release-tracks/:id/staged/demote ``` **Request Body:** + ```json { - "object_refs": [ - {"id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z"} - ] + "object_refs": [{ "id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z" }] } ``` @@ -687,6 +734,7 @@ PUT /api/release-tracks/:id/config ``` **Request Body:** + ```json { "candidacy_threshold": "work-in-progress" | "awaiting-review" | "reviewed", @@ -698,7 +746,7 @@ PUT /api/release-tracks/:id/config ## Preview & Dry Run -> **Note on `include` Query Parameter:** The `include` query parameter (used on snapshot retrieval endpoints to filter which tiers are returned) is **NOT supported** on bump preview or dry-run operations. Bump previews and dry-runs are intended to show the user exactly what *will* happen when a bump occurs; ad-hoc filters would be misleading because they do not affect the actual release outcome. +> **Note on `include` Query Parameter:** The `include` query parameter (used on snapshot retrieval endpoints to filter which tiers are returned) is **NOT supported** on bump preview or dry-run operations. Bump previews and dry-runs are intended to show the user exactly what _will_ happen when a bump occurs; ad-hoc filters would be misleading because they do not affect the actual release outcome. ### Preview Next Release (Read-Only) @@ -709,9 +757,11 @@ GET /api/release-tracks/:id/bump/preview ``` **Query Parameters:** + - `format` - `bundle` | `filesystemstore` | `workbench` (default: `workbench`; `filesystemstore` is not yet implemented) **Response Example:** + ```json { "current_version": "1.1", @@ -746,6 +796,7 @@ POST /api/release-tracks/:id/bump ``` **Request Body:** + ```json { "type": "minor", @@ -768,6 +819,7 @@ POST /api/release-tracks/:id/candidates/:objectRef/update-version ``` **Request Body:** + ```json { "old_modified": "2024-01-15T10:00:00Z", @@ -776,6 +828,7 @@ POST /api/release-tracks/:id/candidates/:objectRef/update-version ``` **Use Cases:** + - Upgrading a candidate to the latest version of an object - Downgrading to a previous stable version - Synchronizing with another release track's version @@ -791,6 +844,7 @@ GET /api/release-tracks/:id/objects/:objectRef/versions ``` **Response Example:** + ```json { "object_ref": "attack-pattern--T1234", @@ -809,6 +863,22 @@ GET /api/release-tracks/:id/objects/:objectRef/versions } ``` +### List Tagged Releases Containing an Object + +Lists tagged snapshots across all release tracks whose `members` tier directly +contains the supplied STIX ID. The result spans all revisions and reports the +exact `object_modified` pin used by each release. + +``` +GET /api/release-tracks/objects/:objectRef/releases +``` + +Optional query parameters are `type=standard|virtual`, `order=asc|desc`, +`limit`, and `offset`. Drafts, candidates, staged/quarantined entries, and +secondary objects added only during bundle export are excluded. See +[Find Tagged Releases Containing an Object](releases-by-object.md) for the +complete response contract and semantics. + --- ## Output Formats @@ -844,6 +914,12 @@ Snapshot already has a version assigned. Invalid version format or not greater than previous versions. +### TaggedSnapshotDeletionError + +**Status:** 409 Conflict + +Tagged snapshots are immutable and cannot be deleted. + ### NotFoundError **Status:** 404 Not Found @@ -857,6 +933,7 @@ Release track not found. Virtual release tracks are computed aggregations of other release tracks. Unlike standard tracks, virtual tracks don't directly manage objects through the candidate → staged → released workflow. Instead, they compose content from multiple "component tracks" based on configurable rules. **Key Characteristics:** + - Compute contents from component standard or virtual tracks - Only reference **tagged snapshots** from component tracks (never drafts) - Create snapshots **manually or on schedule** (never event-driven) @@ -864,6 +941,7 @@ Virtual release tracks are computed aggregations of other release tracks. Unlike - Support **resolution strategies** to control which component versions are included **Resolution Strategies:** + 1. `latest_tagged` - Always use the most recent tagged snapshot from component 2. `specific_version` - Pin to a specific semantic version (e.g., "5.0") 3. `specific_snapshot` - Pin to a specific snapshot by timestamp @@ -877,6 +955,7 @@ POST /api/release-tracks/new ``` **Request Body:** + ```json { "type": "virtual", @@ -912,6 +991,7 @@ PUT /api/release-tracks/:id/composition ``` **Request Body:** + ```json { "component_tracks": [ @@ -937,6 +1017,7 @@ POST /api/release-tracks/:id/snapshots/create ``` **Request Body:** + ```json { "description": "Q1 2024 snapshot" @@ -944,6 +1025,7 @@ POST /api/release-tracks/:id/snapshots/create ``` **Response:** + ```json { "stix": { @@ -979,6 +1061,7 @@ GET /api/release-tracks/:id/snapshots/preview ``` **Response:** + ```json { "preview": { @@ -1012,6 +1095,7 @@ The ephemeral bundle endpoint supports `format`, but not tier `include`, because it does not read from a persisted release-track snapshot. **Include Parameter** (workbench format — controls which tiers are returned): + ``` GET /api/release-tracks/:id # Default: all tiers GET /api/release-tracks/:id?include=members # Members tier only @@ -1023,6 +1107,7 @@ GET /api/release-tracks/:id?include=all # All tiers **Include Parameter** (bundle format — controls which tiers are hydrated into the bundle; members are always included): + ``` GET /api/release-tracks/:id?format=bundle # Members only GET /api/release-tracks/:id?format=bundle&include=staged # Members + staged @@ -1032,12 +1117,14 @@ GET /api/release-tracks/:id?format=bundle&include=candidates,staged # Members + **State Parameter** (bundle format only — narrows the tiers selected via `include` by workflow status; `reviewed` entries are always included): + ``` GET /api/release-tracks/:id?format=bundle&include=candidates&state=work-in-progress GET /api/release-tracks/:id?format=bundle&include=candidates,staged&state=work-in-progress,awaiting-review ``` **Format Parameter** (controls output format): + ``` GET /api/release-tracks/:id?format=workbench # Workbench snapshot with metadata (default) GET /api/release-tracks/:id?format=bundle # Standard STIX bundle @@ -1045,6 +1132,7 @@ GET /api/release-tracks/:id?format=filesystemstore # Not implemented; return ``` **Combined Example:** + ``` GET /api/release-tracks/:id?include=all&format=workbench ``` @@ -1056,4 +1144,4 @@ The `include` query parameter is **NOT supported** on bump preview or dry-run en - `GET /api/release-tracks/:id/bump/preview` — only `format` is supported - `POST /api/release-tracks/:id/bump` with `dry_run: true` — only `format` is supported (via request body) -These endpoints are designed to show exactly what *will* happen during a release bump. Allowing ad-hoc tier filters would be misleading because they do not affect the actual release outcome. +These endpoints are designed to show exactly what _will_ happen during a release bump. Allowing ad-hoc tier filters would be misleading because they do not affect the actual release outcome. diff --git a/docs/user/release-tracks/releases-by-object.md b/docs/user/release-tracks/releases-by-object.md new file mode 100644 index 00000000..cce386e0 --- /dev/null +++ b/docs/user/release-tracks/releases-by-object.md @@ -0,0 +1,67 @@ +# Find Tagged Releases Containing an Object + +Use the releases-by-object endpoint to find every tagged release whose +`members` tier directly contains a STIX object: + +```http +GET /api/release-tracks/objects/{objectRef}/releases +``` + +`objectRef` is the object's STIX ID, such as +`attack-pattern--11111111-1111-4111-8111-111111111111`. The lookup spans all +stored revisions of that STIX ID. Each result identifies the exact revision +that the release pinned. + +## Query parameters + +| Parameter | Values | Default | Meaning | +| --------- | --------------------- | ------- | ------------------------------------------------------------------------------- | +| `type` | `standard`, `virtual` | all | Restrict results to one release-track type | +| `order` | `asc`, `desc` | `asc` | Sort by snapshot `modified` time; ascending shows lineage from oldest to newest | +| `limit` | positive integer | `50` | Maximum results to return | +| `offset` | non-negative integer | `0` | Results to skip | + +## Example + +```http +GET /api/release-tracks/objects/attack-pattern--11111111-1111-4111-8111-111111111111/releases?order=asc +``` + +```json +{ + "object_ref": "attack-pattern--11111111-1111-4111-8111-111111111111", + "data": [ + { + "track_id": "release-track--22222222-2222-4222-8222-222222222222", + "track_type": "standard", + "track_name": "Enterprise ATT&CK", + "version": "18.0", + "snapshot_modified": "2025-10-01T15:00:00.000Z", + "tagged_at": "2025-10-03T17:12:00.000Z", + "tagged_by": "user-id", + "object_modified": "2025-09-22T14:30:00.000Z" + } + ], + "pagination": { + "total": 1, + "limit": 50, + "offset": 0 + } +} +``` + +## What counts as an appearance + +Only direct membership in a tagged snapshot is returned. The endpoint does +not report: + +- draft snapshots; +- candidates, staged objects, or quarantined objects; +- secondary objects that appear only because bundle export expands a + release's direct members. + +The oldest result is the object's first known _tagged_ appearance. It does not +identify when the object first entered an untagged working draft. + +A syntactically valid STIX ID with no matching releases returns `200 OK` with +an empty `data` array. A malformed STIX ID returns `400 Bad Request`. diff --git a/migrations/20260716000000-backfill-release-track-tagged-releases.js b/migrations/20260716000000-backfill-release-track-tagged-releases.js new file mode 100644 index 00000000..43afa32b --- /dev/null +++ b/migrations/20260716000000-backfill-release-track-tagged-releases.js @@ -0,0 +1,130 @@ +'use strict'; + +/** + * Backfill the compact tagged-release catalogue in releaseTrackRegistry and + * create the tagged-members lookup index in every dynamic track collection. + * + * Dynamic snapshot collections remain authoritative. The registry projection + * is rebuilt rather than incrementally patched, making this migration safe to + * rerun and useful as a repair operation. + */ + +const INDEX_NAME = 'tagged_members_object_ref'; +const CONCURRENCY = 8; + +function compareVersions(left, right) { + const [leftMajor, leftMinor] = left.split('.').map(Number); + const [rightMajor, rightMinor] = right.split('.').map(Number); + if (leftMajor !== rightMajor) return leftMajor - rightMajor; + return leftMinor - rightMinor; +} + +function sameInstant(left, right) { + return new Date(left).getTime() === new Date(right).getTime(); +} + +function taggedReleaseFromSnapshot(snapshot) { + const historyEntry = (snapshot.version_history || []).find( + (entry) => + entry.version === snapshot.version && sameInstant(entry.snapshot_id, snapshot.modified), + ); + + return { + snapshot_modified: snapshot.modified, + version: snapshot.version, + tagged_at: historyEntry?.tagged_at || snapshot.modified, + tagged_by: historyEntry?.tagged_by || 'system', + }; +} + +async function mapWithConcurrency(items, mapper) { + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + await mapper(items[index]); + } + } + + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, () => worker())); +} + +module.exports = { + async up(db) { + const registry = db.collection('releaseTrackRegistry'); + const tracks = await registry.find({}).project({ track_id: 1 }).toArray(); + let taggedReleaseCount = 0; + + await mapWithConcurrency(tracks, async (track) => { + const collectionExists = await db + .listCollections({ name: track.track_id }, { nameOnly: true }) + .hasNext(); + if (!collectionExists) return; + + const snapshots = await db + .collection(track.track_id) + .find( + { version: { $type: 'string' } }, + { projection: { modified: 1, version: 1, version_history: 1 } }, + ) + .sort({ modified: 1 }) + .toArray(); + const taggedReleases = snapshots.map(taggedReleaseFromSnapshot); + const latestTaggedVersion = taggedReleases.reduce( + (highest, release) => + !highest || compareVersions(release.version, highest) > 0 ? release.version : highest, + null, + ); + + await registry.updateOne( + { track_id: track.track_id }, + { + $set: { + tagged_releases: taggedReleases, + tagged_release_count: taggedReleases.length, + latest_tagged_version: latestTaggedVersion, + updated_at: new Date(), + }, + }, + ); + + await db.collection(track.track_id).createIndex( + { 'members.object_ref': 1, modified: -1 }, + { + name: INDEX_NAME, + partialFilterExpression: { version: { $type: 'string' } }, + }, + ); + taggedReleaseCount += taggedReleases.length; + }); + + console.log( + `Backfilled ${taggedReleaseCount} tagged release reference(s) across ${tracks.length} track(s)`, + ); + }, + + async down(db) { + const registry = db.collection('releaseTrackRegistry'); + const tracks = await registry.find({}).project({ track_id: 1 }).toArray(); + + await registry.updateMany({}, { $unset: { tagged_releases: '' } }); + + await mapWithConcurrency(tracks, async (track) => { + const collectionExists = await db + .listCollections({ name: track.track_id }, { nameOnly: true }) + .hasNext(); + if (!collectionExists) return; + + const indexes = await db.collection(track.track_id).indexes(); + if (indexes.some((index) => index.name === INDEX_NAME)) { + await db.collection(track.track_id).dropIndex(INDEX_NAME); + } + }); + }, + + _private: { + compareVersions, + taggedReleaseFromSnapshot, + }, +}; From af585963e8423d80b431b48908856441e5436da9 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:20:57 -0400 Subject: [PATCH 16/55] fix(release-tracks): enforce cross-tier revision uniqueness Normalize exact STIX revision pins across release-track tiers and make exact transitions idempotent. Repair legacy duplicate state during mutations and tagging, with regression and documentation coverage. --- .../paths/release-tracks-paths.yml | 14 +- app/lib/release-tracks/conflict-resolution.js | 9 + .../release-tracks/tier-revision-invariant.js | 89 +++++ .../release-tracks/snapshot-service.js | 20 +- .../release-tracks/standard-track-service.js | 33 +- .../release-tracks/versioning-service.js | 29 +- .../release-tracks-tier-invariant.spec.js | 306 ++++++++++++++++++ docs/developer/TODO.md | 24 +- .../release-tracks/backref-reconciliation.md | 6 +- docs/developer/release-tracks/entities.md | 2 + .../release-tracks/implementation-notes.md | 27 +- docs/user/release-tracks/api-reference.md | 20 +- docs/user/release-tracks/release-workflow.md | 20 +- 13 files changed, 572 insertions(+), 27 deletions(-) create mode 100644 app/lib/release-tracks/tier-revision-invariant.js create mode 100644 app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 39d5bb5c..9e6739c0 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -413,6 +413,8 @@ paths: operationId: 'release-tracks-update-contents-latest' description: | Replace the members tier with new contents (x_mitre_contents format). + Exact revisions already present in another tier are retained only in members; + different revisions of the same object remain valid across tiers. Creates a new snapshot clone. Request body validated via Zod in controller. tags: @@ -453,6 +455,7 @@ paths: description: | Tag the latest snapshot with a version number. For standard tracks: promotes staged → members. + Exact staged/member duplicates are idempotent and normalized rather than treated as conflicts. For virtual tracks: N/A (already resolved). Request body validated via Zod in controller: { type: 'major'|'minor', version?: string, dry_run?: boolean } tags: @@ -549,6 +552,8 @@ paths: description: | Add one or more objects to the candidates tier. If modified is omitted or 'latest', resolves to the latest version of the object. + If that exact revision is already pinned in any tier, the add is idempotently skipped. + Different revisions of the same object may occupy different tiers. If auto_promote is enabled and candidates meet the threshold, they are auto-promoted to staged. Request body validated via Zod: { object_refs: Array } tags: @@ -570,6 +575,7 @@ paths: description: | Transition candidates from one workflow status to another (forward-only). If auto_promote is enabled and candidates meet the threshold after transition, they are auto-promoted to staged. + Tier changes retain an exact revision in only one tier and repair legacy cross-tier duplicates. `from` also accepts the server-assigned `modified-in-place` status; `to` accepts only the user-settable statuses (work-in-progress, awaiting-review, reviewed). Request body validated via Zod: { from, to, object_refs? } @@ -591,7 +597,8 @@ paths: operationId: 'release-tracks-candidates-promote' description: | Manually promote specific candidates to the staged tier, bypassing auto-promotion logic. - Applies conflict resolution policy. + Applies conflict resolution policy only to different revisions of the same object. + Exact revisions are retained in one tier, with members taking precedence. Request body validated via Zod: { object_refs: string[] } tags: - 'Release Tracks' @@ -637,6 +644,7 @@ paths: operationId: 'release-tracks-candidates-update-version' description: | Change which version of an object is being tracked in the candidates tier. + If the new pin exactly matches another tier, the authoritative existing tier is retained. Request body validated via Zod: { old_modified, new_modified } tags: - 'Release Tracks' @@ -691,6 +699,7 @@ paths: operationId: 'release-tracks-staged-demote' description: | Move objects from staged tier back to candidates tier. + Applies into_candidates conflicts only to different revisions; exact revisions remain in one tier. Request body validated via Zod: { object_refs: Array<{id, modified}> } tags: - 'Release Tracks' @@ -1011,6 +1020,8 @@ paths: operationId: 'release-tracks-update-contents-by-modified' description: | Update member contents on a historical snapshot. + Exact revisions already present in another tier are retained only in members; + different revisions of the same object remain valid across tiers. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -1059,6 +1070,7 @@ paths: operationId: 'release-tracks-bump-by-modified' description: | Tag a historical snapshot with a version number. + Exact staged/member duplicates are idempotent and normalized rather than treated as conflicts. Request body validated via Zod in controller. tags: - 'Release Tracks' diff --git a/app/lib/release-tracks/conflict-resolution.js b/app/lib/release-tracks/conflict-resolution.js index 5c8b3f9d..abb426e8 100644 --- a/app/lib/release-tracks/conflict-resolution.js +++ b/app/lib/release-tracks/conflict-resolution.js @@ -15,6 +15,7 @@ // ============================================================================= const { ReleaseConflictError } = require('../../exceptions'); +const { sameRevision } = require('./tier-revision-invariant'); /** * Merge incoming entries into an existing tier, applying a conflict policy. @@ -31,6 +32,14 @@ exports.applyConflictPolicy = function applyConflictPolicy(existingTier, incomin const conflicts = []; // Collect all conflicts for 'abort' policy for (const incoming of incomingEntries) { + const exactDuplicate = merged.some((entry) => sameRevision(entry, incoming)); + if (exactDuplicate) { + // The destination already contains this precise revision. Treat the + // move as successful/idempotent so callers remove it from the source + // tier instead of putting it back as a rejected conflict. + continue; + } + const conflictIdx = merged.findIndex((e) => e.object_ref === incoming.object_ref); if (conflictIdx === -1) { diff --git a/app/lib/release-tracks/tier-revision-invariant.js b/app/lib/release-tracks/tier-revision-invariant.js new file mode 100644 index 00000000..c98225ad --- /dev/null +++ b/app/lib/release-tracks/tier-revision-invariant.js @@ -0,0 +1,89 @@ +'use strict'; + +// A released/member pin is authoritative over workflow and quarantine pins. +// This order also matches backref reconciliation's long-standing defensive +// "first tier wins" behavior. +const TIER_PRECEDENCE = ['members', 'staged', 'candidates', 'quarantine']; + +function modifiedKey(value) { + const timestamp = new Date(value).getTime(); + return Number.isNaN(timestamp) ? String(value) : String(timestamp); +} + +/** + * Build the identity key for a pinned STIX revision. + * + * @param {Object} entry + * @returns {string} + */ +function revisionKey(entry) { + return `${entry.object_ref}\u0000${modifiedKey(entry.object_modified)}`; +} + +/** + * Compare two tier entries by their pinned STIX revision. + * + * @param {Object} left + * @param {Object} right + * @returns {boolean} + */ +function sameRevision(left, right) { + return revisionKey(left) === revisionKey(right); +} + +/** + * Remove exact revision duplicates that occur in different snapshot tiers. + * + * Different revisions of one object remain valid across tiers. Duplicate + * entries within one tier are left intact because quarantine entries can + * intentionally retain per-source provenance. + * + * @param {Object} snapshot + * @returns {{snapshot: Object, removed: Array, changedTiers: Set}} + */ +function normalizeSnapshot(snapshot) { + const normalized = { ...snapshot }; + const firstTierByRevision = new Map(); + const removed = []; + const changedTiers = new Set(); + + for (const tier of TIER_PRECEDENCE) { + const entries = snapshot[tier]; + if (!Array.isArray(entries)) continue; + + const kept = []; + for (const entry of entries) { + const key = revisionKey(entry); + const incumbentTier = firstTierByRevision.get(key); + + if (incumbentTier && incumbentTier !== tier) { + removed.push({ + object_ref: entry.object_ref, + object_modified: entry.object_modified, + kept_tier: incumbentTier, + removed_tier: tier, + }); + changedTiers.add(tier); + continue; + } + + if (!incumbentTier) { + firstTierByRevision.set(key, tier); + } + kept.push(entry); + } + + if (changedTiers.has(tier)) { + normalized[tier] = kept; + } + } + + return { snapshot: normalized, removed, changedTiers }; +} + +module.exports = { + TIER_PRECEDENCE, + revisionKey, + sameRevision, + normalizeSnapshot, +}; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index b54aa447..11a76345 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -20,6 +20,7 @@ const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); const versionUtils = require('../../lib/release-tracks/version-utils'); +const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const { TrackNotFoundError, NotFoundError, @@ -251,12 +252,19 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov } } - const saved = await dynamicRepo.saveSnapshot(trackId, clone); + const normalized = tierRevisionInvariant.normalizeSnapshot(clone); + const saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); await syncRegistryCounters(trackId); // The clone (modified = now) is the track's new latest snapshot await emitContentsChanged(trackId, saved); + if (normalized.removed.length > 0) { + logger.warn( + `SnapshotService: Removed ${normalized.removed.length} exact cross-tier revision ` + + `duplicate(s) while cloning track "${trackId}"`, + ); + } logger.verbose(`SnapshotService: Cloned snapshot for track "${trackId}"`); return saved; }; @@ -306,8 +314,10 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { clone.created_by_ref = options.userAccountId || sourceSnapshot.created_by_ref; clone.version_history = []; + const normalized = tierRevisionInvariant.normalizeSnapshot(clone); + await modelFactory.ensureIndexes(newTrackId); - const saved = await dynamicRepo.saveSnapshot(newTrackId, clone); + const saved = await dynamicRepo.saveSnapshot(newTrackId, normalized.snapshot); await registryRepo.create({ track_id: newTrackId, @@ -324,6 +334,12 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { // The new track's initial snapshot carries the source track's contents await emitContentsChanged(newTrackId, saved); + if (normalized.removed.length > 0) { + logger.warn( + `SnapshotService: Removed ${normalized.removed.length} exact cross-tier revision ` + + `duplicate(s) while cloning new track "${newTrackId}"`, + ); + } logger.verbose(`SnapshotService: Cloned track to new track "${clone.name}" (${newTrackId})`); return saved; } diff --git a/app/services/release-tracks/standard-track-service.js b/app/services/release-tracks/standard-track-service.js index 03c2030d..d59ed3e1 100644 --- a/app/services/release-tracks/standard-track-service.js +++ b/app/services/release-tracks/standard-track-service.js @@ -15,6 +15,7 @@ const snapshotService = require('./snapshot-service'); const objectResolver = require('../../lib/release-tracks/object-resolver'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); +const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const logger = require('../../lib/logger'); const { NotFoundError, BadRequestError } = require('../../exceptions'); @@ -74,7 +75,7 @@ function normalizeObjectRef(entry) { * * For each entry: * - If `modified` is "latest" or omitted, resolve via the STIX service layer. - * - Skip duplicates (same object_ref + object_modified already in candidates). + * - Skip duplicates (same object_ref + object_modified already in any tier). * - New candidates start as "work-in-progress". * * @param {string} trackId @@ -85,9 +86,16 @@ function normalizeObjectRef(entry) { exports.addCandidates = async function addCandidates(trackId, objectRefs, userId) { const source = await snapshotService.getLatestSnapshot(trackId); assertStandardTrack(source); + const normalizedSource = tierRevisionInvariant.normalizeSnapshot(source); + const workingSource = normalizedSource.snapshot; const now = new Date(); - const existingCandidates = source.candidates || []; + const existingCandidates = workingSource.candidates || []; + const existingRevisionKeys = new Set( + tierRevisionInvariant.TIER_PRECEDENCE.flatMap((tier) => workingSource[tier] || []).map( + tierRevisionInvariant.revisionKey, + ), + ); const newEntries = []; for (const raw of objectRefs) { @@ -101,25 +109,30 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId modified = new Date(entry.modified); } - // Skip if this exact (object_ref + object_modified) already exists in candidates - const isDuplicate = existingCandidates.some( - (c) => - c.object_ref === entry.id && new Date(c.object_modified).getTime() === modified.getTime(), - ); + const revision = { object_ref: entry.id, object_modified: modified }; + const revisionKey = tierRevisionInvariant.revisionKey(revision); + const isDuplicate = existingRevisionKeys.has(revisionKey); if (isDuplicate) { logger.verbose( - `StandardTrackService: Skipping duplicate candidate ${entry.id} @ ${modified.toISOString()}`, + `StandardTrackService: Skipping already-pinned candidate ${entry.id} @ ` + + modified.toISOString(), ); continue; } newEntries.push({ - object_ref: entry.id, - object_modified: modified, + ...revision, object_status: 'work-in-progress', object_added_at: now, object_added_by: userId, }); + existingRevisionKeys.add(revisionKey); + } + + if (newEntries.length === 0) { + return normalizedSource.removed.length > 0 + ? snapshotService.cloneSnapshot(trackId, source) + : source; } // Same-object conflicts (the object_ref is already pinned in candidates at diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 372cd8ed..e4438328 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -18,6 +18,7 @@ const snapshotService = require('./snapshot-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const versionUtils = require('../../lib/release-tracks/version-utils'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); +const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const releaseHistoryService = require('./release-history-service'); const logger = require('../../lib/logger'); const { AlreadyReleasedError } = require('../../exceptions'); @@ -41,6 +42,9 @@ async function _doBump(trackId, snapshot, options) { throw new AlreadyReleasedError(snapshot.version); } + const normalized = tierRevisionInvariant.normalizeSnapshot(snapshot); + const workingSnapshot = normalized.snapshot; + // A historical draft's embedded version_history can predate newer tags. // Read the track-wide tagged releases so retroactive tagging cannot reuse or // regress a version. @@ -53,8 +57,8 @@ async function _doBump(trackId, snapshot, options) { versionUtils.validateVersionProgression(version, versionHistory); // Promote staged → members (standard tracks only) - const staged = snapshot.staged || []; - const existingMembers = snapshot.members || []; + const staged = workingSnapshot.staged || []; + const existingMembers = workingSnapshot.members || []; let mergedMembers = existingMembers; let promotedCount = 0; @@ -66,9 +70,9 @@ async function _doBump(trackId, snapshot, options) { })); const policy = - (snapshot.config && - snapshot.config.promotion_conflicts && - snapshot.config.promotion_conflicts.staged_to_members) || + (workingSnapshot.config && + workingSnapshot.config.promotion_conflicts && + workingSnapshot.config.promotion_conflicts.staged_to_members) || 'abort'; const { merged } = conflictResolution.applyConflictPolicy( @@ -93,7 +97,7 @@ async function _doBump(trackId, snapshot, options) { members_count: mergedMembers.length, promoted_count: promotedCount, staged_count: staged.length, - candidate_count: (snapshot.candidates || []).length, + candidate_count: (workingSnapshot.candidates || []).length, }, }; @@ -112,6 +116,9 @@ async function _doBump(trackId, snapshot, options) { // Build additional atomic ops for the tag update const additionalOps = {}; + for (const tier of normalized.changedTiers) { + additionalOps[tier] = workingSnapshot[tier]; + } if (staged.length > 0) { additionalOps.members = mergedMembers; additionalOps.staged = []; @@ -145,6 +152,13 @@ async function _doBump(trackId, snapshot, options) { `(promoted ${promotedCount} staged → members)`, ); + if (normalized.removed.length > 0) { + logger.warn( + `VersioningService: Removed ${normalized.removed.length} exact cross-tier revision ` + + `duplicate(s) while tagging track "${trackId}"`, + ); + } + return tagged; } @@ -195,7 +209,8 @@ exports.bumpByModified = async function bumpByModified(trackId, modified, option */ // eslint-disable-next-line no-unused-vars exports.previewBump = async function previewBump(trackId, _format) { - const snapshot = await snapshotService.getLatestSnapshot(trackId); + const sourceSnapshot = await snapshotService.getLatestSnapshot(trackId); + const snapshot = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot).snapshot; // The latest draft may have been cloned before a historical snapshot was // retroactively tagged. Use the authoritative track-wide ledger here for diff --git a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js new file mode 100644 index 00000000..2a4dd59c --- /dev/null +++ b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js @@ -0,0 +1,306 @@ +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const snapshotService = require('../../../services/release-tracks/snapshot-service'); + +const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; +const tiers = ['members', 'staged', 'candidates', 'quarantine']; + +function buildTechnique(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +function memberEntry(object) { + return { + object_ref: object.stix.id, + object_modified: object.stix.modified, + }; +} + +function candidateEntry(object, status = 'work-in-progress') { + return { + ...memberEntry(object), + object_status: status, + object_added_at: new Date(), + object_added_by: 'legacy-state', + }; +} + +function stagedEntry(object, status = 'reviewed') { + return { + ...memberEntry(object), + object_status: status, + object_staged_at: new Date(), + object_staged_by: 'legacy-state', + }; +} + +function occurrences(snapshot, object) { + const modified = new Date(object.stix.modified).getTime(); + return tiers.flatMap((tier) => + (snapshot[tier] || []) + .filter( + (entry) => + entry.object_ref === object.stix.id && + new Date(entry.object_modified).getTime() === modified, + ) + .map(() => tier), + ); +} + +describe('Release-track cross-tier revision uniqueness', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, expectedStatus = 200) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return response.body; + } + + async function put(path, body) { + const response = await request(app) + .put(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return response.body; + } + + async function createTechnique(name, previous) { + return post('/api/techniques', buildTechnique(name, previous), 201); + } + + async function createTrack(name, type = 'standard') { + return post('/api/release-tracks/new', { name, type }, 201); + } + + async function getLatest(trackId) { + const response = await request(app) + .get(`/api/release-tracks/${trackId}`) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return response.body; + } + + async function setMembers(trackId, objects) { + return post(`/api/release-tracks/${trackId}/contents`, { + x_mitre_contents: objects.map((object) => ({ + obj_ref: object.stix.id, + obj_modified: object.stix.modified, + })), + }); + } + + async function useManualMemberSync(trackId) { + return put(`/api/release-tracks/${trackId}/config`, { + member_sync: { strategy: 'manual' }, + }); + } + + async function injectLatestSnapshot(trackId, overrides) { + const source = await snapshotService.getLatestSnapshot(trackId); + return dynamicRepo.updateSnapshot(trackId, source.modified, { $set: overrides }); + } + + it('skips an exact member revision on candidate add but allows a newer revision', async function () { + const revisionA = await createTechnique('Tier Invariant Add'); + const track = await createTrack('Tier Invariant Add Track'); + await useManualMemberSync(track.id); + await setMembers(track.id, [revisionA]); + const revisionB = await createTechnique('Tier Invariant Add v2', revisionA); + const beforeExactAdd = await getLatest(track.id); + + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: revisionA.stix.id, modified: revisionA.stix.modified }], + }); + const afterExactAdd = await getLatest(track.id); + expect(afterExactAdd.modified).toBe(beforeExactAdd.modified); + expect(occurrences(afterExactAdd, revisionA)).toEqual(['members']); + + await injectLatestSnapshot(track.id, { + candidates: [candidateEntry(revisionA)], + }); + const legacySnapshot = await getLatest(track.id); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: revisionA.stix.id, modified: revisionA.stix.modified }], + }); + const repairedSnapshot = await getLatest(track.id); + expect(repairedSnapshot.modified).not.toBe(legacySnapshot.modified); + expect(occurrences(repairedSnapshot, revisionA)).toEqual(['members']); + + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: revisionB.stix.id, modified: revisionB.stix.modified }], + }); + const latest = await getLatest(track.id); + expect(occurrences(latest, revisionA)).toEqual(['members']); + expect(occurrences(latest, revisionB)).toEqual(['candidates']); + }); + + it('repairs a legacy member/candidate duplicate during manual promotion', async function () { + const technique = await createTechnique('Tier Invariant Promote'); + const track = await createTrack('Tier Invariant Promote Track'); + await injectLatestSnapshot(track.id, { + members: [memberEntry(technique)], + candidates: [candidateEntry(technique)], + }); + + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [technique.stix.id], + }); + + expect(occurrences(await getLatest(track.id), technique)).toEqual(['members']); + }); + + it('treats an exact staged/candidate promotion as idempotent under reject policy', async function () { + const technique = await createTechnique('Tier Invariant Exact Promote'); + const track = await createTrack('Tier Invariant Exact Promote Track'); + await put(`/api/release-tracks/${track.id}/config`, { + promotion_conflicts: { candidates_to_staged: 'always_reject' }, + }); + await injectLatestSnapshot(track.id, { + staged: [stagedEntry(technique)], + candidates: [candidateEntry(technique)], + }); + + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [technique.stix.id], + }); + + expect(occurrences(await getLatest(track.id), technique)).toEqual(['staged']); + }); + + it('repairs a legacy member/staged duplicate during demotion', async function () { + const technique = await createTechnique('Tier Invariant Demote'); + const track = await createTrack('Tier Invariant Demote Track'); + await injectLatestSnapshot(track.id, { + members: [memberEntry(technique)], + staged: [stagedEntry(technique)], + }); + + await post(`/api/release-tracks/${track.id}/staged/demote`, { + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + + expect(occurrences(await getLatest(track.id), technique)).toEqual(['members']); + }); + + it('repairs a legacy member/candidate duplicate during a bulk status transition', async function () { + const technique = await createTechnique('Tier Invariant Review'); + const track = await createTrack('Tier Invariant Review Track'); + await injectLatestSnapshot(track.id, { + members: [memberEntry(technique)], + candidates: [candidateEntry(technique)], + }); + + await post(`/api/release-tracks/${track.id}/candidates/review`, { + from: 'work-in-progress', + to: 'reviewed', + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + + expect(occurrences(await getLatest(track.id), technique)).toEqual(['members']); + }); + + it('drops a candidate pin updated to an exact member revision', async function () { + const revisionA = await createTechnique('Tier Invariant Pin'); + const track = await createTrack('Tier Invariant Pin Track'); + await useManualMemberSync(track.id); + await setMembers(track.id, [revisionA]); + const revisionB = await createTechnique('Tier Invariant Pin v2', revisionA); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: revisionB.stix.id, modified: revisionB.stix.modified }], + }); + + await post(`/api/release-tracks/${track.id}/candidates/${revisionA.stix.id}/update-version`, { + old_modified: revisionB.stix.modified, + new_modified: revisionA.stix.modified, + }); + + const latest = await getLatest(track.id); + expect(occurrences(latest, revisionA)).toEqual(['members']); + expect(occurrences(latest, revisionB)).toEqual([]); + }); + + it('tags and repairs an exact staged/member duplicate instead of reporting a conflict', async function () { + const technique = await createTechnique('Tier Invariant Bump'); + const track = await createTrack('Tier Invariant Bump Track'); + await injectLatestSnapshot(track.id, { + members: [memberEntry(technique)], + staged: [stagedEntry(technique)], + candidates: [candidateEntry(technique)], + }); + + const tagged = await post(`/api/release-tracks/${track.id}/bump`, { type: 'minor' }); + + expect(tagged.version).toBe('1.0'); + expect(occurrences(tagged, technique)).toEqual(['members']); + }); + + it('repairs a legacy virtual members/quarantine duplicate on the next mutation', async function () { + const technique = await createTechnique('Tier Invariant Quarantine'); + const track = await createTrack('Tier Invariant Virtual Track', 'virtual'); + await injectLatestSnapshot(track.id, { + members: [memberEntry(technique)], + quarantine: [ + { + ...memberEntry(technique), + source_track_id: track.id, + source_track_name: track.name, + conflict_reason: 'legacy duplicate', + }, + ], + }); + + await post(`/api/release-tracks/${track.id}/meta`, { + description: 'Trigger invariant repair', + }); + + expect(occurrences(await getLatest(track.id), technique)).toEqual(['members']); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 6cdd7b65..a89cd574 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -10,6 +10,22 @@ Residual: rare (≈1 per run under heavy machine load) single-test failures of a different character (a count assertion, a 20s timeout in a pagination GET) still appear occasionally and pass in isolation — likely load-related; keep observing before chasing further. +## Release-track cross-tier revision uniqueness + +- [x] Read the release-track user and developer documentation and identify the + intended exact-revision invariant. +- [x] Trace every standard/virtual tier ingress and transition path. +- [x] Add regression coverage proving one `(stix.id, stix.modified)` revision + cannot occupy multiple tiers while different revisions of one ID can. +- [x] Enforce the invariant for candidate adds, promotions, demotions, bulk + status transitions, release bumps, member sync, and quarantine workflows. +- [x] Update user/developer documentation (and OpenAPI/Bruno only if the API + contract changes). +- [x] Run focused specs and the complete `npm test` suite. The task-specific + and constituent suites pass; repeated aggregate runs each encountered one + unrelated roaming API failure that passed immediately in isolation. +- [x] Review the final diff and propose a conventional commit message. + ## Snapshot Output Format @@ -91,6 +107,10 @@ For release track retrieval requests that include the `format=bundle` query para - `state: ['work-in-progress', 'awaiting-review']`: If specified, the value must be equal to an array of at least one value. Notably, objects marked as `"reviewed"` are always included (by nature of all members being included —— all members are inherently "reviewed"), irrespective of this query parameter. The parameter acts as a union filter that logically combines with `include`. In other words, when `include` and `state` are both specified, `include` is applied first, then `state` is applied to the remaining `include`-filtered subset. (i.e., Of the candidates and/or staged objects that are ready to be included in the emitted bundle, only include the ones that are marked as "work-in-progress", "awaiting-review", or either). - `stixVersion` should be **preserved**. This parameter allows users to control which STIX version is used in the emitted bundle (`2.0` or `2.1`). It defaults to `2.1`. +### Fixing the /bump/preview endpoint + +Currently there exists support for the `format` query parameter on the `GET /api/release-tracks/:id/bump/preview` endpoint. It's not actually functional (has no impact on the response body) and should be removed. + ### In Summary: - [x] Read the existing release track user + developer documentation in `docs/user/release-tracks/` and `docs/developer/release-tracks/`, respectively. @@ -101,7 +121,7 @@ For release track retrieval requests that include the `format=bundle` query para - [x] Ensure that all required logic (query parameters) is/are implemented in the new endpoints as outlined above. - [x] Implement regression tests for the new functionality (`release-tracks-bundle.spec.js`, `ephemeral-bundle.spec.js`) - [x] Update the aforementioned user + developer documentation. The user documentation should simply describe how the behavior _is_ while the developer documentation should described _why_ and _how_, and additionally cover what has been described here: explaining what _was_ and how the functionality has evolved from before the introduction of release tracks to after. (See `docs/developer/release-tracks/bundle-export.md`.) - +- [] Remove support for the `query` parameter on the `GET /api/release-tracks/:id/bump/preview` endpoint ## Bidirectional References @@ -154,7 +174,7 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho ## Get Releases By Object -- [ ] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a +- [X] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a caller can retrieve every tagged snapshot whose `members` tier directly contains the supplied STIX ID, across all object revisions and release tracks. diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index d8ece526..5e36751e 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -71,8 +71,10 @@ are consistent by the time the triggering API call returns. For one `(repository, trackId, snapshot, includeRef)`: 1. **Desired set** — walk the snapshot tiers in order `members`, `staged`, - `candidates`, `quarantine` (first tier wins if a revision somehow appears - twice), keyed by `(object_ref, object_modified)`. Status mapping: + `candidates`, `quarantine`, keyed by `(object_ref, object_modified)`. + Snapshot persistence enforces this exact-revision uniqueness invariant; + first-tier-wins remains a defensive fallback for legacy/directly written + invalid documents. Status mapping: members → `reviewed`; staged/candidates → the entry's `object_status`; quarantine → none. 2. **Current set** — `find({ 'workspace.release_tracks.id': trackId })`, diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index c049ef6f..bf1c5400 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -270,6 +270,8 @@ field semantics): - `workspace.release_tracks` provides reverse lookup for queries like "show me all release tracks containing this object" - Entries reflect each track's **latest** snapshot and are pinned to the specific object revision the tier entry references +- One precise revision (`stix.id` + `stix.modified`) can occupy only one tier + in a snapshot; different revisions of the same object may occupy different tiers - Same object version can have different statuses in different release tracks - Multiple versions of same object can exist, each potentially referenced by different release tracks - The field is server-controlled and maintained by event-driven reconciliation (`release-track::contents-changed`) diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 472bb038..4868e87d 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -15,13 +15,38 @@ db.objects.createIndex({ 'workspace.workflow.status': 1 }); ## Validation Rules -- **Same object version** can only be in one tier per collection (candidates OR staged OR released) +- **Same object version** can only be in one tier per release-track snapshot + (`members`, `staged`, `candidates`, or `quarantine`) - **Different versions** of same object CAN exist in multiple tiers simultaneously - Status transitions must be valid: WIP → Awaiting → Reviewed (no backwards transitions) - Candidacy threshold must be valid enum value - Object version must exist before adding as candidate (validate `stix.id` and `stix.modified` exist) - Version pin (`object_modified`) is immutable once set for a tier entry +### Cross-tier revision enforcement + +`app/lib/release-tracks/tier-revision-invariant.js` owns exact-revision +identity (`object_ref` + normalized `object_modified`) and normalization. +Every clone-based mutation passes through `snapshot-service.cloneSnapshot`; +track cloning uses the same normalizer. Tagging is the one in-place mutation, +so `versioning-service` normalizes before the atomic tag update. This covers +candidate adds, manual/automatic promotion, demotion, status transitions, +candidate pin changes, member sync, direct content replacement, bundle +import, standard/virtual snapshot creation, and release bumps without +route-specific guards. + +Normalization keeps the first occurrence in the authoritative order +`members` → `staged` → `candidates` → `quarantine`. The order matches +backref reconciliation's defensive precedence: published membership wins +over in-flight workflow state, and resolved virtual membership wins over +quarantine. Exact duplicates within one tier are not collapsed because +quarantine entries may retain source-specific provenance. + +`conflict-resolution.applyConflictPolicy` separately treats an exact +destination duplicate as an idempotent successful move. It does not reject +the incoming entry, so callers remove its source-tier occurrence. Conflict +policies remain responsible only for different revisions of one object. + ## Performance Considerations - Bulk operations should use batch updates diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 88a165b0..db0f7a55 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -548,6 +548,11 @@ DELETE /api/release-tracks/:id/snapshots/:modified Adds STIX objects as candidates to the latest draft snapshot. Each object is identified by its `stix.id` field, as well as (optionally) its `stix.modified` field. If `stix.modified` is omitted, the latest permutation of the relevant STIX object will be added. The candidacy reference will follow the latest version of the object until the moment the draft is converted to a release, at which point the reference will become locked to the specific permutation of the object that was considered "latest" at the time the release bump occurred. +If the resolved revision (the same `stix.id` and `stix.modified`) is already +present in any tier of the snapshot, the add is idempotently skipped. A newer +or older revision of an object already in `members` can still be added as a +candidate. + ``` POST /api/release-tracks/:id/candidates ``` @@ -630,9 +635,13 @@ Bidirectional status transition is supported here. For example, objects can be t Notably, changes to an object's status (e.g., "work-in-progress" → "awaiting-review") will automatically update its release track membership standing (e.g., candidate, staged, member). In the most restrictive (typical) scenario, a candidate object transitioning to the "reviewed" state will trigger a new draft snapshot creation wherein the object is now staged. +Tier transitions preserve the exact-revision uniqueness invariant. If legacy +state already contains the same revision in `members` and `candidates`, the +transition repairs the duplicate and retains the `members` occurrence. + ``` POST /api/release-tracks/:id/candidates/review -``` +```/ **Request Body:** @@ -677,6 +686,11 @@ GET /api/release-tracks/:id/staged ### Promote Candidate Objects To Staged +Promotion conflict policies apply when `staged` contains a different revision +of the same object. An exact revision already present in another tier is not a +conflict; the operation retains a single occurrence, with `members` taking +precedence over workflow tiers. + ``` POST /api/release-tracks/:id/candidates/promote ``` @@ -705,6 +719,10 @@ POST /api/release-tracks/:id/candidates/promote ### Demote Staged Objects To Candidates +Demotion follows the same rule: different revisions are handled by +`promotion_conflicts.into_candidates`, while an exact revision is retained in +only one tier. + ``` POST /api/release-tracks/:id/staged/demote ``` diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 3855076a..e409917d 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -280,7 +280,21 @@ config: { } ``` -Exact duplicates (same `stix.id` *and* same `stix.modified`) are never conflicts: re-adding an identical revision to `candidates` is idempotent and simply skipped. +Exact duplicates (same `stix.id` *and* same `stix.modified`) are never +conflicts. A precise revision can occupy only one tier in a release-track +snapshot: + +- Re-adding a revision that is already in `members`, `staged`, `candidates`, + or `quarantine` is idempotent and skipped. +- Moving a revision into a tier that already contains that exact revision + removes the source-tier occurrence and retains the destination occurrence. +- Conflict policies apply only when the same `stix.id` is pinned to + **different** `stix.modified` values. + +Different revisions of the same object remain valid across tiers—for example, +the released revision in `members` and a newer revision in `candidates`. +Snapshots created from legacy invalid state are normalized with the +authoritative tier order `members` → `staged` → `candidates` → `quarantine`. #### Policy Options @@ -432,6 +446,10 @@ POST /api/release-tracks/release-track--123/bump **Why report all conflicts:** When multiple conflicts exist, reporting all of them in a single error response allows editors to address all issues at once, rather than discovering them one at a time through repeated release attempts. This significantly improves the workflow efficiency when dealing with complex release scenarios. +An exact staged/member duplicate does not trigger `abort`: it is the same +revision, not a competing revision. The redundant staged occurrence is +removed when the snapshot is tagged. + #### Configuring Conflict Resolution Policies **Update release track configuration:** From c1d3ddb024ae69166fab33e0190cefd49a063a76 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:45:29 -0400 Subject: [PATCH 17/55] feat(release-tracks): make snapshot retrieval routes explicit Add paginated snapshot history and canonical latest-snapshot endpoints. Remove implicit latest retrieval from the release-track resource path and migrate API consumers, tests, documentation, and Bruno requests. --- .../definitions/components/release-tracks.yml | 70 +++++ app/api/definitions/openapi.yml | 6 + .../paths/release-tracks-paths.yml | 240 +++++++++++------- app/controllers/release-tracks-controller.js | 26 +- .../release-tracks/release-track-schemas.js | 6 + .../release-track-dynamic.repository.js | 53 ++++ app/routes/release-tracks-routes.js | 25 +- .../release-tracks/release-tracks-service.js | 4 + .../release-tracks/snapshot-service.js | 47 ++++ .../release-tracks-bundle.spec.js | 66 +++-- .../release-tracks-change-capture.spec.js | 8 +- .../release-tracks-tier-invariant.spec.js | 2 +- .../api/release-tracks/release-tracks.spec.js | 6 +- .../release-tracks/snapshot-history.spec.js | 231 +++++++++++++++++ docs/developer/TODO.md | 50 +++- .../developer/release-tracks/bundle-export.md | 2 +- .../release-tracks/implementation-notes.md | 27 ++ docs/user/release-tracks/api-reference.md | 125 ++++++--- docs/user/release-tracks/output-formats.md | 28 +- docs/user/release-tracks/release-workflow.md | 4 +- docs/user/release-tracks/summary.md | 2 +- docs/user/release-tracks/virtual-tracks.md | 6 +- 22 files changed, 840 insertions(+), 194 deletions(-) create mode 100644 app/tests/api/release-tracks/snapshot-history.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 4850f618..c80e0f0d 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -78,6 +78,76 @@ components: items: $ref: '#/components/schemas/version-history-entry' + snapshot-summary: + type: object + description: 'Lightweight metadata shared by standard and virtual snapshot summaries' + required: + - id + - type + - modified + - version + - name + - members_count + properties: + id: + type: string + description: 'Release track ID' + type: + type: string + enum: + - standard + - virtual + modified: + type: string + format: date-time + version: + type: string + nullable: true + description: 'Tagged version, or null for an untagged draft' + name: + type: string + description: + type: string + members_count: + type: integer + minimum: 0 + + standard-snapshot-summary: + allOf: + - $ref: '#/components/schemas/snapshot-summary' + - type: object + description: 'Snapshot summary for a standard release track' + required: + - staged_count + - candidates_count + properties: + type: + type: string + enum: + - standard + staged_count: + type: integer + minimum: 0 + candidates_count: + type: integer + minimum: 0 + + virtual-snapshot-summary: + allOf: + - $ref: '#/components/schemas/snapshot-summary' + - type: object + description: 'Snapshot summary for a virtual release track' + required: + - quarantine_count + properties: + type: + type: string + enum: + - virtual + quarantine_count: + type: integer + minimum: 0 + tier-entry: type: object description: 'A reference to a specific version of a STIX object' diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 722384b8..b9996789 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -400,6 +400,12 @@ paths: /api/release-tracks/{id}/snapshots/preview: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1preview' + /api/release-tracks/{id}/snapshots: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots' + + /api/release-tracks/{id}/snapshots/latest: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1latest' + /api/release-tracks/{id}/snapshots/{modified}: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 9e6739c0..d2a3f5a2 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -268,97 +268,9 @@ paths: description: 'Not yet implemented' # ============================================================================= - # Track retrieval and deletion + # Track deletion # ============================================================================= /api/release-tracks/{id}: - get: - summary: 'Get the latest snapshot of a release track' - operationId: 'release-tracks-get-latest' - description: | - Retrieve the most recent snapshot for a release track. - By default returns the Workbench snapshot shape with all tiers present. - Use the include query parameter to narrow tier arrays when desired. - Use format=bundle to export the snapshot as a STIX bundle. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - description: 'Release track ID' - schema: - type: string - example: 'release-track--a1b2c3d4-e5f6-7890-abcd-ef1234567890' - - name: include - in: query - description: | - Format-sensitive tier selector. - For format=workbench (default): a single value controlling which tier arrays - are returned — members | staged | candidates | quarantine | all (default: all). - For format=bundle: a list of additional tiers (staged and/or candidates, - comma-separated or repeated) to include alongside members. If omitted, only - members are included in the bundle. - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string - - name: format - in: query - description: 'Output format. filesystemstore is not yet implemented.' - schema: - type: string - enum: - - bundle - - workbench - - filesystemstore - default: workbench - - name: state - in: query - description: | - Workflow-status filter for the staged/candidate tiers selected via include - (bundle format only). Accepts modified-in-place, work-in-progress and/or - awaiting-review (comma-separated or repeated). Entries marked reviewed are - always included, irrespective of this parameter. Members are unaffected. - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string - - name: stixVersion - in: query - description: | - STIX version that the exported bundle should conform to (bundle format only). - schema: - type: string - enum: - - '2.0' - - '2.1' - default: '2.1' - - name: includeToc - in: query - description: | - Whether to include a table-of-contents object (of type `x-mitre-collection`) - derived from the release-track metadata (bundle format only). - schema: - type: boolean - default: true - responses: - '200': - description: 'Latest snapshot retrieved successfully' - content: - application/json: - schema: - $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' - '404': - description: 'Release track not found' - '501': - description: 'Requested format is not yet implemented' - delete: summary: 'Delete a release track' operationId: 'release-tracks-delete' @@ -876,6 +788,156 @@ paths: # ============================================================================= # Snapshot-specific operations # ============================================================================= + /api/release-tracks/{id}/snapshots: + get: + summary: 'List snapshots for a release track' + operationId: 'release-tracks-snapshots-list' + description: | + Return lightweight summaries of every snapshot in the release track, + ordered by modified timestamp from newest to oldest. Standard snapshot + summaries contain members, staged, and candidates counts. Virtual + snapshot summaries contain members and quarantine counts. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + description: 'Release track ID' + schema: + type: string + - name: tagged + in: query + description: | + Filter by tagged state. true returns snapshots with a version; + false returns untagged drafts. Omit to return both. + schema: + type: boolean + - name: limit + in: query + description: 'Number of snapshot summaries to return' + schema: + type: integer + minimum: 1 + maximum: 200 + default: 50 + - name: offset + in: query + description: 'Number of matching snapshot summaries to skip' + schema: + type: integer + minimum: 0 + default: 0 + responses: + '200': + description: 'Snapshot summaries retrieved successfully' + content: + application/json: + schema: + type: object + required: + - data + - pagination + properties: + data: + type: array + items: + oneOf: + - $ref: '../components/release-tracks.yml#/components/schemas/standard-snapshot-summary' + - $ref: '../components/release-tracks.yml#/components/schemas/virtual-snapshot-summary' + pagination: + type: object + required: + - total + - limit + - offset + properties: + total: + type: integer + limit: + type: integer + offset: + type: integer + '400': + description: 'Invalid filter or pagination parameter' + '404': + description: 'Release track not found' + + /api/release-tracks/{id}/snapshots/latest: + get: + summary: 'Get the latest snapshot of a release track' + operationId: 'release-tracks-snapshot-get-latest' + description: | + Return the most recent full snapshot for a release track. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + description: 'Release track ID' + schema: + type: string + - name: include + in: query + description: | + Format-sensitive tier selector. For workbench responses, selects + members, staged, candidates, quarantine, or all. For bundle + responses, selects staged and/or candidates in addition to members. + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: format + in: query + description: 'Output format. filesystemstore is not yet implemented.' + schema: + type: string + enum: + - bundle + - workbench + - filesystemstore + default: workbench + - name: state + in: query + description: 'Workflow-status filter for bundle staged/candidate tiers' + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: stixVersion + in: query + description: 'STIX version for bundle responses' + schema: + type: string + enum: + - '2.0' + - '2.1' + default: '2.1' + - name: includeToc + in: query + description: 'Include the x-mitre-collection TOC in bundle responses' + schema: + type: boolean + default: true + responses: + '200': + description: 'Latest snapshot retrieved successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' + '404': + description: 'Release track not found' + '501': + description: 'Requested format is not yet implemented' + /api/release-tracks/{id}/snapshots/{modified}: get: summary: 'Get a specific snapshot by modified timestamp' diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 0d1dd346..e121609c 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -27,6 +27,7 @@ const { bundleStateQuerySchema, stixVersionQuerySchema, booleanQuerySchema, + snapshotTaggedQuerySchema, trackTypeQuerySchema, releaseOrderQuerySchema, releaseLimitQuerySchema, @@ -318,7 +319,7 @@ exports.importReleaseTrack = async function importReleaseTrack(_req, _res, next) ); }; -/** GET /api/release-tracks/:id */ +/** GET /api/release-tracks/:id/snapshots/latest */ exports.retrieveLatestSnapshot = async function retrieveLatestSnapshot(req, res, next) { try { const queryOptions = parseSnapshotQueryParams(req.query); @@ -336,6 +337,29 @@ exports.retrieveLatestSnapshot = async function retrieveLatestSnapshot(req, res, } }; +/** GET /api/release-tracks/:id/snapshots */ +exports.listSnapshots = async function listSnapshots(req, res, next) { + try { + const options = { + tagged: parseOptionalQueryStrict( + req.query.tagged, + snapshotTaggedQuerySchema, + undefined, + 'tagged', + ), + limit: parseOptionalQueryStrict(req.query.limit, releaseLimitQuerySchema, 50, 'limit'), + offset: parseOptionalQueryStrict(req.query.offset, releaseOffsetQuerySchema, 0, 'offset'), + }; + + const result = await releaseTracksService.listSnapshots(req.params.id, options); + logger.debug(`Success: Retrieved snapshots for track ${req.params.id}`); + return res.status(200).send(result); + } catch (err) { + logger.error('Failed to retrieve snapshots: ' + err); + return next(err); + } +}; + /** POST /api/release-tracks/:id/meta */ exports.updateMetadataByLatest = async function updateMetadataByLatest(req, res, next) { try { diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index d8e3143c..4c7a58f2 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -189,6 +189,11 @@ const stixVersionQuerySchema = z.enum(['2.0', '2.1']); // OpenAPI validator has already coerced them to booleans. const booleanQuerySchema = z.union([z.boolean(), z.stringbool()]); +const snapshotTaggedQuerySchema = z.union([ + z.boolean(), + z.enum(['true', 'false']).transform((value) => value === 'true'), +]); + const trackTypeQuerySchema = z.enum(['standard', 'virtual']); const releaseOrderQuerySchema = z.enum(['asc', 'desc']); @@ -433,6 +438,7 @@ module.exports = { bundleStateQuerySchema, stixVersionQuerySchema, booleanQuerySchema, + snapshotTaggedQuerySchema, trackTypeQuerySchema, releaseOrderQuerySchema, releaseLimitQuerySchema, diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 9731f3f1..80af68e9 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -172,6 +172,59 @@ class ReleaseTrackDynamicRepository { } } + async getSnapshotSummaries(trackId, options = {}) { + try { + const Model = this._getModel(trackId); + const query = { id: trackId }; + + if (options.tagged === true) { + query.version = { $type: 'string' }; + } else if (options.tagged === false) { + query.version = null; + } + + const totalCount = await Model.countDocuments(query).exec(); + const aggregation = [ + { $match: query }, + { $sort: { modified: -1 } }, + { $skip: options.offset || 0 }, + ]; + + if (options.limit) { + aggregation.push({ $limit: options.limit }); + } + + aggregation.push({ + $project: { + _id: 0, + id: 1, + type: 1, + modified: 1, + version: 1, + name: 1, + description: 1, + members_count: { $size: { $ifNull: ['$members', []] } }, + staged_count: { $size: { $ifNull: ['$staged', []] } }, + candidates_count: { $size: { $ifNull: ['$candidates', []] } }, + quarantine_count: { $size: { $ifNull: ['$quarantine', []] } }, + }, + }); + + const documents = await Model.aggregate(aggregation).exec(); + + return { + data: documents, + pagination: { + total: totalCount, + offset: options.offset || 0, + limit: options.limit || 0, + }, + }; + } catch (err) { + throw new DatabaseError(err); + } + } + async saveSnapshot(trackId, snapshotData) { try { const Model = this._getModel(trackId); diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index d7c16ce6..28ae4899 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -213,9 +213,25 @@ router ); // ============================================================================= -// Virtual track operations (static snapshot sub-paths before :modified param) +// Snapshot collection and static sub-paths (before :modified param) // ============================================================================= +router + .route('/release-tracks/:id/snapshots') + .get( + authn.authenticate, + authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + releaseTracksController.listSnapshots, + ); + +router + .route('/release-tracks/:id/snapshots/latest') + .get( + authn.authenticate, + authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + releaseTracksController.retrieveLatestSnapshot, + ); + router .route('/release-tracks/:id/snapshots/preview') .get( @@ -294,16 +310,11 @@ router ); // ============================================================================= -// Retrieve / delete release track (must be last -- :id is a catch-all param) +// Delete release track (must be last -- :id is a catch-all param) // ============================================================================= router .route('/release-tracks/:id') - .get( - authn.authenticate, - authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), - releaseTracksController.retrieveLatestSnapshot, - ) .delete( authn.authenticate, authz.requireRole(authz.editorOrHigher), diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index db8ed11b..eaac3109 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -178,6 +178,10 @@ exports.createTrackFromBundle = function createTrackFromBundle(bundleData) { return bundleImportService.createTrackFromBundle(bundleData); }; +exports.listSnapshots = function listSnapshots(trackId, options) { + return snapshotService.listSnapshots(trackId, options); +}; + // eslint-disable-next-line no-unused-vars exports.importTrack = async function importTrack(_data) { notImplemented('importTrack'); diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 11a76345..0700b3c5 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -186,6 +186,53 @@ exports.createTrack = async function createTrack(data) { // Snapshot retrieval // ============================================================================= +/** + * List lightweight summaries of a track's snapshots. + * + * Standard summaries expose members/staged/candidates counts. Virtual + * summaries expose members/quarantine counts. + * + * @param {string} trackId + * @param {Object} options - { tagged?, limit, offset } + * @returns {Promise<{data: Object[], pagination: Object}>} + * @throws {TrackNotFoundError} If the release track does not exist + */ +exports.listSnapshots = async function listSnapshots(trackId, options) { + const track = await registryRepo.findByTrackId(trackId); + if (!track) { + throw new TrackNotFoundError(trackId); + } + + const result = await dynamicRepo.getSnapshotSummaries(trackId, options); + return { + ...result, + data: result.data.map((snapshot) => { + const common = { + id: snapshot.id, + type: snapshot.type, + modified: snapshot.modified, + version: snapshot.version, + name: snapshot.name, + description: snapshot.description, + members_count: snapshot.members_count, + }; + + if (snapshot.type === 'virtual') { + return { + ...common, + quarantine_count: snapshot.quarantine_count, + }; + } + + return { + ...common, + staged_count: snapshot.staged_count, + candidates_count: snapshot.candidates_count, + }; + }), + }; +}; + /** * Retrieve the most recent snapshot for a track. * diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index 57d084d7..3a4ce529 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -5,7 +5,7 @@ * Regression tests for the `format=bundle` output format on the snapshot * retrieval endpoints: * - * - GET /api/release-tracks/:id + * - GET /api/release-tracks/:id/snapshots/latest * - GET /api/release-tracks/:id/snapshots/:modified * * Covered behavior: @@ -210,8 +210,8 @@ describe('Release Tracks Bundle Export API', function () { snapshotModified = promoteRes.modified; }); - it('GET /api/release-tracks/:id?format=bundle returns a members-only STIX 2.1 bundle', async function () { - const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle`); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle returns a members-only STIX 2.1 bundle', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle`); expect(bundle.type).toBe('bundle'); expect(bundle.id).toMatch(/^bundle--/); @@ -240,8 +240,8 @@ describe('Release Tracks Bundle Export API', function () { expect(member.workspace).toBeUndefined(); }); - it('GET /api/release-tracks/:id?format=bundle includes a TOC derived from the track metadata', async function () { - const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle`); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle includes a TOC derived from the track metadata', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle`); const toc = bundle.objects[0]; expect(toc.type).toBe('x-mitre-collection'); @@ -262,21 +262,23 @@ describe('Release Tracks Bundle Export API', function () { expect(contentRefs).not.toContain(toc.id); }); - it('GET /api/release-tracks/:id?format=bundle&includeToc=false omits the TOC', async function () { - const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle&includeToc=false`); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&includeToc=false omits the TOC', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, + ); const tocObjects = bundle.objects.filter((o) => o.type === 'x-mitre-collection'); expect(tocObjects.length).toBe(0); }); - it('GET /api/release-tracks/:id?format=bundle converts LinkById tags to markdown citations', async function () { - const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle`); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle converts LinkById tags to markdown citations', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle`); const member = bundle.objects.find((o) => o.id === memberObject.stix.id); expect(member.description).toBe(`See [Linked Technique](${linkedAttackUrl}) for details.`); }); - it('GET /api/release-tracks/:id?format=bundle&include=candidates adds the candidates tier', async function () { + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates adds the candidates tier', async function () { const bundle = await getBundle( - `/api/release-tracks/${trackId}?format=bundle&include=candidates`, + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates`, ); const ids = bundleObjectIds(bundle); @@ -287,8 +289,10 @@ describe('Release Tracks Bundle Export API', function () { expect(ids).not.toContain(stagedObject.stix.id); }); - it('GET /api/release-tracks/:id?format=bundle&include=staged adds the staged tier', async function () { - const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle&include=staged`); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=staged adds the staged tier', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=staged`, + ); const ids = bundleObjectIds(bundle); expect(ids).toContain(memberObject.stix.id); @@ -296,9 +300,9 @@ describe('Release Tracks Bundle Export API', function () { expect(ids).not.toContain(candidateWip.stix.id); }); - it('GET /api/release-tracks/:id?format=bundle&include=candidates,staged adds both tiers', async function () { + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged adds both tiers', async function () { const bundle = await getBundle( - `/api/release-tracks/${trackId}?format=bundle&include=candidates,staged`, + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates,staged`, ); const ids = bundleObjectIds(bundle); @@ -309,9 +313,9 @@ describe('Release Tracks Bundle Export API', function () { expect(ids).toContain(stagedObject.stix.id); }); - it('GET /api/release-tracks/:id?format=bundle accepts singular tier names and repeated params', async function () { + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle accepts singular tier names and repeated params', async function () { const bundle = await getBundle( - `/api/release-tracks/${trackId}?format=bundle&include=candidate&include=staged`, + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidate&include=staged`, ); const ids = bundleObjectIds(bundle); @@ -321,7 +325,7 @@ describe('Release Tracks Bundle Export API', function () { it('state narrows included candidates but reviewed entries are always included', async function () { const bundle = await getBundle( - `/api/release-tracks/${trackId}?format=bundle&include=candidates&state=work-in-progress`, + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates&state=work-in-progress`, ); const ids = bundleObjectIds(bundle); @@ -338,18 +342,20 @@ describe('Release Tracks Bundle Export API', function () { it('state applies to the staged tier as well', async function () { // The staged object retained its work-in-progress status through promotion const withMatchingState = await getBundle( - `/api/release-tracks/${trackId}?format=bundle&include=staged&state=work-in-progress`, + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=staged&state=work-in-progress`, ); expect(bundleObjectIds(withMatchingState)).toContain(stagedObject.stix.id); const withoutMatchingState = await getBundle( - `/api/release-tracks/${trackId}?format=bundle&include=staged&state=awaiting-review`, + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=staged&state=awaiting-review`, ); expect(bundleObjectIds(withoutMatchingState)).not.toContain(stagedObject.stix.id); }); - it('GET /api/release-tracks/:id?format=bundle&stixVersion=2.0 conforms the bundle to STIX 2.0', async function () { - const bundle = await getBundle(`/api/release-tracks/${trackId}?format=bundle&stixVersion=2.0`); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&stixVersion=2.0 conforms the bundle to STIX 2.0', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&stixVersion=2.0`, + ); expect(bundle.spec_version).toBe('2.0'); const member = bundle.objects.find((o) => o.id === memberObject.stix.id); @@ -357,12 +363,18 @@ describe('Release Tracks Bundle Export API', function () { }); it('rejects invalid include, state, and stixVersion values for bundle exports', async function () { - await getBundle(`/api/release-tracks/${trackId}?format=bundle&include=quarantine`, 400); await getBundle( - `/api/release-tracks/${trackId}?format=bundle&include=candidates&state=reviewed`, + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=quarantine`, + 400, + ); + await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates&state=reviewed`, + 400, + ); + await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&stixVersion=3.0`, 400, ); - await getBundle(`/api/release-tracks/${trackId}?format=bundle&stixVersion=3.0`, 400); }); it('GET /api/release-tracks/:id/snapshots/:modified?format=bundle exports a historical snapshot', async function () { @@ -380,8 +392,8 @@ describe('Release Tracks Bundle Export API', function () { expect(ids).toContain(stagedObject.stix.id); }); - it('GET /api/release-tracks/:id (workbench default) is unaffected by bundle parameters', async function () { - const snapshot = await getBundle(`/api/release-tracks/${trackId}`); + it('GET /api/release-tracks/:id/snapshots/latest (workbench default) is unaffected by bundle parameters', async function () { + const snapshot = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest`); expect(snapshot.members).toBeDefined(); expect(snapshot.candidates).toBeDefined(); expect(snapshot.staged).toBeDefined(); diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index 292af399..23d79c6a 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -110,7 +110,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { } async function latestSnapshotModified(trackId) { - const snapshot = await getJson(`/api/release-tracks/${trackId}`); + const snapshot = await getJson(`/api/release-tracks/${trackId}/snapshots/latest`); return snapshot.modified; } @@ -245,7 +245,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { status: 'modified-in-place', }); - const snapshot = await getJson(`/api/release-tracks/${trackId}`); + const snapshot = await getJson(`/api/release-tracks/${trackId}/snapshots/latest`); expect(snapshot.staged).toHaveLength(0); expect(snapshot.candidates).toHaveLength(1); }); @@ -262,7 +262,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { // In a permissive track the fresh candidate auto-promotes immediately await addCandidate(trackId, technique); - let snapshot = await getJson(`/api/release-tracks/${trackId}`); + let snapshot = await getJson(`/api/release-tracks/${trackId}/snapshots/latest`); expect(snapshot.staged).toHaveLength(1); // An in-place edit is marked, but the tier is decided by the workflow @@ -279,7 +279,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { tier: 'staged', status: 'modified-in-place', }); - snapshot = await getJson(`/api/release-tracks/${trackId}`); + snapshot = await getJson(`/api/release-tracks/${trackId}/snapshots/latest`); expect(snapshot.staged).toHaveLength(1); expect(snapshot.staged[0].object_status).toBe('modified-in-place'); expect(snapshot.candidates).toHaveLength(0); diff --git a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js index 2a4dd59c..22e3b4d6 100644 --- a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js +++ b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js @@ -117,7 +117,7 @@ describe('Release-track cross-tier revision uniqueness', function () { async function getLatest(trackId) { const response = await request(app) - .get(`/api/release-tracks/${trackId}`) + .get(`/api/release-tracks/${trackId}/snapshots/latest`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 9b9a4e78..45691556 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -180,7 +180,7 @@ describe('Release Tracks API', function () { }); const latestRes = await request(app) - .get(`/api/release-tracks/${trackId}`) + .get(`/api/release-tracks/${trackId}/snapshots/latest`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200) @@ -227,13 +227,13 @@ describe('Release Tracks API', function () { expectObjectInfo(historicalStaged, stagedObject); await request(app) - .get(`/api/release-tracks/${trackId}?format=snapshot`) + .get(`/api/release-tracks/${trackId}/snapshots/latest?format=snapshot`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(400); await request(app) - .get(`/api/release-tracks/${trackId}?format=filesystemstore`) + .get(`/api/release-tracks/${trackId}/snapshots/latest?format=filesystemstore`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(501); diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js new file mode 100644 index 00000000..7e1f2fe5 --- /dev/null +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -0,0 +1,231 @@ +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); + +const objectRefs = [ + 'attack-pattern--00000000-0000-4000-8000-000000000001', + 'attack-pattern--00000000-0000-4000-8000-000000000002', + 'attack-pattern--00000000-0000-4000-8000-000000000003', + 'attack-pattern--00000000-0000-4000-8000-000000000004', + 'attack-pattern--00000000-0000-4000-8000-000000000005', + 'attack-pattern--00000000-0000-4000-8000-000000000006', +]; + +function memberEntry(index, modified) { + return { + object_ref: objectRefs[index], + object_modified: modified, + }; +} + +function stagedEntry(index, modified) { + return { + ...memberEntry(index, modified), + object_status: 'reviewed', + object_staged_at: modified, + object_staged_by: 'snapshot-history-test', + }; +} + +function candidateEntry(index, modified) { + return { + ...memberEntry(index, modified), + object_status: 'work-in-progress', + object_added_at: modified, + object_added_by: 'snapshot-history-test', + }; +} + +function snapshotBase(snapshot) { + const clone = { ...snapshot }; + delete clone._id; + delete clone.__v; + return clone; +} + +describe('GET /api/release-tracks/:id/snapshots', function () { + let app; + let passportCookie; + let standardTrack; + let virtualTrack; + let standardTaggedModified; + let standardLatestModified; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + + standardTrack = await createTrack('Snapshot History Standard', 'standard'); + virtualTrack = await createTrack('Snapshot History Virtual', 'virtual'); + + const standardCreated = new Date(standardTrack.modified); + standardTaggedModified = new Date(standardCreated.getTime() + 1000); + standardLatestModified = new Date(standardCreated.getTime() + 2000); + + await dynamicRepo.saveSnapshot(standardTrack.id, { + ...snapshotBase(standardTrack), + modified: standardTaggedModified, + version: '1.0', + members: [memberEntry(0, standardTaggedModified), memberEntry(1, standardTaggedModified)], + staged: [stagedEntry(2, standardTaggedModified)], + candidates: [ + candidateEntry(3, standardTaggedModified), + candidateEntry(4, standardTaggedModified), + candidateEntry(5, standardTaggedModified), + ], + }); + await dynamicRepo.saveSnapshot(standardTrack.id, { + ...snapshotBase(standardTrack), + modified: standardLatestModified, + version: null, + members: [memberEntry(0, standardLatestModified)], + staged: [stagedEntry(1, standardLatestModified), stagedEntry(2, standardLatestModified)], + candidates: [candidateEntry(3, standardLatestModified)], + }); + + const virtualCreated = new Date(virtualTrack.modified); + const virtualTaggedModified = new Date(virtualCreated.getTime() + 1000); + await dynamicRepo.saveSnapshot(virtualTrack.id, { + ...snapshotBase(virtualTrack), + modified: virtualTaggedModified, + version: '1.0', + members: [memberEntry(0, virtualTaggedModified), memberEntry(1, virtualTaggedModified)], + quarantine: [ + { + ...memberEntry(2, virtualTaggedModified), + source_track_id: standardTrack.id, + source_track_name: standardTrack.name, + source_snapshot_version: '1.0', + conflict_reason: 'conflicting object revisions', + }, + ], + }); + }); + + async function createTrack(name, type) { + const response = await request(app) + .post('/api/release-tracks/new') + .send({ name, type }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201); + return response.body; + } + + function get(path, status = 200) { + return request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + } + + it('returns every standard snapshot newest first with standard tier counts', async function () { + const response = await get(`/api/release-tracks/${standardTrack.id}/snapshots`); + + expect(response.body.pagination).toEqual({ + total: 3, + limit: 50, + offset: 0, + }); + expect(response.body.data).toHaveLength(3); + expect(response.body.data[0]).toMatchObject({ + id: standardTrack.id, + type: 'standard', + modified: standardLatestModified.toISOString(), + version: null, + members_count: 1, + staged_count: 2, + candidates_count: 1, + }); + expect(response.body.data[0]).not.toHaveProperty('quarantine_count'); + expect(response.body.data[1]).toMatchObject({ + modified: standardTaggedModified.toISOString(), + version: '1.0', + members_count: 2, + staged_count: 1, + candidates_count: 3, + }); + }); + + it('returns type-oriented counts for virtual snapshots', async function () { + const response = await get(`/api/release-tracks/${virtualTrack.id}/snapshots?tagged=true`); + + expect(response.body.pagination.total).toBe(1); + expect(response.body.data).toHaveLength(1); + expect(response.body.data[0]).toMatchObject({ + id: virtualTrack.id, + type: 'virtual', + version: '1.0', + members_count: 2, + quarantine_count: 1, + }); + expect(response.body.data[0]).not.toHaveProperty('staged_count'); + expect(response.body.data[0]).not.toHaveProperty('candidates_count'); + }); + + it('filters tagged and untagged snapshots before pagination', async function () { + const tagged = await get( + `/api/release-tracks/${standardTrack.id}/snapshots?tagged=true&limit=1&offset=0`, + ); + expect(tagged.body.pagination).toEqual({ + total: 1, + limit: 1, + offset: 0, + }); + expect(tagged.body.data.map((snapshot) => snapshot.version)).toEqual(['1.0']); + + const untagged = await get( + `/api/release-tracks/${standardTrack.id}/snapshots?tagged=false&limit=1&offset=1`, + ); + expect(untagged.body.pagination).toEqual({ + total: 2, + limit: 1, + offset: 1, + }); + expect(untagged.body.data).toHaveLength(1); + expect(untagged.body.data[0].version).toBeNull(); + }); + + it('retrieves the latest snapshot from the canonical endpoint', async function () { + const response = await get(`/api/release-tracks/${standardTrack.id}/snapshots/latest`); + + expect(response.body.modified).toBe(standardLatestModified.toISOString()); + expect(response.body.members).toHaveLength(1); + expect(response.body.staged).toHaveLength(2); + expect(response.body.candidates).toHaveLength(1); + }); + + it('does not allow latest-snapshot retrieval at the release-track resource path', async function () { + await get(`/api/release-tracks/${standardTrack.id}`, 405); + }); + + it('rejects invalid filter and pagination values', async function () { + await get(`/api/release-tracks/${standardTrack.id}/snapshots?tagged=yes`, 400); + await get(`/api/release-tracks/${standardTrack.id}/snapshots?limit=0`, 400); + await get(`/api/release-tracks/${standardTrack.id}/snapshots?limit=201`, 400); + await get(`/api/release-tracks/${standardTrack.id}/snapshots?offset=-1`, 400); + }); + + it('returns 404 when the release track does not exist', async function () { + await get( + '/api/release-tracks/release-track--00000000-0000-4000-8000-000000000099/snapshots', + 404, + ); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index a89cd574..301914a7 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,31 @@ # Release Track TODOs +## Remove implicit latest-snapshot route + +- [x] Remove `GET /api/release-tracks/:id` while preserving track deletion. +- [x] Make `/snapshots/latest` canonical across OpenAPI, tests, docs, Bruno, + and the frontend consumer. +- [x] Add regression coverage proving the removed method returns 405. +- [x] Run focused regression specs followed by the complete `npm test` suite. + The focused suites pass. The aggregate run reached 894 passing with three + unrelated documented roaming failures; all three affected specs pass + together in isolation (51 passing). +- [x] Review the final diff and propose a conventional commit message. + +## Snapshot history collection endpoint + +- [x] Add `GET /api/release-tracks/:id/snapshots` with strict tagged filtering + and pagination, plus an explicit `/snapshots/latest` alias. +- [x] Return lightweight, type-oriented summaries: standard snapshots include + member/staged/candidate counts; virtual snapshots include member/quarantine + counts. +- [x] Add regression coverage for defaults, filters, pagination, validation, + track types, and not-found behavior. +- [x] Update OpenAPI, user/developer documentation, and Bruno requests. +- [x] Run the focused regression spec followed by the complete `npm test` + suite. +- [x] Review the final diff and propose a conventional commit message. + ## Regression Tests - [ ] Implement regression tests @@ -47,7 +73,7 @@ The following release-track snapshot retrieval endpoints support `include` and `format` query parameters: -- `GET /api/release-tracks/:id` (get latest snapshot) +- `GET /api/release-tracks/:id/snapshots/latest` (get latest snapshot) - `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) > [!Note] @@ -56,24 +82,24 @@ The following release-track snapshot retrieval endpoints support `include` and **Include Parameter** (controls which tiers are returned): ``` -GET /api/release-tracks/:id # Default: all tiers -GET /api/release-tracks/:id?include=members # Members tier only -GET /api/release-tracks/:id?include=staged # Members and staged tiers -GET /api/release-tracks/:id?include=candidates # Members and candidates tiers -GET /api/release-tracks/:id?include=quarantine # Members and quarantine tiers -GET /api/release-tracks/:id?include=all # All tiers +GET /api/release-tracks/:id/snapshots/latest # Default: all tiers +GET /api/release-tracks/:id/snapshots/latest?include=members # Members tier only +GET /api/release-tracks/:id/snapshots/latest?include=staged # Members and staged tiers +GET /api/release-tracks/:id/snapshots/latest?include=candidates # Members and candidates tiers +GET /api/release-tracks/:id/snapshots/latest?include=quarantine # Members and quarantine tiers +GET /api/release-tracks/:id/snapshots/latest?include=all # All tiers ``` **Format Parameter** (controls output format): ``` -GET /api/release-tracks/:id?format=workbench # Workbench snapshot with metadata (default) -GET /api/release-tracks/:id?format=bundle # Standard STIX 2.1 bundle -GET /api/release-tracks/:id?format=filesystemstore # Not implemented; returns 501 +GET /api/release-tracks/:id/snapshots/latest?format=workbench # Workbench snapshot with metadata (default) +GET /api/release-tracks/:id/snapshots/latest?format=bundle # Standard STIX 2.1 bundle +GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Not implemented; returns 501 ``` **Combined Example:** ``` -GET /api/release-tracks/:id?include=all&format=workbench +GET /api/release-tracks/:id/snapshots/latest?include=all&format=workbench ``` > [!Note] @@ -116,7 +142,7 @@ Currently there exists support for the `format` query parameter on the `GET /api - [x] Read the existing release track user + developer documentation in `docs/user/release-tracks/` and `docs/developer/release-tracks/`, respectively. - [x] Review the new `GET /api/release-tracks/ephemeral/:domain` endpoint implementation as well as the legacy `GET /api/stix-bundles` endpoint. - [x] Implement support for the `format=bundle` query parameter in the following two endpoints: - - `GET /api/release-tracks/:id` (get latest snapshot) + - `GET /api/release-tracks/:id/snapshots/latest` (get latest snapshot) - `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) - [x] Ensure that all required logic (query parameters) is/are implemented in the new endpoints as outlined above. - [x] Implement regression tests for the new functionality (`release-tracks-bundle.spec.js`, `ephemeral-bundle.spec.js`) diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index af688cdf..385888c8 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -56,7 +56,7 @@ spec) and will be removed in a future release. Its replacements: | Legacy usage | Replacement | |--------------|-------------| | Domain-scoped ad hoc bundle | `GET /api/release-tracks/ephemeral/:domain` | -| Release/publication bundle | `GET /api/release-tracks/:id?format=bundle` (or `/snapshots/:modified?format=bundle`) | +| Release/publication bundle | `GET /api/release-tracks/:id/snapshots/latest?format=bundle` (or `/snapshots/:modified?format=bundle`) | ### Ephemeral endpoint parameter mapping diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 4868e87d..273c39c6 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -54,6 +54,33 @@ policies remain responsible only for different revisions of one object. - Large collections (>10k objects) may need pagination - Consider caching for `bump/preview` on large collections +### Snapshot history reads + +Snapshot history is exposed as a nested collection at +`GET /api/release-tracks/:id/snapshots`; latest-snapshot retrieval is exposed +only at `GET /api/release-tracks/:id/snapshots/latest`. The track resource path +retains `DELETE` but intentionally has no `GET` method because the release-track +API was still prerelease when this contract was adopted. The collection route +also replaces the previously documented but unimplemented `?versions=all` +polymorphism, so a single endpoint never changes between a full snapshot object +and a list response. + +`release-track-dynamic.repository.getSnapshotSummaries` performs tagged-state +filtering, descending timestamp ordering, pagination, and tier counts in +MongoDB. It projects counts with `$size` rather than hydrating the potentially +large tier arrays. The filter is applied to both the data query and +`countDocuments`, making `pagination.total` the filtered total. + +The service shapes projected counts according to `snapshot.type`: + +- standard: `members_count`, `staged_count`, `candidates_count` +- virtual: `members_count`, `quarantine_count` + +This omits structurally inapplicable counts instead of making a zero value +ambiguous. An omitted `tagged` parameter adds no version predicate; +`tagged=true` matches string versions and `tagged=false` matches null draft +versions. + ## Integrating with the Event-Driven Architecture ### Events Published diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index db0f7a55..0d123298 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -46,7 +46,6 @@ GET /api/release-tracks/objects/:objectRef/releases POST /api/release-tracks/new POST /api/release-tracks/new-from-bundle POST /api/release-tracks/import -GET /api/release-tracks/:id POST /api/release-tracks/:id/meta POST /api/release-tracks/:id/contents POST /api/release-tracks/:id/bump @@ -57,6 +56,8 @@ DELETE /api/release-tracks/:id ### Snapshot Operations ``` +GET /api/release-tracks/:id/snapshots +GET /api/release-tracks/:id/snapshots/latest GET /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/meta POST /api/release-tracks/:id/snapshots/:modified/bump @@ -305,9 +306,13 @@ POST /api/release-tracks/import Retrieves the most recent snapshot from the release track (by `modified` timestamp). ``` -GET /api/release-tracks/:id +GET /api/release-tracks/:id/snapshots/latest ``` +`GET /api/release-tracks/:id` is not supported. That resource path is reserved +for operations such as deleting the track; use `/snapshots/latest` whenever the +full latest snapshot is required. + Workbench responses return the release-track snapshot shape. Entries in the `members`, `staged`, `candidates`, and `quarantine` tiers include UI-friendly object details: @@ -324,7 +329,6 @@ Workbench responses return the release-track snapshot shape. Entries in the `mem | `include` | `members` \| `staged` \| `candidates` \| `quarantine` \| `all` | Which tier arrays to include in `workbench` responses (default: all tiers) | | `releases` | `only` | Return only the latest tagged release instead of latest snapshot | | `version` | `X.Y` | Return specific version (e.g., `14.1`) | -| `versions` | `all` | List all snapshots with metadata | **Additional query parameters for `format=bundle`:** @@ -341,28 +345,91 @@ See [Output Formats](output-formats.md) for details on the bundle structure. ```bash # Get latest snapshot for the Workbench UI -GET /api/release-tracks/:id +GET /api/release-tracks/:id/snapshots/latest # Get latest snapshot as STIX bundle (members only) -GET /api/release-tracks/:id?format=bundle +GET /api/release-tracks/:id/snapshots/latest?format=bundle # Get latest snapshot as STIX bundle with staged and candidate objects -GET /api/release-tracks/:id?format=bundle&include=candidates,staged +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged # Get latest snapshot as STIX bundle with candidates awaiting review -GET /api/release-tracks/:id?format=bundle&include=candidates&state=awaiting-review +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates&state=awaiting-review # Get latest snapshot with members and quarantine only -GET /api/release-tracks/:id?include=quarantine +GET /api/release-tracks/:id/snapshots/latest?include=quarantine # Get latest tagged release (not draft) -GET /api/release-tracks/:id?releases=only +GET /api/release-tracks/:id/snapshots/latest?releases=only # Get specific version -GET /api/release-tracks/:id?version=14.1 +GET /api/release-tracks/:id/snapshots/latest?version=14.1 +``` + +### List Snapshots + +Returns a paginated history of lightweight snapshot summaries, ordered by +`modified` from newest to oldest. Omitting `tagged` applies no tagged-state +filter. + +``` +GET /api/release-tracks/:id/snapshots +``` + +**Query Parameters:** + +| Parameter | Values | Default | Description | +| --------- | --------------- | ------- | ------------------------------------------------ | +| `tagged` | `true`\|`false` | omitted | Include only tagged snapshots or untagged drafts | +| `limit` | `1`–`200` | `50` | Maximum summaries to return | +| `offset` | integer ≥ `0` | `0` | Matching summaries to skip | + +Filtering occurs before pagination, so `pagination.total` is the total number +of snapshots matching `tagged`, not the total number in the track. + +Every summary contains `id`, `type`, `modified`, `version`, `name`, +`description` (when set), and `members_count`. Count keys then reflect the +track type: + +- `type: "standard"` adds `staged_count` and `candidates_count`. +- `type: "virtual"` adds `quarantine_count`. + +Inapplicable count keys are omitted rather than returned as zero. + +```json +{ + "data": [ + { + "id": "release-track--a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "type": "standard", + "modified": "2024-01-15T16:20:00.000Z", + "version": "14.1", + "name": "Enterprise ATT&CK", + "description": "Enterprise domain release track", + "members_count": 3247, + "staged_count": 18, + "candidates_count": 5 + } + ], + "pagination": { + "total": 47, + "limit": 50, + "offset": 0 + } +} +``` + +**Examples:** + +```bash +# All tagged and untagged snapshots +GET /api/release-tracks/:id/snapshots + +# Tagged releases only +GET /api/release-tracks/:id/snapshots?tagged=true -# List all snapshots -GET /api/release-tracks/:id?versions=all +# Untagged drafts only, second page +GET /api/release-tracks/:id/snapshots?tagged=false&limit=25&offset=25 ``` ### Update Metadata @@ -1106,7 +1173,7 @@ GET /api/release-tracks/:id/snapshots/preview The following release-track snapshot retrieval endpoints support `include` and `format` query parameters: -- `GET /api/release-tracks/:id` (get latest snapshot) +- `GET /api/release-tracks/:id/snapshots/latest` (get latest snapshot) - `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) The ephemeral bundle endpoint supports `format`, but not tier `include`, because @@ -1115,44 +1182,44 @@ it does not read from a persisted release-track snapshot. **Include Parameter** (workbench format — controls which tiers are returned): ``` -GET /api/release-tracks/:id # Default: all tiers -GET /api/release-tracks/:id?include=members # Members tier only -GET /api/release-tracks/:id?include=staged # Members and staged tiers -GET /api/release-tracks/:id?include=candidates # Members and candidates tiers -GET /api/release-tracks/:id?include=quarantine # Members and quarantine tiers -GET /api/release-tracks/:id?include=all # All tiers +GET /api/release-tracks/:id/snapshots/latest # Default: all tiers +GET /api/release-tracks/:id/snapshots/latest?include=members # Members tier only +GET /api/release-tracks/:id/snapshots/latest?include=staged # Members and staged tiers +GET /api/release-tracks/:id/snapshots/latest?include=candidates # Members and candidates tiers +GET /api/release-tracks/:id/snapshots/latest?include=quarantine # Members and quarantine tiers +GET /api/release-tracks/:id/snapshots/latest?include=all # All tiers ``` **Include Parameter** (bundle format — controls which tiers are hydrated into the bundle; members are always included): ``` -GET /api/release-tracks/:id?format=bundle # Members only -GET /api/release-tracks/:id?format=bundle&include=staged # Members + staged -GET /api/release-tracks/:id?format=bundle&include=candidates # Members + candidates -GET /api/release-tracks/:id?format=bundle&include=candidates,staged # Members + both +GET /api/release-tracks/:id/snapshots/latest?format=bundle # Members only +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=staged # Members + staged +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates # Members + candidates +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged # Members + both ``` **State Parameter** (bundle format only — narrows the tiers selected via `include` by workflow status; `reviewed` entries are always included): ``` -GET /api/release-tracks/:id?format=bundle&include=candidates&state=work-in-progress -GET /api/release-tracks/:id?format=bundle&include=candidates,staged&state=work-in-progress,awaiting-review +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates&state=work-in-progress +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged&state=work-in-progress,awaiting-review ``` **Format Parameter** (controls output format): ``` -GET /api/release-tracks/:id?format=workbench # Workbench snapshot with metadata (default) -GET /api/release-tracks/:id?format=bundle # Standard STIX bundle -GET /api/release-tracks/:id?format=filesystemstore # Not implemented; returns 501 +GET /api/release-tracks/:id/snapshots/latest?format=workbench # Workbench snapshot with metadata (default) +GET /api/release-tracks/:id/snapshots/latest?format=bundle # Standard STIX bundle +GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Not implemented; returns 501 ``` **Combined Example:** ``` -GET /api/release-tracks/:id?include=all&format=workbench +GET /api/release-tracks/:id/snapshots/latest?include=all&format=workbench ``` ### Bump Operations (Preview & Dry Run) diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index 62378761..dcf9843b 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -3,7 +3,7 @@ Release tracks (or rather, each snapshot) can serialize/export to multiple formats via query parameter: ``` -GET /api/release-tracks/:id?format= +GET /api/release-tracks/:id/snapshots/latest?format= ``` ### Format: `workbench` (Default) @@ -48,11 +48,11 @@ shape for snapshot retrieval endpoints and is intended for the Workbench fronten Use `include` to narrow tier arrays in `workbench` responses: ```bash -GET /api/release-tracks/:id?include=members -GET /api/release-tracks/:id?include=staged -GET /api/release-tracks/:id?include=candidates -GET /api/release-tracks/:id?include=quarantine -GET /api/release-tracks/:id?include=all +GET /api/release-tracks/:id/snapshots/latest?include=members +GET /api/release-tracks/:id/snapshots/latest?include=staged +GET /api/release-tracks/:id/snapshots/latest?include=candidates +GET /api/release-tracks/:id/snapshots/latest?include=quarantine +GET /api/release-tracks/:id/snapshots/latest?include=all ``` ### Format: `bundle` @@ -109,16 +109,16 @@ Examples: ```bash # Members only (default) -GET /api/release-tracks/:id?format=bundle +GET /api/release-tracks/:id/snapshots/latest?format=bundle # Members + staged objects -GET /api/release-tracks/:id?format=bundle&include=staged +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=staged # Members + candidates and staged objects that are work-in-progress or reviewed -GET /api/release-tracks/:id?format=bundle&include=candidates,staged&state=work-in-progress +GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged&state=work-in-progress # STIX 2.0 bundle without a table of contents -GET /api/release-tracks/:id?format=bundle&stixVersion=2.0&includeToc=false +GET /api/release-tracks/:id/snapshots/latest?format=bundle&stixVersion=2.0&includeToc=false ``` **The table of contents (TOC) object** @@ -180,14 +180,14 @@ collection-123/ ```bash # Workbench UI response -GET /api/release-tracks/:id -GET /api/release-tracks/:id?format=workbench +GET /api/release-tracks/:id/snapshots/latest +GET /api/release-tracks/:id/snapshots/latest?format=workbench # Standard STIX bundle for publication -GET /api/release-tracks/:id?format=bundle +GET /api/release-tracks/:id/snapshots/latest?format=bundle # FileSystemStore export is not implemented yet -GET /api/release-tracks/:id?format=filesystemstore # Returns HTTP 501 +GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Returns HTTP 501 # Dry run with detailed preview GET /api/release-tracks/:id/bump/preview?format=workbench diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index e409917d..cf1110b5 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -486,7 +486,7 @@ Workbench snapshot responses include all tier arrays by default. Set the `include` query parameter to `members`, `staged`, `candidates`, `quarantine`, or `all` to view a narrower subset of a given snapshot. ``` -GET /api/release-tracks/:id?include=all +GET /api/release-tracks/:id/snapshots/latest?include=all ``` **Response:** @@ -1008,7 +1008,7 @@ POST /api/release-tracks/:id/candidates/review Regularly check candidate status: ```bash -GET /api/release-tracks/:id?include=all +GET /api/release-tracks/:id/snapshots/latest?include=all ``` ### 5. Use Events for Automation diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index 7f27c60e..cc3d2ada 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -62,7 +62,7 @@ GET /api/release-tracks/ephemeral/:domain # Release track management POST /api/release-tracks/new -GET /api/release-tracks/:id +GET /api/release-tracks/:id/snapshots/latest POST /api/release-tracks/:id/config POST /api/release-tracks/:id/meta POST /api/release-tracks/:id/clone diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 023fac9b..813a4645 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -854,7 +854,7 @@ POST /api/release-tracks/:id/snapshots/:modified/bump ### Get Virtual Track with Resolved Content ```bash -GET /api/release-tracks/:id?format=workbench&include=all +GET /api/release-tracks/:id/snapshots/latest?format=workbench&include=all ``` **Query params:** @@ -898,7 +898,7 @@ When using the `quarantine` deduplication strategy, conflicting objects are stor **View quarantined objects:** ```bash -GET /api/release-tracks/:id?include=quarantine +GET /api/release-tracks/:id/snapshots/latest?include=quarantine ``` **Manually promote a quarantined object to members:** @@ -1064,7 +1064,7 @@ await cache.set(cacheKey, resolved, { ttl: 3600 }); // 1 hour cache ### 2. Lazy Resolution -For `GET /api/release-tracks/:id` (latest snapshot), only resolve if: +For `GET /api/release-tracks/:id/snapshots/latest` (latest snapshot), only resolve if: - Query param `resolve=true` is specified - Format requires resolution (e.g., `format=bundle`) From a890704c092264f4d7e07ac6537933e30c7d3e6e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:57:44 -0400 Subject: [PATCH 18/55] feat(release-tracks): unify release operations and previews Replace bump routes with snapshot-scoped release endpoints and shared summary, workbench, and bundle planning. Resolve latest snapshots at request time and use modified paths to target specific snapshots --- .../definitions/components/release-tracks.yml | 2 +- app/api/definitions/openapi.yml | 19 +- .../paths/release-tracks-paths.yml | 187 +++++++-- app/controllers/release-tracks-controller.js | 138 +++++-- app/lib/release-tracks/backref-reconciler.js | 2 +- app/lib/release-tracks/export-schemas.js | 2 +- .../release-tracks/release-track-schemas.js | 27 +- app/lib/release-tracks/version-utils.js | 16 +- .../release-track-snapshot-schema.js | 3 +- app/routes/release-tracks-routes.js | 45 ++- .../release-tracks/release-tracks-service.js | 37 +- .../release-tracks/versioning-service.js | 357 +++++++----------- .../release-tracks-backrefs.spec.js | 16 +- .../release-tracks-release.spec.js | 244 ++++++++++++ .../release-tracks-tier-invariant.spec.js | 10 +- .../release-tracks/releases-by-object.spec.js | 37 +- docs/developer/TODO.md | 39 ++ .../release-tracks/backref-reconciliation.md | 6 +- docs/developer/release-tracks/entities.md | 4 +- .../release-tracks/error-handling.md | 6 +- .../release-tracks/implementation-notes.md | 10 +- docs/user/release-tracks/api-reference.md | 123 +++--- docs/user/release-tracks/object-backrefs.md | 2 +- docs/user/release-tracks/output-formats.md | 4 +- docs/user/release-tracks/release-workflow.md | 130 +++---- docs/user/release-tracks/summary.md | 25 +- docs/user/release-tracks/terminology.md | 2 +- docs/user/release-tracks/versioning.md | 56 +-- docs/user/release-tracks/virtual-tracks.md | 14 +- docs/user/release-tracks/workflow-examples.md | 26 +- 30 files changed, 1025 insertions(+), 564 deletions(-) create mode 100644 app/tests/api/release-tracks/release-tracks-release.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index c80e0f0d..9f815f47 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -370,7 +370,7 @@ components: staged_count: type: number description: 'Objects remaining in staged after release' - candidate_count: + candidates_count: type: number description: 'Objects in candidates at time of release' diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index b9996789..e81fd13c 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -358,12 +358,6 @@ paths: /api/release-tracks/{id}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1clone' - /api/release-tracks/{id}/bump: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1bump' - - /api/release-tracks/{id}/bump/preview: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1bump~1preview' - /api/release-tracks/{id}/candidates: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1candidates' @@ -406,6 +400,12 @@ paths: /api/release-tracks/{id}/snapshots/latest: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1latest' + /api/release-tracks/{id}/snapshots/latest/release: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1latest~1release' + + /api/release-tracks/{id}/snapshots/latest/release/preview: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1latest~1release~1preview' + /api/release-tracks/{id}/snapshots/{modified}: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}' @@ -418,8 +418,11 @@ paths: /api/release-tracks/{id}/snapshots/{modified}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1clone' - /api/release-tracks/{id}/snapshots/{modified}/bump: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1bump' + /api/release-tracks/{id}/snapshots/{modified}/release: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1release' + + /api/release-tracks/{id}/snapshots/{modified}/release/preview: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1release~1preview' # System Configuration /api/config/system-version: diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index d2a3f5a2..78933c9f 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -360,16 +360,16 @@ paths: '201': description: 'Release track cloned successfully' - /api/release-tracks/{id}/bump: + /api/release-tracks/{id}/snapshots/latest/release: post: - summary: 'Tag the latest snapshot (create a release)' - operationId: 'release-tracks-bump-latest' + summary: 'Release the latest snapshot' + operationId: 'release-tracks-release-latest' description: | - Tag the latest snapshot with a version number. - For standard tracks: promotes staged → members. - Exact staged/member duplicates are idempotent and normalized rather than treated as conflicts. - For virtual tracks: N/A (already resolved). - Request body validated via Zod in controller: { type: 'major'|'minor', version?: string, dry_run?: boolean } + Immutably tag the latest snapshot with a version. Standard tracks + promote staged entries to members. The `latest` selector is resolved + when the request is handled. Supply either `increment` (`major` or + `minor`) or an explicit `version` in `MAJOR.MINOR` form, but never + both. Omitting both defaults to a minor increment. tags: - 'Release Tracks' parameters: @@ -378,21 +378,33 @@ paths: required: true schema: type: string + requestBody: + required: true + description: | + Version selection. `increment` and `version` are mutually exclusive; + supplying both returns 400. An empty object defaults to a minor + increment. + content: + application/json: + schema: + type: object + additionalProperties: true responses: '200': - description: 'Snapshot tagged successfully' + description: 'Snapshot released successfully' + '400': + description: 'Invalid release request' '409': - description: 'Snapshot already tagged or conflict during promotion' - '501': - description: 'Not yet implemented' + description: 'Already released or conflicting snapshot' - /api/release-tracks/{id}/bump/preview: + /api/release-tracks/{id}/snapshots/latest/release/preview: get: - summary: 'Preview the next release' - operationId: 'release-tracks-bump-preview' + summary: 'Preview releasing the latest snapshot' + operationId: 'release-tracks-preview-latest-release' description: | - Compute what the next tagged release will contain without persisting changes. - Shows which objects will be promoted from staged → members. + Plan without persisting. Summary is the default; workbench and bundle + render the complete would-be release. `increment` and `version` are + mutually exclusive; omitting both defaults to a minor increment. tags: - 'Release Tracks' parameters: @@ -403,14 +415,53 @@ paths: type: string - name: format in: query - description: 'Output format. filesystemstore is not yet implemented.' + description: 'Preview representation. Defaults to summary.' schema: type: string enum: + - summary - bundle - workbench - filesystemstore - default: workbench + default: summary + - name: increment + in: query + description: 'Version increment; mutually exclusive with version.' + schema: + type: string + enum: [major, minor] + - name: version + in: query + description: 'Explicit MAJOR.MINOR version; mutually exclusive with increment.' + schema: + type: string + - name: include + in: query + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: state + in: query + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: stixVersion + in: query + schema: + type: string + enum: ['2.0', '2.1'] + - name: includeToc + in: query + schema: + type: boolean responses: '200': description: 'Release preview generated' @@ -1126,14 +1177,15 @@ paths: '201': description: 'Release track cloned successfully' - /api/release-tracks/{id}/snapshots/{modified}/bump: + /api/release-tracks/{id}/snapshots/{modified}/release: post: - summary: 'Tag a specific snapshot' - operationId: 'release-tracks-bump-by-modified' + summary: 'Release a specific snapshot' + operationId: 'release-tracks-release-by-modified' description: | - Tag a historical snapshot with a version number. - Exact staged/member duplicates are idempotent and normalized rather than treated as conflicts. - Request body validated via Zod in controller. + Immutably tag the snapshot selected by the modified timestamp using + the same version-selection contract as the latest release operation: + supply `increment` or `version`, never both; omit both for a minor + increment. tags: - 'Release Tracks' parameters: @@ -1147,8 +1199,89 @@ paths: required: true schema: type: string + requestBody: + required: true + description: | + Version selection. `increment` and `version` are mutually exclusive; + supplying both returns 400. An empty object defaults to a minor + increment. + content: + application/json: + schema: + type: object + additionalProperties: true responses: '200': - description: 'Snapshot tagged successfully' + description: 'Snapshot released successfully' + '400': + description: 'Invalid release request' + '409': + description: 'Already released or conflicting snapshot' + + /api/release-tracks/{id}/snapshots/{modified}/release/preview: + get: + summary: 'Preview releasing a specific snapshot' + operationId: 'release-tracks-preview-release-by-modified' + description: | + Plan without persisting. `increment` and `version` are mutually + exclusive; omitting both defaults to a minor increment. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: modified + in: path + required: true + schema: + type: string + - name: format + in: query + schema: + type: string + enum: [summary, workbench, bundle, filesystemstore] + default: summary + - name: increment + in: query + schema: + type: string + enum: [major, minor] + - name: version + in: query + schema: + type: string + - name: include + in: query + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: state + in: query + allowReserved: true + schema: + oneOf: + - type: string + - type: array + items: + type: string + - name: stixVersion + in: query + schema: + type: string + enum: ['2.0', '2.1'] + - name: includeToc + in: query + schema: + type: boolean + responses: + '200': + description: 'Release preview generated' '501': - description: 'Not yet implemented' + description: 'Requested format is not yet implemented' diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index e121609c..f7aa1447 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -22,6 +22,7 @@ const { const { domainParamSchema, formatQuerySchema, + releasePreviewFormatSchema, includeQuerySchema, bundleIncludeQuerySchema, bundleStateQuerySchema, @@ -38,7 +39,8 @@ const { createFromBundleBodySchema, updateMetadataBodySchema, updateContentsBodySchema, - bumpBodySchema, + releaseBodySchema, + releaseVersionSelectionSchema, cloneBodySchema, addCandidatesBodySchema, reviewCandidatesBodySchema, @@ -139,6 +141,59 @@ function parseSnapshotQueryParams(query) { }; } +function parseReleasePreviewQueryParams(query) { + const format = parseOptionalQueryStrict( + query.format, + releasePreviewFormatSchema, + 'summary', + 'format', + ); + const versionSelection = releaseVersionSelectionSchema.safeParse({ + increment: query.increment, + version: query.version, + }); + if (!versionSelection.success) { + throw new InvalidQueryStringParameterError({ + parameterName: 'increment,version', + message: 'Invalid release version selection', + details: versionSelection.error.errors, + }); + } + + const options = { format, ...versionSelection.data }; + if (format === 'bundle') { + return { + ...options, + include: parseOptionalQueryStrict( + query.include, + bundleIncludeQuerySchema, + undefined, + 'include', + ), + state: parseOptionalQueryStrict(query.state, bundleStateQuerySchema, undefined, 'state'), + stixVersion: parseOptionalQueryStrict( + query.stixVersion, + stixVersionQuerySchema, + '2.1', + 'stixVersion', + ), + includeToc: parseOptionalQueryStrict( + query.includeToc, + booleanQuerySchema, + true, + 'includeToc', + ), + }; + } + if (format === 'workbench') { + return { + ...options, + include: parseOptionalQueryStrict(query.include, includeQuerySchema, undefined, 'include'), + }; + } + return options; +} + // ============================================================================= // Ephemeral // ============================================================================= @@ -412,27 +467,27 @@ exports.updateContentsByLatest = async function updateContentsByLatest(req, res, } }; -/** POST /api/release-tracks/:id/bump */ -exports.bumpByLatest = async function bumpByLatest(req, res, next) { +/** POST /api/release-tracks/:id/snapshots/latest/release */ +exports.releaseLatest = async function releaseLatest(req, res, next) { try { - const bodyResult = bumpBodySchema.safeParse(req.body || {}); + const bodyResult = releaseBodySchema.safeParse(req.body || {}); if (!bodyResult.success) { return next( new BadRequestError({ - message: 'Invalid bump request', + message: 'Invalid release request', details: bodyResult.error.errors, }), ); } - const result = await releaseTracksService.bumpLatest(req.params.id, { + const result = await releaseTracksService.releaseLatest(req.params.id, { ...bodyResult.data, userAccountId: req.user?.userAccountId, }); - logger.debug(`Success: Bumped version for track ${req.params.id}`); + logger.debug(`Success: Released latest snapshot for track ${req.params.id}`); return res.status(200).send(result); } catch (err) { - logger.error('Failed to bump track version: ' + err); + logger.error('Failed to release latest snapshot: ' + err); return next(err); } }; @@ -557,27 +612,31 @@ exports.updateContentsByModified = async function updateContentsByModified(req, } }; -/** POST /api/release-tracks/:id/snapshots/:modified/bump */ -exports.bumpByModified = async function bumpByModified(req, res, next) { +/** POST /api/release-tracks/:id/snapshots/:modified/release */ +exports.releaseByModified = async function releaseByModified(req, res, next) { try { - const bodyResult = bumpBodySchema.safeParse(req.body || {}); + const bodyResult = releaseBodySchema.safeParse(req.body || {}); if (!bodyResult.success) { return next( new BadRequestError({ - message: 'Invalid bump request', + message: 'Invalid release request', details: bodyResult.error.errors, }), ); } - const result = await releaseTracksService.bumpByModified(req.params.id, req.params.modified, { - ...bodyResult.data, - userAccountId: req.user?.userAccountId, - }); - logger.debug(`Success: Bumped version for snapshot ${req.params.modified}`); + const result = await releaseTracksService.releaseByModified( + req.params.id, + req.params.modified, + { + ...bodyResult.data, + userAccountId: req.user?.userAccountId, + }, + ); + logger.debug(`Success: Released snapshot ${req.params.modified}`); return res.status(200).send(result); } catch (err) { - logger.error('Failed to bump snapshot version: ' + err); + logger.error('Failed to release snapshot: ' + err); return next(err); } }; @@ -846,28 +905,45 @@ exports.updateConfig = async function updateConfig(req, res, next) { }; // ============================================================================= -// Preview & dry run +// Release previews // ============================================================================= -/** GET /api/release-tracks/:id/bump/preview */ -exports.previewBump = async function previewBump(req, res, next) { +/** GET /api/release-tracks/:id/snapshots/latest/release/preview */ +exports.previewLatestRelease = async function previewLatestRelease(req, res, next) { try { - const format = parseOptionalQueryStrict( - req.query.format, - formatQuerySchema, - 'workbench', - 'format', - ); - const formatError = rejectFilesystemStoreFormat(format, 'previewBump'); + const options = parseReleasePreviewQueryParams(req.query); + const formatError = rejectFilesystemStoreFormat(options.format, 'previewLatestRelease'); if (formatError) { return next(formatError); } - const result = await releaseTracksService.previewBump(req.params.id, format); - logger.debug(`Success: Generated bump preview for track ${req.params.id}`); + const result = await releaseTracksService.previewLatestRelease(req.params.id, options); + logger.debug(`Success: Previewed release for track ${req.params.id}`); + return res.status(200).send(result); + } catch (err) { + logger.error('Failed to preview latest release: ' + err); + return next(err); + } +}; + +/** GET /api/release-tracks/:id/snapshots/:modified/release/preview */ +exports.previewReleaseByModified = async function previewReleaseByModified(req, res, next) { + try { + const options = parseReleasePreviewQueryParams(req.query); + const formatError = rejectFilesystemStoreFormat(options.format, 'previewReleaseByModified'); + if (formatError) { + return next(formatError); + } + + const result = await releaseTracksService.previewReleaseByModified( + req.params.id, + req.params.modified, + options, + ); + logger.debug(`Success: Previewed release for snapshot ${req.params.modified}`); return res.status(200).send(result); } catch (err) { - logger.error('Failed to preview bump: ' + err); + logger.error('Failed to preview snapshot release: ' + err); return next(err); } }; diff --git a/app/lib/release-tracks/backref-reconciler.js b/app/lib/release-tracks/backref-reconciler.js index 44995d6f..fd0cd48a 100644 --- a/app/lib/release-tracks/backref-reconciler.js +++ b/app/lib/release-tracks/backref-reconciler.js @@ -21,7 +21,7 @@ // (latest) snapshot, compute the desired set of backrefs and diff it against // the documents that currently carry an entry for that track. This single // code path covers every membership mutation (add/remove/review/promote/ -// demote/bump/member-sync/clone/bundle-import/updateContents) as well as +// demote/release/member-sync/clone/bundle-import/updateContents) as well as // snapshot deletion (membership reverts to the new latest snapshot) and // track deletion (snapshot = null removes all entries). // diff --git a/app/lib/release-tracks/export-schemas.js b/app/lib/release-tracks/export-schemas.js index ae1a6eab..52f8ec58 100644 --- a/app/lib/release-tracks/export-schemas.js +++ b/app/lib/release-tracks/export-schemas.js @@ -216,7 +216,7 @@ const workbenchTransformSchema = exportInputSchema.transform((input) => { summary: { released_count: (input.snapshot.members || []).length, staged_count: (input.snapshot.staged || []).length, - candidate_count: (input.snapshot.candidates || []).length, + candidates_count: (input.snapshot.candidates || []).length, }, }; }); diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 4c7a58f2..0ecd1a7d 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -151,6 +151,7 @@ const cronSchema = z const domainParamSchema = z.enum(['enterprise', 'ics', 'mobile']); const formatQuerySchema = z.enum(['bundle', 'filesystemstore', 'workbench']); +const releasePreviewFormatSchema = z.enum(['summary', 'bundle', 'filesystemstore', 'workbench']); const includeQuerySchema = z.enum(['members', 'staged', 'candidates', 'quarantine', 'all']); @@ -202,7 +203,7 @@ const releaseLimitQuerySchema = z.coerce.number().int().min(1).max(200); const releaseOffsetQuerySchema = z.coerce.number().int().min(0); -const bumpTypeSchema = z.enum(['major', 'minor']); +const releaseIncrementSchema = z.enum(['major', 'minor']); const workflowStatusSchema = z.enum(['work-in-progress', 'awaiting-review', 'reviewed']); @@ -320,12 +321,18 @@ const updateContentsBodySchema = z.object({ .min(1), }); -/** POST /release-tracks/:id/bump */ -const bumpBodySchema = z.object({ - type: bumpTypeSchema.optional(), - version: xMitreVersionSchema.optional(), - dry_run: z.boolean().optional(), -}); +const releaseVersionSelectionSchema = z + .object({ + increment: releaseIncrementSchema.optional(), + version: xMitreVersionSchema.optional(), + }) + .strict() + .refine((value) => !(value.increment && value.version), { + message: 'increment and version are mutually exclusive', + }); + +/** POST /release-tracks/:id/snapshots/{target}/release */ +const releaseBodySchema = releaseVersionSelectionSchema; /** POST /release-tracks/:id/clone */ const cloneBodySchema = z @@ -433,6 +440,7 @@ module.exports = { // Query parameter schemas domainParamSchema, formatQuerySchema, + releasePreviewFormatSchema, includeQuerySchema, bundleIncludeQuerySchema, bundleStateQuerySchema, @@ -443,7 +451,8 @@ module.exports = { releaseOrderQuerySchema, releaseLimitQuerySchema, releaseOffsetQuerySchema, - bumpTypeSchema, + releaseIncrementSchema, + releaseVersionSelectionSchema, workflowStatusSchema, trackEntryStatusSchema, candidacyThresholdSchema, @@ -459,7 +468,7 @@ module.exports = { createFromBundleBodySchema, updateMetadataBodySchema, updateContentsBodySchema, - bumpBodySchema, + releaseBodySchema, cloneBodySchema, addCandidatesBodySchema, reviewCandidatesBodySchema, diff --git a/app/lib/release-tracks/version-utils.js b/app/lib/release-tracks/version-utils.js index 917b5df2..f58fa57f 100644 --- a/app/lib/release-tracks/version-utils.js +++ b/app/lib/release-tracks/version-utils.js @@ -46,23 +46,31 @@ exports.compareVersions = function compareVersions(a, b) { }; /** - * Calculate the next version based on version history and bump type. + * Calculate the next version based on version history and release increment. * * If an explicit version is provided, it is returned as-is (validation * is handled separately by validateVersionProgression). * + * Increment and explicit version selectors are mutually exclusive. + * * If the version history is empty, the first version defaults to "1.0". * * @param {Array<{ version: string }>} versionHistory - Existing version history entries - * @param {string} [bumpType='minor'] - 'major' or 'minor' + * @param {string} [increment='minor'] - 'major' or 'minor' * @param {string} [explicitVersion] - Explicit version override * @returns {string} The calculated version string + * @throws {InvalidVersionError} If both selectors are supplied or the explicit + * version is invalid */ exports.calculateNextVersion = function calculateNextVersion( versionHistory, - bumpType, + increment, explicitVersion, ) { + if (increment && explicitVersion) { + throw new InvalidVersionError('increment and version are mutually exclusive'); + } + if (explicitVersion) { // Validate format only; monotonicity is checked by validateVersionProgression exports.parseVersion(explicitVersion); @@ -82,7 +90,7 @@ exports.calculateNextVersion = function calculateNextVersion( } const { major, minor } = exports.parseVersion(highest); - const type = bumpType || 'minor'; + const type = increment || 'minor'; return type === 'major' ? `${major + 1}.0` : `${major}.${minor + 1}`; }; diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index c2630d4e..2d2c18a7 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -272,7 +272,8 @@ const versionHistoryEntryDefinition = { members_count: { type: Number }, promoted_count: { type: Number }, staged_count: { type: Number }, - candidate_count: { type: Number }, + candidates_count: { type: Number }, + quarantine_count: { type: Number }, }, // Virtual tracks only: records which component versions were included component_versions: { type: mongoose.Schema.Types.Mixed, default: undefined }, diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 28ae4899..f25fb08e 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -68,15 +68,6 @@ router // Latest snapshot operations (parameterised by :id) // ============================================================================= -/** Bump preview must be registered before :id/bump to avoid param conflict */ -router - .route('/release-tracks/:id/bump/preview') - .get( - authn.authenticate, - authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), - releaseTracksController.previewBump, - ); - router .route('/release-tracks/:id/meta') .post( @@ -98,14 +89,6 @@ router releaseTracksController.updateContentsByLatest, ); -router - .route('/release-tracks/:id/bump') - .post( - authn.authenticate, - authz.requireRole(authz.editorOrHigher), - releaseTracksController.bumpByLatest, - ); - router .route('/release-tracks/:id/clone') .post( @@ -232,6 +215,22 @@ router releaseTracksController.retrieveLatestSnapshot, ); +router + .route('/release-tracks/:id/snapshots/latest/release/preview') + .get( + authn.authenticate, + authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + releaseTracksController.previewLatestRelease, + ); + +router + .route('/release-tracks/:id/snapshots/latest/release') + .post( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.releaseLatest, + ); + router .route('/release-tracks/:id/snapshots/preview') .get( @@ -269,11 +268,19 @@ router ); router - .route('/release-tracks/:id/snapshots/:modified/bump') + .route('/release-tracks/:id/snapshots/:modified/release/preview') + .get( + authn.authenticate, + authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + releaseTracksController.previewReleaseByModified, + ); + +router + .route('/release-tracks/:id/snapshots/:modified/release') .post( authn.authenticate, authz.requireRole(authz.editorOrHigher), - releaseTracksController.bumpByModified, + releaseTracksController.releaseByModified, ); router diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index eaac3109..03a9ec78 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -9,7 +9,7 @@ // Phase 1: Track management, snapshot CRUD, config → snapshot-service // Phase 2: Candidates, staged, object versions → standard-track-service // Phase 3: Auto-promotion, workflow → workflow-service -// Phase 4: Bump/tag, versioning → versioning-service +// Phase 4: Release planning and versioning → versioning-service // Phase 5: Virtual track composition → virtual-track-service // Phase 6: Export, ephemeral, bundle import → export-service, ephemeral-service, bundle-import-service // ============================================================================= @@ -308,17 +308,38 @@ exports.demoteStaged = function demoteStaged(trackId, objectRefs, userId) { // Versioning (Phase 4 → versioning-service) // ----------------------------------------------------------------------------- -exports.bumpLatest = function bumpLatest(trackId, options) { - return versioningService.bumpLatest(trackId, options); +exports.releaseLatest = function releaseLatest(trackId, options) { + return versioningService.releaseLatest(trackId, options); }; -exports.bumpByModified = function bumpByModified(trackId, modified, options) { - return versioningService.bumpByModified(trackId, modified, options); +exports.releaseByModified = function releaseByModified(trackId, modified, options) { + return versioningService.releaseByModified(trackId, modified, options); }; -exports.previewBump = function previewBump(trackId, format) { - rejectFilesystemStoreFormat(format, 'previewBump'); - return versioningService.previewBump(trackId, format); +async function renderReleasePlan(plan, options) { + const format = options.format || 'summary'; + rejectFilesystemStoreFormat(format, 'previewRelease'); + + if (format === 'summary') return plan.summary; + if (plan.blockingError) throw plan.blockingError; + if (format === 'bundle') { + return exportService.exportSnapshot(plan.plannedSnapshot, format, options); + } + return formatWorkbenchSnapshot(plan.plannedSnapshot, options); +} + +exports.previewLatestRelease = async function previewLatestRelease(trackId, options) { + const plan = await versioningService.planLatestRelease(trackId, options); + return renderReleasePlan(plan, options); +}; + +exports.previewReleaseByModified = async function previewReleaseByModified( + trackId, + modified, + options, +) { + const plan = await versioningService.planReleaseByModified(trackId, modified, options); + return renderReleasePlan(plan, options); }; // ----------------------------------------------------------------------------- diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index e4438328..ef9cb2df 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -1,18 +1,8 @@ 'use strict'; -// ============================================================================= -// Versioning Service -// -// Manages the bump/tag lifecycle for release track snapshots: -// - Calculate and assign version numbers (MAJOR.MINOR) -// - Promote staged entries to members atomically with tagging -// - Preview upcoming bumps without persisting -// -// Tagging is the ONLY in-place mutation on a snapshot. All other changes -// produce new snapshot clones via snapshot-service. -// -// See docs/COLLECTIONS_V2/03_VERSIONING.md for versioning rules. -// ============================================================================= +// Plans and commits immutable releases from release-track snapshots. Planning +// is side-effect free; persistence, reconciliation, and events occur only in +// the commit path. const snapshotService = require('./snapshot-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); @@ -21,264 +11,191 @@ const conflictResolution = require('../../lib/release-tracks/conflict-resolution const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const releaseHistoryService = require('./release-history-service'); const logger = require('../../lib/logger'); -const { AlreadyReleasedError } = require('../../exceptions'); +const { AlreadyReleasedError, ReleaseConflictError } = require('../../exceptions'); -// ============================================================================= -// Internal helpers -// ============================================================================= +function iso(value) { + return new Date(value).toISOString(); +} + +function tierCounts(snapshot) { + if (snapshot.type === 'virtual') { + return { + members_count: (snapshot.members || []).length, + quarantine_count: (snapshot.quarantine || []).length, + }; + } + + return { + members_count: (snapshot.members || []).length, + staged_count: (snapshot.staged || []).length, + candidates_count: (snapshot.candidates || []).length, + }; +} /** - * Core bump logic shared by bumpLatest and bumpByModified. + * Build the complete release plan without reading or writing external state. * * @param {string} trackId - * @param {Object} snapshot - The snapshot to tag - * @param {Object} options - { type?, version?, dry_run?, userAccountId } - * @returns {Promise} The tagged snapshot (or preview if dry_run) + * @param {Object} sourceSnapshot + * @param {Array} versionHistory + * @param {Object} options + * @param {Date} now + * @returns {Object} */ -async function _doBump(trackId, snapshot, options) { - // Guard: cannot re-tag an already-tagged snapshot - if (snapshot.version != null) { - await releaseHistoryService.reconcileTaggedReleases(trackId); - throw new AlreadyReleasedError(snapshot.version); +function planRelease(trackId, sourceSnapshot, versionHistory, options = {}, now = new Date()) { + if (sourceSnapshot.version != null) { + throw new AlreadyReleasedError(sourceSnapshot.version); } - const normalized = tierRevisionInvariant.normalizeSnapshot(snapshot); - const workingSnapshot = normalized.snapshot; - - // A historical draft's embedded version_history can predate newer tags. - // Read the track-wide tagged releases so retroactive tagging cannot reuse or - // regress a version. - const versionHistory = await releaseHistoryService.getTrackWideVersionHistory(trackId); - - // Calculate version - const version = versionUtils.calculateNextVersion(versionHistory, options.type, options.version); - - // Validate monotonic progression + const normalized = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot); + const snapshot = normalized.snapshot; + const version = versionUtils.calculateNextVersion( + versionHistory, + options.increment, + options.version, + ); versionUtils.validateVersionProgression(version, versionHistory); - // Promote staged → members (standard tracks only) - const staged = workingSnapshot.staged || []; - const existingMembers = workingSnapshot.members || []; + const before = tierCounts(snapshot); + const staged = snapshot.type === 'standard' ? snapshot.staged || [] : []; + const existingMembers = snapshot.members || []; let mergedMembers = existingMembers; - let promotedCount = 0; + let blockingError; if (staged.length > 0) { - // Convert staged entries to member entries (strip staged-specific fields) - const stagedAsMembers = staged.map((s) => ({ - object_ref: s.object_ref, - object_modified: s.object_modified, + const incoming = staged.map(({ object_ref, object_modified }) => ({ + object_ref, + object_modified, })); + const policy = snapshot.config?.promotion_conflicts?.staged_to_members || 'abort'; - const policy = - (workingSnapshot.config && - workingSnapshot.config.promotion_conflicts && - workingSnapshot.config.promotion_conflicts.staged_to_members) || - 'abort'; - - const { merged } = conflictResolution.applyConflictPolicy( - existingMembers, - stagedAsMembers, - policy, - ); - - mergedMembers = merged; - promotedCount = staged.length; + try { + mergedMembers = conflictResolution.applyConflictPolicy( + existingMembers, + incoming, + policy, + ).merged; + } catch (err) { + if (!(err instanceof ReleaseConflictError)) throw err; + blockingError = err; + } } - const now = new Date(); + const additionalOps = {}; + for (const tier of normalized.changedTiers) { + additionalOps[tier] = snapshot[tier]; + } + if (staged.length > 0 && !blockingError) { + additionalOps.members = mergedMembers; + additionalOps.staged = []; + } - // Build version history entry + const afterSnapshot = { + ...snapshot, + version, + members: mergedMembers, + ...(snapshot.type === 'standard' ? { staged: [] } : {}), + }; + const after = tierCounts(afterSnapshot); const versionHistoryEntry = { version, tagged_at: now, tagged_by: options.userAccountId || 'system', - snapshot_id: snapshot.modified, + snapshot_id: sourceSnapshot.modified, summary: { - members_count: mergedMembers.length, - promoted_count: promotedCount, - staged_count: staged.length, - candidate_count: (workingSnapshot.candidates || []).length, + ...after, + promoted_count: blockingError ? 0 : staged.length, }, }; + const plannedSnapshot = blockingError + ? null + : { + ...afterSnapshot, + version_history: [...(snapshot.version_history || []), versionHistoryEntry], + }; - // Dry-run: return preview without persisting - if (options.dry_run) { - return { - dry_run: true, + return { + trackId, + sourceSnapshot, + plannedSnapshot, + version, + versionHistoryEntry, + additionalOps, + normalizedRemoved: normalized.removed, + blockingError, + summary: { track_id: trackId, - snapshot_modified: snapshot.modified, + type: snapshot.type, + source_snapshot_modified: iso(sourceSnapshot.modified), version, - staged_to_promote: staged.length, - members_after: mergedMembers.length, - version_history_entry: versionHistoryEntry, - }; - } + releasable: !blockingError, + before, + after: blockingError ? before : after, + changes: { + promoted_count: blockingError ? 0 : staged.length, + }, + conflicts: blockingError?.conflicts || [], + }, + }; +} - // Build additional atomic ops for the tag update - const additionalOps = {}; - for (const tier of normalized.changedTiers) { - additionalOps[tier] = workingSnapshot[tier]; - } - if (staged.length > 0) { - additionalOps.members = mergedMembers; - additionalOps.staged = []; - } +async function planLoadedSnapshot(trackId, snapshot, options) { + const versionHistory = await releaseHistoryService.getTrackWideVersionHistory(trackId); + return planRelease(trackId, snapshot, versionHistory, options); +} - // Atomic tag + promotion - const tagged = await dynamicRepo.tagSnapshotInPlace(trackId, snapshot.modified, { - version, - versionHistoryEntry, - additionalOps: Object.keys(additionalOps).length > 0 ? additionalOps : undefined, +async function commitPlan(plan) { + if (plan.blockingError) throw plan.blockingError; + + const tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { + version: plan.version, + versionHistoryEntry: plan.versionHistoryEntry, + additionalOps: Object.keys(plan.additionalOps).length > 0 ? plan.additionalOps : undefined, }); if (!tagged) { - // Race condition: snapshot was already tagged between our read and update - await releaseHistoryService.reconcileTaggedReleases(trackId); - throw new AlreadyReleasedError('(concurrent tag)'); + await releaseHistoryService.reconcileTaggedReleases(plan.trackId); + throw new AlreadyReleasedError('(concurrent release)'); } - // Rebuild the registry's compact tagged-release catalogue from the source - // snapshots. This is idempotent and repairs missed/partial prior updates. - await releaseHistoryService.reconcileTaggedReleases(trackId); - - // The staged → members promotion changed tier membership. Re-read the - // latest snapshot rather than using `tagged` — bumpByModified may have - // tagged an older snapshot, and backrefs track the latest one. - const latest = await dynamicRepo.getLatestSnapshot(trackId); - await snapshotService.emitContentsChanged(trackId, latest); + await releaseHistoryService.reconcileTaggedReleases(plan.trackId); + const latest = await dynamicRepo.getLatestSnapshot(plan.trackId); + await snapshotService.emitContentsChanged(plan.trackId, latest); logger.verbose( - `VersioningService: Tagged track "${trackId}" as v${version} ` + - `(promoted ${promotedCount} staged → members)`, + `VersioningService: Released track "${plan.trackId}" as v${plan.version} ` + + `(promoted ${plan.summary.changes.promoted_count} staged → members)`, ); - - if (normalized.removed.length > 0) { + if (plan.normalizedRemoved.length > 0) { logger.warn( - `VersioningService: Removed ${normalized.removed.length} exact cross-tier revision ` + - `duplicate(s) while tagging track "${trackId}"`, + `VersioningService: Removed ${plan.normalizedRemoved.length} exact cross-tier revision ` + + `duplicate(s) while releasing track "${plan.trackId}"`, ); } return tagged; } -// ============================================================================= -// Public API -// ============================================================================= +exports.planRelease = planRelease; -/** - * Tag the latest snapshot of a track as a versioned release. - * - * - Calculates the next version (or uses explicit version from options) - * - Promotes all staged entries to members atomically - * - Records the version in version_history - * - Updates registry counters - * - * @param {string} trackId - * @param {Object} options - { type?: 'major'|'minor', version?: string, dry_run?: boolean, userAccountId?: string } - * @returns {Promise} The tagged snapshot (or preview object if dry_run) - */ -exports.bumpLatest = async function bumpLatest(trackId, options = {}) { +exports.planLatestRelease = async function planLatestRelease(trackId, options = {}) { const snapshot = await snapshotService.getLatestSnapshot(trackId); - return _doBump(trackId, snapshot, options); + return planLoadedSnapshot(trackId, snapshot, options); }; -/** - * Tag a specific snapshot (by modified timestamp) as a versioned release. - * - * Same semantics as bumpLatest but targets a specific snapshot. - * - * @param {string} trackId - * @param {string|Date} modified - The snapshot's modified timestamp - * @param {Object} options - { type?: 'major'|'minor', version?: string, dry_run?: boolean, userAccountId?: string } - * @returns {Promise} The tagged snapshot (or preview object if dry_run) - */ -exports.bumpByModified = async function bumpByModified(trackId, modified, options = {}) { +exports.planReleaseByModified = async function planReleaseByModified( + trackId, + modified, + options = {}, +) { const snapshot = await snapshotService.getSnapshotByModified(trackId, modified); - return _doBump(trackId, snapshot, options); + return planLoadedSnapshot(trackId, snapshot, options); }; -/** - * Preview what a bump on the latest snapshot would produce without persisting. - * - * Returns the calculated version, staged-to-members diff, and summary stats. - * - * @param {string} trackId - * @param {string} [_format] - Reserved for future export format support - * @returns {Promise} Preview object - */ -// eslint-disable-next-line no-unused-vars -exports.previewBump = async function previewBump(trackId, _format) { - const sourceSnapshot = await snapshotService.getLatestSnapshot(trackId); - const snapshot = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot).snapshot; - - // The latest draft may have been cloned before a historical snapshot was - // retroactively tagged. Use the authoritative track-wide ledger here for - // the same reason _doBump does, otherwise preview can advertise a version - // that the subsequent bump rejects. - const versionHistory = await releaseHistoryService.getTrackWideVersionHistory(trackId); - const staged = snapshot.staged || []; - const existingMembers = snapshot.members || []; - - // Calculate what the next version would be (default minor bump) - const isAlreadyTagged = snapshot.version != null; - const nextMinor = isAlreadyTagged - ? null - : versionUtils.calculateNextVersion(versionHistory, 'minor'); - const nextMajor = isAlreadyTagged - ? null - : versionUtils.calculateNextVersion(versionHistory, 'major'); - - // Preview staged → members merge - let mergedMembersCount = existingMembers.length; - if (staged.length > 0 && !isAlreadyTagged) { - const stagedAsMembers = staged.map((s) => ({ - object_ref: s.object_ref, - object_modified: s.object_modified, - })); - - const policy = - (snapshot.config && - snapshot.config.promotion_conflicts && - snapshot.config.promotion_conflicts.staged_to_members) || - 'abort'; - - try { - const { merged } = conflictResolution.applyConflictPolicy( - existingMembers, - stagedAsMembers, - policy, - ); - mergedMembersCount = merged.length; - } catch (err) { - // If policy is 'abort' and conflicts exist, report it in the preview - return { - track_id: trackId, - snapshot_modified: snapshot.modified, - is_already_tagged: isAlreadyTagged, - current_version: snapshot.version, - next_version_minor: nextMinor, - next_version_major: nextMajor, - staged_count: staged.length, - members_count: existingMembers.length, - candidates_count: (snapshot.candidates || []).length, - conflicts: err.conflicts || [], // Include full conflicts array - }; - } - } +exports.releaseLatest = async function releaseLatest(trackId, options = {}) { + return commitPlan(await exports.planLatestRelease(trackId, options)); +}; - return { - track_id: trackId, - snapshot_modified: snapshot.modified, - is_already_tagged: isAlreadyTagged, - current_version: snapshot.version, - next_version_minor: nextMinor, - next_version_major: nextMajor, - staged_count: staged.length, - staged_to_promote: isAlreadyTagged ? 0 : staged.length, - members_count: existingMembers.length, - members_after_promotion: isAlreadyTagged ? existingMembers.length : mergedMembersCount, - candidates_count: (snapshot.candidates || []).length, - version_history: versionHistory, - }; +exports.releaseByModified = async function releaseByModified(trackId, modified, options = {}) { + return commitPlan(await exports.planReleaseByModified(trackId, modified, options)); }; diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 349d7e47..def1dff1 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -89,6 +89,16 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { ); } + async function releaseLatest(trackId) { + return postObject( + `/api/release-tracks/${trackId}/snapshots/latest/release`, + { + increment: 'minor', + }, + 200, + ); + } + function trackEntries(object) { return object.workspace.release_tracks || []; } @@ -166,13 +176,13 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { }); }); - it('bumping the track promotes staged backrefs to member/reviewed', async function () { + it('releasing the track promotes staged backrefs to member/reviewed', async function () { await postObject( `/api/release-tracks/${trackId}/candidates/promote`, { object_refs: [technique.stix.id] }, 200, ); - await postObject(`/api/release-tracks/${trackId}/bump`, { type: 'minor' }, 200); + await releaseLatest(trackId); const retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ @@ -205,7 +215,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { { object_refs: [technique.stix.id] }, 200, ); - await postObject(`/api/release-tracks/${componentTrackId}/bump`, { type: 'minor' }, 200); + await releaseLatest(componentTrackId); // Compose a virtual track over it and create a snapshot const virtual = await postObject('/api/release-tracks/new', { diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js new file mode 100644 index 00000000..825ced47 --- /dev/null +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -0,0 +1,244 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const versioningService = require('../../../services/release-tracks/versioning-service'); + +const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track release planning and commit API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function get(path, status = 200) { + return request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + } + + async function post(path, body, status = 200) { + return request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + } + + async function createTrack(name, type = 'standard') { + return (await post('/api/release-tracks/new', { name, type }, 201)).body; + } + + it('defaults to a non-persisting summary preview with type-oriented counts', async function () { + const track = await createTrack('Release Preview Summary'); + const preview = await get(`/api/release-tracks/${track.id}/snapshots/latest/release/preview`); + + expect(preview.body).toMatchObject({ + track_id: track.id, + type: 'standard', + source_snapshot_modified: track.modified, + version: '1.0', + releasable: true, + before: { members_count: 0, staged_count: 0, candidates_count: 0 }, + after: { members_count: 0, staged_count: 0, candidates_count: 0 }, + changes: { promoted_count: 0 }, + conflicts: [], + }); + + const unchanged = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(unchanged.body.version).toBeNull(); + expect(unchanged.body.version_history).toEqual([]); + }); + + it('renders the same plan as a workbench snapshot or STIX bundle', async function () { + const track = await createTrack('Release Preview Formats'); + const workbench = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=workbench&version=2.4`, + ); + expect(workbench.body.version).toBe('2.4'); + expect(workbench.body.version_history).toHaveLength(1); + expect(workbench.body.version_history[0].summary).toMatchObject({ + members_count: 0, + staged_count: 0, + candidates_count: 0, + }); + + const bundle = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=bundle&version=2.4&includeToc=false`, + ); + expect(bundle.body.type).toBe('bundle'); + expect(bundle.body.objects).toEqual([]); + + const unchanged = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(unchanged.body.version).toBeNull(); + }); + + it('commits the planned version', async function () { + const track = await createTrack('Release Commit'); + const preview = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?increment=major`, + ); + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + increment: 'major', + }); + + expect(released.body.version).toBe(preview.body.version); + expect(released.body.version_history.at(-1).summary).toMatchObject(preview.body.after); + }); + + it('resolves latest when the release request is handled', async function () { + const track = await createTrack('Release Latest Selector'); + const preview = await get(`/api/release-tracks/${track.id}/snapshots/latest/release/preview`); + const updated = await post(`/api/release-tracks/${track.id}/meta`, { + description: 'new latest', + }); + + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}); + expect(released.body.modified).toBe(updated.body.modified); + expect(released.body.modified).not.toBe(preview.body.source_snapshot_modified); + expect(released.body.version).toBe('1.0'); + }); + + it('previews and releases an explicitly selected historical snapshot', async function () { + const track = await createTrack('Historical Release'); + await post(`/api/release-tracks/${track.id}/meta`, { description: 'new latest' }); + const preview = await get( + `/api/release-tracks/${track.id}/snapshots/${track.modified}/release/preview?version=3.0`, + ); + expect(preview.body.source_snapshot_modified).toBe(track.modified); + const released = await post( + `/api/release-tracks/${track.id}/snapshots/${track.modified}/release`, + { + version: '3.0', + }, + ); + expect(released.body.modified).toBe(track.modified); + expect(released.body.version).toBe('3.0'); + }); + + it('orients virtual previews around members and quarantine', async function () { + const track = await createTrack('Virtual Release Preview', 'virtual'); + const preview = await get(`/api/release-tracks/${track.id}/snapshots/latest/release/preview`); + expect(preview.body.type).toBe('virtual'); + expect(preview.body.before).toEqual({ members_count: 0, quarantine_count: 0 }); + expect(preview.body.before).not.toHaveProperty('staged_count'); + expect(preview.body.before).not.toHaveProperty('candidates_count'); + }); + + it('reports blocking promotion conflicts in summaries and rejects materialization', async function () { + const revisionA = (await post('/api/techniques', buildTechnique('Release Conflict A'), 201)) + .body; + const revisionB = ( + await post('/api/techniques', buildTechnique('Release Conflict B', revisionA), 201) + ).body; + const track = await createTrack('Release Conflict'); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: revisionB.stix.id, modified: revisionB.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [revisionB.stix.id], + }); + + const summary = await get(`/api/release-tracks/${track.id}/snapshots/latest/release/preview`); + expect(summary.body.releasable).toBe(false); + expect(summary.body.conflicts).toHaveLength(1); + + await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=workbench`, + 409, + ); + await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { + increment: 'minor', + }, + 409, + ); + }); + + it('rejects ambiguous and legacy release inputs', async function () { + const track = await createTrack('Release Validation'); + expect(() => + versioningService.planRelease(track.id, track, [], { + increment: 'minor', + version: '2.0', + }), + ).toThrow('increment and version are mutually exclusive'); + + await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?increment=minor&version=2.0`, + 400, + ); + await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { + increment: 'minor', + version: '2.0', + }, + 400, + ); + await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { + type: 'minor', + dry_run: true, + }, + 400, + ); + }); + + it('reserves filesystemstore previews as not implemented', async function () { + const track = await createTrack('Release FilesystemStore Preview'); + await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=filesystemstore`, + 501, + ); + }); + + it('does not expose the removed bump endpoints', async function () { + const track = await createTrack('Removed Bump Route'); + await get(`/api/release-tracks/${track.id}/bump/preview`, 404); + await post(`/api/release-tracks/${track.id}/bump`, { type: 'minor' }, 404); + }); +}); diff --git a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js index 22e3b4d6..8cbbeb03 100644 --- a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js +++ b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js @@ -263,16 +263,18 @@ describe('Release-track cross-tier revision uniqueness', function () { expect(occurrences(latest, revisionB)).toEqual([]); }); - it('tags and repairs an exact staged/member duplicate instead of reporting a conflict', async function () { - const technique = await createTechnique('Tier Invariant Bump'); - const track = await createTrack('Tier Invariant Bump Track'); + it('releases and repairs an exact staged/member duplicate instead of reporting a conflict', async function () { + const technique = await createTechnique('Tier Invariant Release'); + const track = await createTrack('Tier Invariant Release Track'); await injectLatestSnapshot(track.id, { members: [memberEntry(technique)], staged: [stagedEntry(technique)], candidates: [candidateEntry(technique)], }); - const tagged = await post(`/api/release-tracks/${track.id}/bump`, { type: 'minor' }); + const tagged = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + increment: 'minor', + }); expect(tagged.version).toBe('1.0'); expect(occurrences(tagged, technique)).toEqual(['members']); diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index 0e9d3612..6fb1ac04 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -68,26 +68,28 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { trackA = createdA.id; const initialSnapshotModified = createdA.modified; trackATaggedSnapshot = await setMembers(trackA, [objectRevisionA]); - await post(`/api/release-tracks/${trackA}/bump`, { type: 'minor' }, 200); + await releaseLatest(trackA); // Remove the requested object from the latest state and tag that state. // The earlier tagged release must remain discoverable despite its current // backref disappearing. await setMembers(trackA, [otherObject]); - await post(`/api/release-tracks/${trackA}/bump`, { type: 'minor' }, 200); + await releaseLatest(trackA); // Retroactively tag the original empty draft. Its embedded history is // stale, so the track-wide version ledger must produce 1.2 rather than 1.0. await post( - `/api/release-tracks/${trackA}/snapshots/${initialSnapshotModified}/bump`, - { type: 'minor' }, + `/api/release-tracks/${trackA}/snapshots/${initialSnapshotModified}/release`, + { + increment: 'minor', + }, 200, ); const createdB = await createTrack('Releases By Object B'); trackB = createdB.id; await setMembers(trackB, [objectRevisionB]); - await post(`/api/release-tracks/${trackB}/bump`, { type: 'minor' }, 200); + await releaseLatest(trackB); // A tagged snapshot where the object is only a candidate must not match. const candidateOnly = await createTrack('Releases Candidate Only'); @@ -96,7 +98,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { { object_refs: [{ id: objectRevisionA.stix.id, modified: objectRevisionA.stix.modified }] }, 200, ); - await post(`/api/release-tracks/${candidateOnly.id}/bump`, { type: 'minor' }, 200); + await releaseLatest(candidateOnly.id); // Virtual tagged releases use the same direct-members semantics. const virtual = await post( @@ -109,7 +111,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { component_tracks: [{ track_id: trackB, resolution_strategy: 'latest_tagged', priority: 0 }], }); await post(`/api/release-tracks/${virtualTrack}/snapshots/create`, {}, 201); - await post(`/api/release-tracks/${virtualTrack}/bump`, { type: 'minor' }, 200); + await releaseLatest(virtualTrack); }); async function post(path, body, status) { @@ -161,6 +163,16 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { ); } + async function releaseLatest(trackId, increment = 'minor') { + return post( + `/api/release-tracks/${trackId}/snapshots/latest/release`, + { + increment, + }, + 200, + ); + } + it('returns historical tagged member occurrences across tracks and revisions', async function () { const response = await get(`/api/release-tracks/objects/${objectRevisionA.stix.id}/releases`); @@ -212,9 +224,14 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { 200, ); - const preview = await get(`/api/release-tracks/${trackA}/bump/preview`); - expect(preview.body.next_version_minor).toBe('1.3'); - expect(preview.body.next_version_major).toBe('2.0'); + const minor = await get( + `/api/release-tracks/${trackA}/snapshots/latest/release/preview?increment=minor`, + ); + const major = await get( + `/api/release-tracks/${trackA}/snapshots/latest/release/preview?increment=major`, + ); + expect(minor.body.version).toBe('1.3'); + expect(major.body.version).toBe('2.0'); }); it('supports type filtering, ordering, and pagination', async function () { diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 301914a7..5e057ca4 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,44 @@ # Release Track TODOs +## Harden release version selection + +- [x] Reject simultaneous `increment` and `version` selectors inside the + release planner, even when controller validation is bypassed. +- [x] Add regression coverage for planner-level mutual exclusivity. +- [x] Make exact, incremental, default, and ambiguous selection behavior + explicit in OpenAPI, user/developer docs, and Bruno. +- [x] Run the focused release-track spec, lint, and complete `npm test` suite. + The focused release spec passes (10 tests), lint passes, and the complete + backend suite passes (OpenAPI: 2, config: 21, API: 907, middleware: 24). + Targeted frontend Prettier and ESLint pass; TypeScript remains blocked by + the checkout's existing Angular dependency-resolution and unrelated type + errors. +- [x] Review the final diff and propose a conventional commit message. + +## Release command and unified previews + +- [x] Replace bump routes and symbols with explicit release operations for + latest and historical snapshots. +- [x] Implement one pure release planner shared by summary, workbench, bundle, + and commit paths. +- [x] Remove `dry_run`, rename version `type` to `increment`, and reject + conflicting version-selection inputs. +- [x] Keep release targeting semantics explicit: `latest` resolves at request + time, while `:modified` pins a specific snapshot; no client precondition is + required. +- [x] Add regression coverage for preview parity, non-persistence, conflicts, + formats, validation, historical releases, and removed bump routes. +- [x] Update OpenAPI, user/developer documentation, Bruno, and frontend + consumers. +- [x] Run focused tests and frontend checks, then the complete `npm test` + backend suite. + Focused release-track suites pass (49 tests), and the affected backref suite + passes again in isolation (23 tests). The complete backend suite passes on + retry. Targeted frontend formatting and ESLint pass; frontend Vitest and + TypeScript startup remain blocked by the checkout's existing + ESM/dependency-resolution errors. +- [x] Review the final diff and propose a conventional commit message. + ## Remove implicit latest-snapshot route - [x] Remove `GET /api/release-tracks/:id` while preserving track deletion. diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index 5e36751e..05d1ba5d 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -17,7 +17,7 @@ tracks follow that precedent but maintain the pointers event-driven. ## Why reconciliation instead of incremental updates Membership changes through many routes: add/remove candidates, review, -manual and auto promotion, demotion, bump (staged → members), member sync, +manual and auto promotion, demotion, release (staged → members), member sync, `updateContents`, track cloning, bundle import, snapshot deletion, and track deletion. Patching each route with a bespoke incremental backref update would be error-prone and would drift. @@ -36,7 +36,7 @@ snapshot-service.cloneSnapshot ┐ (every tier/config/metadata mutation, snapshot-service._cloneToNewTrack │ member sync, auto-promotion, snapshot-service.deleteSnapshot │ bundle import, updateContents, ...) snapshot-service.deleteTrack │ -versioning-service._doBump ┘ (staged → members via tagSnapshotInPlace) +versioning-service.releaseLatest/releaseByModified ┘ (staged → members via tagSnapshotInPlace) │ ▼ awaited EventBus.emit release-track::contents-changed { trackId, snapshot } │ snapshot = track's latest snapshot, @@ -59,7 +59,7 @@ documents. Both delegate to the shared logic in and an `includeRef` predicate. `createTrack` does not emit — a brand-new track's tiers are empty and nothing -can reference its ID yet. `bumpByModified` may tag an older snapshot; the bump +can reference its ID yet. `releaseByModified` may tag an older snapshot; the release path therefore re-reads the *latest* snapshot before emitting rather than using the tagged one. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index bf1c5400..1b4d21c9 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -131,7 +131,7 @@ Each release track snapshot will be tracked as an individual MongoDB Document in // Staged for next release staged: [ - // Objects that are reviewed (in THIS release track) and ready for next bump + // Objects that are reviewed (in THIS release track) and ready for next release // Automatically promoted from candidates when track-scoped status → "reviewed" { object_ref: "attack-pattern--ddd", @@ -196,7 +196,7 @@ Each release track snapshot will be tracked as an individual MongoDB Document in members_count: 3, // Objects in members promoted_count: 1, // Objects promoted from staged to members staged_count: 0, // Objects left in staged (if any) - candidate_count: 2 // Objects left in candidates (if any) + candidates_count: 2 // Objects left in candidates (if any) } } ] diff --git a/docs/developer/release-tracks/error-handling.md b/docs/developer/release-tracks/error-handling.md index d52f7470..cc553b03 100644 --- a/docs/developer/release-tracks/error-handling.md +++ b/docs/developer/release-tracks/error-handling.md @@ -2,7 +2,7 @@ ### AlreadyReleasedError -**Thrown when:** Attempting to bump a snapshot that already has `x_mitre_version` set. +**Thrown when:** Attempting to release a snapshot that already has `x_mitre_version` set. **HTTP Status:** 409 Conflict @@ -14,7 +14,7 @@ } ``` -**Solution:** Create a new snapshot by modifying the collection, then bump the new snapshot. +**Solution:** Create a new snapshot by modifying the collection, then release the new snapshot. ### InvalidVersionError @@ -22,7 +22,7 @@ - Explicit version is not valid MAJOR.MINOR format - Explicit version is not greater than the previous highest version -- Version bump would result in regression +- Version release would result in regression **HTTP Status:** 400 Bad Request diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 273c39c6..002c93f1 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -22,6 +22,10 @@ db.objects.createIndex({ 'workspace.workflow.status': 1 }); - Candidacy threshold must be valid enum value - Object version must exist before adding as candidate (validate `stix.id` and `stix.modified` exist) - Version pin (`object_modified`) is immutable once set for a tier entry +- Release version selection accepts either an `increment` or an explicit + `version`, never both. Controller validation returns 400 at the HTTP boundary, + and `version-utils.calculateNextVersion` repeats the invariant so internal + release-planning callers cannot silently choose one selector. ### Cross-tier revision enforcement @@ -32,7 +36,7 @@ track cloning uses the same normalizer. Tagging is the one in-place mutation, so `versioning-service` normalizes before the atomic tag update. This covers candidate adds, manual/automatic promotion, demotion, status transitions, candidate pin changes, member sync, direct content replacement, bundle -import, standard/virtual snapshot creation, and release bumps without +import, standard/virtual snapshot creation, and release commits without route-specific guards. Normalization keeps the first occurrence in the authoritative order @@ -52,7 +56,7 @@ policies remain responsible only for different revisions of one object. - Bulk operations should use batch updates - Event handlers should be async and non-blocking - Large collections (>10k objects) may need pagination -- Consider caching for `bump/preview` on large collections +- Consider caching for `release/preview` on large collections ### Snapshot history reads @@ -115,7 +119,7 @@ eventBus.emit('release-track:object-staged', { promotedBy: 'auto' // or user email }); -// When collection is bumped +// When collection is released eventBus.emit('release-track:released', { collectionId: 'x-mitre-collection--123', version: '1.2', diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 0d123298..e2f76119 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -23,7 +23,7 @@ This document provides the complete API reference for Release Tracks V2 (formerl - [Candidate Management](#candidate-management) - [Staged Objects](#staged-objects) - [Configuration](#configuration) -- [Preview & Dry Run](#preview--dry-run) +- [Release Previews](#release-previews) - [Version Pin Management](#version-pin-management) - [Virtual Release Tracks](#virtual-release-tracks) - [Query Variations](#query-variations) @@ -48,7 +48,7 @@ POST /api/release-tracks/new-from-bundle POST /api/release-tracks/import POST /api/release-tracks/:id/meta POST /api/release-tracks/:id/contents -POST /api/release-tracks/:id/bump +POST /api/release-tracks/:id/snapshots/latest/release POST /api/release-tracks/:id/clone DELETE /api/release-tracks/:id ``` @@ -60,7 +60,7 @@ GET /api/release-tracks/:id/snapshots GET /api/release-tracks/:id/snapshots/latest GET /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/meta -POST /api/release-tracks/:id/snapshots/:modified/bump +POST /api/release-tracks/:id/snapshots/:modified/release POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified ``` @@ -90,10 +90,10 @@ GET /api/release-tracks/:id/config PUT /api/release-tracks/:id/config ``` -### Preview & Dry Run +### Release Previews ``` -GET /api/release-tracks/:id/bump/preview +GET /api/release-tracks/:id/snapshots/latest/release/preview ``` ### Version Management @@ -472,29 +472,34 @@ Creates new snapshot with updated member objects. **This is intended for retroac } ``` -### Bump/Tag Latest Snapshot +### Release Latest Snapshot Converts the latest draft snapshot to a tagged release. Tags the snapshot in-place (does not create new snapshot). Dynamically sets `x_mitre_version` based on the request body options. - If `version` is provided, uses that exact version (must be `X.Y` format) -- If `type` is provided, calculates next version based on bump type -- If omitted, defaults to minor bump +- If `increment` is provided, calculates the next `major` or `minor` version +- `increment` and `version` are mutually exclusive; supplying both returns + `400 Bad Request` rather than choosing one +- If both are omitted, defaults to a minor release - If this is the first release, the version will be `1.0` ``` -POST /api/release-tracks/:id/bump +POST /api/release-tracks/:id/snapshots/latest/release ``` -**Request Body (optional):** +**Request Body:** ```json { - "type": "major" | "minor", // Defaults to "minor" if omitted - "version": "X.Y", // Alternative: explicit version - "dry_run": true // Optional: preview without persisting + "increment": "major" } ``` +Use `"version": "2.4"` instead of `increment` to select an explicit +`MAJOR.MINOR` version. The `latest` selector is resolved when the request is +handled. Use the `:modified` release endpoint when a caller needs to pin the +operation to a specific snapshot. + ### Clone Release Track From Latest Bootstraps a new `release-track` instance from an existing snapshot. @@ -581,15 +586,15 @@ Creates new snapshot with updated member objects. **This is intended for retroac **Request Body:** Same as [Update Contents](#update-contents) for latest snapshot. -### Bump/Tag Specific Snapshot +### Release/Tag Specific Snapshot Converts a specific draft snapshot to a tagged release. Tags snapshot in-place (does not create new snapshot). ``` -POST /api/release-tracks/:id/snapshots/:modified/bump +POST /api/release-tracks/:id/snapshots/:modified/release ``` -**Request Body:** Same as [Bump/Tag Latest Snapshot](#bumptag-latest-snapshot). +**Request Body:** Same as [Release Latest Snapshot](#release-latest-snapshot). ### Clone Specific Snapshot @@ -613,7 +618,7 @@ DELETE /api/release-tracks/:id/snapshots/:modified ### Add Candidates -Adds STIX objects as candidates to the latest draft snapshot. Each object is identified by its `stix.id` field, as well as (optionally) its `stix.modified` field. If `stix.modified` is omitted, the latest permutation of the relevant STIX object will be added. The candidacy reference will follow the latest version of the object until the moment the draft is converted to a release, at which point the reference will become locked to the specific permutation of the object that was considered "latest" at the time the release bump occurred. +Adds STIX objects as candidates to the latest draft snapshot. Each object is identified by its `stix.id` field, as well as (optionally) its `stix.modified` field. If `stix.modified` is omitted, the latest permutation of the relevant STIX object will be added. The candidacy reference will follow the latest version of the object until the moment the draft is converted to a release, at which point the reference will become locked to the specific permutation of the object that was considered "latest" at the time the release occurred. If the resolved revision (the same `stix.id` and `stix.modified`) is already present in any tier of the snapshot, the add is idempotently skipped. A newer @@ -829,67 +834,53 @@ PUT /api/release-tracks/:id/config --- -## Preview & Dry Run +## Release Previews -> **Note on `include` Query Parameter:** The `include` query parameter (used on snapshot retrieval endpoints to filter which tiers are returned) is **NOT supported** on bump preview or dry-run operations. Bump previews and dry-runs are intended to show the user exactly what _will_ happen when a bump occurs; ad-hoc filters would be misleading because they do not affect the actual release outcome. +Release previews and commits use the same planner. Preview requests never +persist data. Representation filters change only the rendered preview; they do +not change the release plan. ### Preview Next Release (Read-Only) -Shows a verbose diff of what will change in the next tagged release without creating any data. +Returns a before/after delta by default. Use the historical form +`/snapshots/:modified/release/preview` to target a specific draft. ``` -GET /api/release-tracks/:id/bump/preview +GET /api/release-tracks/:id/snapshots/latest/release/preview ``` **Query Parameters:** -- `format` - `bundle` | `filesystemstore` | `workbench` (default: `workbench`; `filesystemstore` is not yet implemented) +- `format` - `summary` | `workbench` | `bundle` | `filesystemstore` (default: + `summary`; `filesystemstore` returns 501) +- `increment` - `major` | `minor` (default: `minor`) +- `version` - explicit `MAJOR.MINOR` version; mutually exclusive with + `increment` +- Supplying both selectors returns `400 Bad Request`; the server never chooses + one selector over the other +- `include` - for `workbench`, selects returned tiers; for `bundle`, selects + additional non-member tiers +- `state`, `stixVersion`, `includeToc` - bundle representation options **Response Example:** ```json { - "current_version": "1.1", - "next_version": "1.2", - "release_preview": { - "will_include": [ - { - "ref": "attack-pattern--ddd", - "name": "New Technique XYZ", - "status": "reviewed", - "source": "staged" - } - ], - "will_exclude": [ - { - "ref": "attack-pattern--eee", - "name": "WIP Technique", - "status": "work-in-progress", - "reason": "Does not meet candidacy threshold" - } - ] - } -} -``` - -### Dry Run Bump (Returns Exact Output) - -Performs all bump logic and returns the exact release contents without persisting changes to the database. - -``` -POST /api/release-tracks/:id/bump -``` - -**Request Body:** - -```json -{ - "type": "minor", - "dry_run": true + "track_id": "release-track--123", + "type": "standard", + "source_snapshot_modified": "2024-01-15T16:20:00.000Z", + "version": "1.2", + "releasable": true, + "before": { "members_count": 10, "staged_count": 2, "candidates_count": 1 }, + "after": { "members_count": 12, "staged_count": 0, "candidates_count": 1 }, + "changes": { "promoted_count": 2 }, + "conflicts": [] } ``` -**Response:** Returns the exact snapshot that would be created, with all objects and metadata. +`format=workbench` returns the complete would-be persisted snapshot. +`format=bundle` returns its publication-ready STIX bundle. Thus “dry run” is +not a separate command: it is a release preview with the desired format. --- @@ -1222,11 +1213,9 @@ GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Not im GET /api/release-tracks/:id/snapshots/latest?include=all&format=workbench ``` -### Bump Operations (Preview & Dry Run) - -The `include` query parameter is **NOT supported** on bump preview or dry-run endpoints: - -- `GET /api/release-tracks/:id/bump/preview` — only `format` is supported -- `POST /api/release-tracks/:id/bump` with `dry_run: true` — only `format` is supported (via request body) +### Release preview representations -These endpoints are designed to show exactly what _will_ happen during a release bump. Allowing ad-hoc tier filters would be misleading because they do not affect the actual release outcome. +`format=summary` describes the release delta. `format=workbench` renders the +would-be snapshot for the UI, and `format=bundle` renders the publication +artifact. `include` and bundle filters affect only those representations, not +what the release command will persist. diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index aeb10d8d..7a2193b9 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -107,6 +107,6 @@ Release tracks are never blind to changes in the objects they pin: POST /api/release-tracks/:id/candidates → { tier: "candidates", status: "work-in-progress" } POST /api/release-tracks/:id/candidates/review → { tier: "candidates", status: "awaiting-review" } POST /api/release-tracks/:id/candidates/promote → { tier: "staged", status: "awaiting-review" } -POST /api/release-tracks/:id/bump → { tier: "members", status: "reviewed" } +POST /api/release-tracks/:id/snapshots/latest/release → { tier: "members", status: "reviewed" } DELETE /api/release-tracks/:id → entry removed ``` diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index dcf9843b..b525c111 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -189,6 +189,6 @@ GET /api/release-tracks/:id/snapshots/latest?format=bundle # FileSystemStore export is not implemented yet GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Returns HTTP 501 -# Dry run with detailed preview -GET /api/release-tracks/:id/bump/preview?format=workbench +# release preview with detailed preview +GET /api/release-tracks/:id/snapshots/latest/release/preview?format=workbench ``` diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index cf1110b5..94298761 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -262,7 +262,7 @@ When promoting objects between tiers, conflicts can occur if multiple versions o - **Demotion** back to candidates (`POST /api/release-tracks/:id/staged/demote`) - **Manual promotion** via REST API endpoint (e.g., `POST /api/release-tracks/:id/candidates/promote`) - **Auto-promotion** based on candidacy threshold (e.g., object status changes to `awaiting-review`) -- **Tagging/release operations** (e.g., `POST /api/release-tracks/:id/bump`) +- **Tagging/release operations** (e.g., `POST /api/release-tracks/:id/snapshots/latest/release`) Note: revision-sync enrollment (`config.member_sync`, strategy `track_latest`) resolves its overlaps through the `supplant` config rather than these policies — see [member-sync-strategies.md](../../developer/release-tracks/member-sync-strategies.md). @@ -372,7 +372,7 @@ Keep whichever version has the newer `modified` timestamp. [](./release-workflow.md#4-abort-taggingrelease-operations-only) **Only available for `staged_to_members` during tagging/release operations.** -If a conflict occurs during a tagging/release operation (`POST /api/release-tracks/:id/bump`), reject and abort the entire release. The snapshot will NOT be tagged, and no immutable snapshot will be created. +If a conflict occurs during a tagging/release operation (`POST /api/release-tracks/:id/snapshots/latest/release`), reject and abort the entire release. The snapshot will NOT be tagged, and no immutable snapshot will be created. **The error response will include ALL conflicting objects**, not just the first one encountered. This allows editors to see the full scope of conflicts that must be resolved before the release can proceed. @@ -383,8 +383,8 @@ If a conflict occurs during a tagging/release operation (`POST /api/release-trac // - staged: attack-pattern--T1234, modified: 2024-02-20 // Tagging request: -POST /api/release-tracks/release-track--123/bump -{ "type": "minor" } +POST /api/release-tracks/release-track--123/snapshots/latest/release +{ "increment": "minor" } // Result with abort: // ERROR Response: @@ -411,8 +411,8 @@ POST /api/release-tracks/release-track--123/bump // - staged: attack-pattern--T9999, modified: 2024-02-22 (no conflict) // Tagging request: -POST /api/release-tracks/release-track--123/bump -{ "type": "minor" } +POST /api/release-tracks/release-track--123/snapshots/latest/release +{ "increment": "minor" } // Result with abort - shows ALL conflicts: // ERROR Response: @@ -477,7 +477,7 @@ PUT /api/release-tracks/:id/config 1. **Production tracks**: Use `abort` for `staged_to_members` to prevent accidental overwrites during releases 2. **Development tracks**: Use `always_overwrite` or `prefer_latest` for faster iteration -3. **Review conflicts before releasing**: Always run `GET /api/release-tracks/:id/bump/preview` to identify potential conflicts +3. **Review conflicts before releasing**: Always run `GET /api/release-tracks/:id/snapshots/latest/release/preview` to identify potential conflicts 4. **Manual resolution**: When `abort` triggers, manually resolve conflicts before retrying the release ### 5. Viewing Latest Snapshot with All Tiers @@ -523,7 +523,7 @@ GET /api/release-tracks/:id/snapshots/latest?include=all "summary": { "members_count": 2, "staged_count": 1, - "candidate_count": 1, + "candidates_count": 1, "total_count": 4 } } @@ -534,49 +534,21 @@ GET /api/release-tracks/:id/snapshots/latest?include=all Compute a release preview, which outputs a verbose diff of what will change in the next release. **This endpoint will detect and report all conflicts** that would prevent the release from proceeding, allowing editors to resolve issues before attempting to tag. ``` -GET /api/release-tracks/:id/bump/preview +GET /api/release-tracks/:id/snapshots/latest/release/preview ``` **Response (success - no conflicts):** ```json { - "current_version": "1.1", - "next_version": "1.2", - "release_preview": { - "will_include": [ - { - "ref": "attack-pattern--aaa", - "modified": "2024-01-10T10:00:00Z", - "object_type": "attack-pattern", - "name": "Technique A", - "status": "reviewed", - "source": "members" - }, - { - "ref": "attack-pattern--ddd", - "modified": "2024-01-14T10:00:00Z", - "object_type": "attack-pattern", - "name": "New Technique XYZ", - "status": "reviewed", - "source": "staged" - } - ], - "will_exclude": [ - { - "ref": "attack-pattern--eee", - "modified": "2024-01-12T09:00:00Z", - "object_type": "attack-pattern", - "name": "WIP Technique", - "status": "work-in-progress", - "reason": "Object is work-in-progress, not meeting candidacy threshold" - } - ] - }, - "statistics": { - "total_objects": 3, - "included_objects": 2, - "excluded_objects": 1 - } + "track_id": "release-track--123", + "type": "standard", + "source_snapshot_modified": "2024-01-15T16:20:00.000Z", + "version": "1.2", + "releasable": true, + "before": { "members_count": 2, "staged_count": 3, "candidates_count": 1 }, + "after": { "members_count": 5, "staged_count": 0, "candidates_count": 1 }, + "changes": { "promoted_count": 3 }, + "conflicts": [] } ``` @@ -584,14 +556,13 @@ GET /api/release-tracks/:id/bump/preview ```json { "track_id": "release-track--123", - "snapshot_modified": "2024-01-15T16:20:00.000Z", - "is_already_tagged": false, - "current_version": null, - "next_version_minor": "1.2", - "next_version_major": "2.0", - "staged_count": 3, - "members_count": 2, - "candidates_count": 1, + "type": "standard", + "source_snapshot_modified": "2024-01-15T16:20:00.000Z", + "version": "1.2", + "releasable": false, + "before": { "members_count": 2, "staged_count": 3, "candidates_count": 1 }, + "after": { "members_count": 2, "staged_count": 3, "candidates_count": 1 }, + "changes": { "promoted_count": 0 }, "conflicts": [ { "object_ref": "attack-pattern--T1234", @@ -609,17 +580,16 @@ GET /api/release-tracks/:id/bump/preview **Note:** When the `staged_to_members` conflict policy is set to `abort` and conflicts are detected, the preview will include a `conflicts` array listing **all** conflicting objects, not just the first one encountered. -### 7. Bump with Staging +### 7. Release with Staging ``` -POST /api/collections/:id/bump +POST /api/release-tracks/:id/snapshots/latest/release ``` **Request:** ```json { - "type": "minor", - "dry_run": false // <-- optionally perform a dry run to preview the next release4 + "increment": "minor" } ``` @@ -726,9 +696,9 @@ POST /api/collections/collection--enterprise/candidates/review } # → Promoted to staged tier -# 5. Bump collection to v1.5 -POST /api/collections/collection--enterprise/bump -{ "type": "minor" } +# 5. Release collection to v1.5 +POST /api/collections/collection--enterprise/snapshots/latest/release +{ "increment": "minor" } # → Release track now at v1.5 # → Released tier: attack-pattern--T1234, modified: 2024-02-01T14:00:00Z @@ -852,12 +822,12 @@ POST /api/collections/collection--123/candidates/review # → auto-promoted to workspace.staged # 5. Preview the release -GET /api/release-tracks/collection--123/bump/preview +GET /api/release-tracks/collection--123/snapshots/latest/release/preview # → Shows attack-pattern--new1 will be included -# 6. Bump the collection -POST /api/collections/collection--123/bump -{ "type": "minor" } +# 6. Release the collection +POST /api/collections/collection--123/snapshots/latest/release +{ "increment": "minor" } # → attack-pattern--new1 moved to x_mitre_contents # → attack-pattern--new2 remains in candidates (still WIP) ``` @@ -881,12 +851,12 @@ POST /api/collections/collection--123/candidates/review # → All 50 auto-promoted to staged # Preview release -GET /api/release-tracks/collection--123/bump/preview +GET /api/release-tracks/collection--123/snapshots/latest/release/preview # → Shows all 50 will be included # Release -POST /api/collections/collection--123/bump -{ "type": "major" } +POST /api/collections/collection--123/snapshots/latest/release +{ "increment": "major" } # → All 50 moved to x_mitre_contents ``` @@ -911,9 +881,9 @@ POST /api/collections/collection--123/candidates/review "to": "reviewed" } -# January 25: Bump to v1.5 (freeze begins for v1.5 release) -POST /api/collections/collection--123/bump -{ "type": "minor" } +# January 25: Release to v1.5 (freeze begins for v1.5 release) +POST /api/collections/collection--123/snapshots/latest/release +{ "increment": "minor" } # v1.5 now released with: # - attack-pattern--A, modified: 2024-01-15T10:00:00Z # - attack-pattern--B, modified: 2024-01-15T11:00:00Z @@ -951,9 +921,9 @@ POST /api/collections/collection--123/candidates/attack-pattern--A/update-versio # - members (v1.5): attack-pattern--A, modified: 2024-01-15 (still frozen) # - candidates: attack-pattern--A, modified: 2024-02-10 (already in review) -# March 5: Bump to v1.6 -POST /api/collections/collection--123/bump -{ "type": "minor" } +# March 5: Release to v1.6 +POST /api/collections/collection--123/snapshots/latest/release +{ "increment": "minor" } # No bottleneck - work continued throughout v1.5 freeze ``` @@ -972,9 +942,9 @@ POST /api/collections/collection--dev/candidates { "object_refs": ["attack-pattern--exp1"] } # → Immediately promoted to staged (meets threshold) -# Bump immediately -POST /api/collections/collection--dev/bump -{ "type": "minor" } +# Release immediately +POST /api/collections/collection--dev/snapshots/latest/release +{ "increment": "minor" } # → WIP objects included in release ``` @@ -986,11 +956,11 @@ POST /api/collections/collection--dev/bump - **Team preview collections**: `candidacy_threshold: "awaiting-review"` - **Development collections**: `candidacy_threshold: "work-in-progress"` -### 2. Leverage Dry Run +### 2. Leverage release preview -Always preview releases before bumping: +Always preview releases before releasing: ```bash -GET /api/release-tracks/:id/bump/preview?format=workbench +GET /api/release-tracks/:id/snapshots/latest/release/preview?format=workbench ``` ### 3. Bulk Operations for Efficiency diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index cc3d2ada..f509f58c 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -66,7 +66,7 @@ GET /api/release-tracks/:id/snapshots/latest POST /api/release-tracks/:id/config POST /api/release-tracks/:id/meta POST /api/release-tracks/:id/clone -PUT /api/release-tracks/:id/bump +PUT /api/release-tracks/:id/snapshots/latest/release POST /api/release-tracks/:id/archive DELETE /api/release-tracks/:id @@ -80,7 +80,7 @@ POST /api/release-tracks/:id/snapshots/:modified/config POST /api/release-tracks/:id/snapshots/:modified/meta POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified -PUT /api/release-tracks/:id/snapshots/:modified/bump +PUT /api/release-tracks/:id/snapshots/:modified/release ``` ### 2. Git-Inspired Versioning @@ -149,26 +149,27 @@ workspace.config.candidacy_threshold = "work-in-progress" // Very permissive - **bundle** - Standard STIX 2.1 bundle (for publication) - **filesystemstore** - Planned STIX FileSystemStore directory structure; not implemented yet and returns HTTP 501 -### Dry Run + Preview +### Release previews -"Preview" will provide a verbose/detailed diff of what will change in the next release +The default format provides a before/after summary: ``` -GET /api/release-tracks/:id/bump/preview - ?format = bundle | workbench +GET /api/release-tracks/:id/snapshots/latest/release/preview + ?format=summary + &increment=minor ``` `format=filesystemstore` is reserved for future FileSystemStore export support and currently returns HTTP 501. -"Dry-run" will output the literal/exact contents of the would-be tagged release +Use `format=workbench` for the literal would-be snapshot or `format=bundle` +for its publication representation. Previewing never persists. + +Commit whichever snapshot is latest when the release request is handled: ``` -POST /api/release-tracks/:id/bump +POST /api/release-tracks/:id/snapshots/latest/release { - "type": "major", - "dry_run": true <-- IMPORTANT!! + "increment": "major" } ``` -Shows exactly what will be in the next release before bumping. - ### Bulk Operations ``` diff --git a/docs/user/release-tracks/terminology.md b/docs/user/release-tracks/terminology.md index 14234894..eb4bb1d3 100644 --- a/docs/user/release-tracks/terminology.md +++ b/docs/user/release-tracks/terminology.md @@ -148,7 +148,7 @@ The **tagging operation** marks an existing snapshot as a tagged release by assi **Characteristics:** - Version must be greater than all previous tagged releases (monotonically increasing) - Cannot tag a snapshot that is already tagged (throws `AlreadyReleasedError`) -- Supports automatic version calculation (MAJOR/MINOR bump) or explicit version +- Supports automatic version calculation (MAJOR/MINOR release) or explicit version **Examples:** - "Tag the latest snapshot as v1.5" diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 2c13c10b..95547b8c 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -32,7 +32,7 @@ A **tagged release** is a snapshot that has been marked as production-ready for **Note:** ATT&CK release tracks use a two-part versioning scheme (MAJOR.MINOR), not the three-part semver format (MAJOR.MINOR.PATCH). The patch component is not tracked in `version`. -Not all snapshots are tagged releases. Only snapshots explicitly tagged via the **bump** operation become tagged releases. +Not all snapshots are tagged releases. Only snapshots explicitly tagged via the **release** operation become tagged releases. **Example Timeline with Tagged Releases:** ``` @@ -43,7 +43,7 @@ id: "release-track--123", modified: "2024-01-02T14:30:00.000Z" version: null ← DRAFT RELEASE (work in progress) id: "release-track--123", modified: "2024-01-05T09:15:00.000Z" - version: "1.0" ← TAGGED RELEASE (via tagging operation) + version: "1.0" ← TAGGED RELEASE (via release operation) version_history: [{ version: "1.0", tagged_at: "2024-01-05T10:00:00Z", @@ -55,18 +55,21 @@ id: "release-track--123", modified: "2024-01-10T11:00:00.000Z" version: null ← DRAFT RELEASE (more development) id: "release-track--123", modified: "2024-01-15T16:20:00.000Z" - version: "1.1" ← TAGGED RELEASE (via tagging operation) + version: "1.1" ← TAGGED RELEASE (via release operation) version_history: [ { version: "1.1", tagged_at: "2024-01-15T17:00:00Z", tagged_by: "user@example.com", modified: "2024-01-15T16:20:00.000Z" }, { version: "1.0", tagged_at: "2024-01-05T10:00:00Z", tagged_by: "user@example.com", modified: "2024-01-05T09:15:00.000Z" } ] ``` -## The Tagging Operation +## The Release Operation -### What is "Tagging"? +### What Does Releasing Do? -The `tag` operation **tags an existing snapshot as a release** by assigning it a semantic version number (without the patch number). It does **NOT** create a new snapshot. +The `release` operation **tags an existing snapshot as a release** by assigning +it a semantic version number (without the patch number). It does **not** create +a new snapshot. `release` is the command; `tagged` describes the resulting +snapshot state. This is analogous to Git's tagging system: - Git commits = release track snapshots (identified by `modified` key) @@ -74,7 +77,7 @@ This is analogous to Git's tagging system: ### In-Place Tagging Strategy -When you tag a snapshot: +When you release a snapshot: 1. The **existing** snapshot is updated in-place 2. `version` is set to the new version @@ -89,30 +92,36 @@ When you tag a snapshot: ### Tagging Endpoints -#### Tag Latest Snapshot +#### Release Latest Snapshot ``` -POST /api/release-tracks/:id/bump +POST /api/release-tracks/:id/snapshots/latest/release ``` -Tags the most recent snapshot (highest `modified`) as a tagged release. +Releases the most recent snapshot (highest `modified`) as a tagged release. -**Request Body (optional):** +**Request Body:** ```json { - "type": "major" | "minor", // Default: "minor" - "version": "2.0" // Alternative: explicit version (MAJOR.MINOR format) + "increment": "major" } ``` +Use `"version": "2.0"` instead of `increment` for an explicit version. +The selectors are mutually exclusive: supplying both returns `400 Bad +Request`, and the server never chooses one over the other. Omitting both +version selectors defaults to a minor increment. The `latest` selector is +resolved when the release request is handled. Callers that need to pin the +operation to one snapshot should use the `:modified` endpoint. + **Examples:** 1. **Automatic version calculation:** ```bash # Current latest tagged release: 1.2 # Tag as: 1.3 (minor increment) -POST /api/release-tracks/release--123/bump +POST /api/release-tracks/release--123/snapshots/latest/release { - "type": "minor" + "increment": "minor" } ``` @@ -120,30 +129,31 @@ POST /api/release-tracks/release--123/bump ```bash # Current latest tagged release: 1.2 # Tag as: 2.0 (major increment) -POST /api/release-tracks/release--123/bump +POST /api/release-tracks/release--123/snapshots/latest/release { - "type": "major" + "increment": "major" } ``` 1. **Explicit version:** ```bash # Set specific version (must be greater than previous) -POST /api/release-tracks/release--123/bump +POST /api/release-tracks/release--123/snapshots/latest/release { "version": "2.0" } ``` -1. **Default behavior (no body):** +1. **Default version selection:** ```bash # Defaults to minor increment -POST /api/release-tracks/release--123/bump +POST /api/release-tracks/release--123/snapshots/latest/release +{} ``` -#### Tag Specific Snapshot +#### Release Specific Snapshot ``` -POST /api/release-tracks/:id/snapshots/:modified/bump +POST /api/release-tracks/:id/snapshots/:modified/release ``` Tags a specific snapshot as a tagged release. Can tag retroactively, (i.e., a non-latest snapshot can be tagged), granted no [versioning rules](#versioning-rules) are violated. @@ -177,4 +187,4 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema For release tracks with no prior tagged releases: - The first tag sets `version: "1.0"` (regardless of increment type) -- Or you can specify an explicit version like `"0.1"` \ No newline at end of file +- Or you can specify an explicit version like `"0.1"` diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 813a4645..758b3eb0 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -529,13 +529,13 @@ GET /api/release-tracks/:id/snapshots/:modified?format=workbench&include=all Once reviewed, explicitly tag the draft snapshot: ```bash -POST /api/release-tracks/:id/snapshots/:modified/bump +POST /api/release-tracks/:id/snapshots/:modified/release ``` **Request:** ```json { - "type": "major", // or "minor", or explicit "version": "14.0" + "increment": "major", // or "minor", or explicit "version": "14.0" } ``` @@ -841,13 +841,13 @@ GET /api/release-tracks/:id/snapshots/preview ### Tag Virtual Snapshot ```bash -POST /api/release-tracks/:id/snapshots/:modified/bump +POST /api/release-tracks/:id/snapshots/:modified/release ``` **Request:** ```json { - "type": "major" + "increment": "major" } ``` @@ -984,7 +984,7 @@ POST /api/release-tracks/release-track--uuid-1/candidates } # Tag initial release -POST /api/release-tracks/release-track--uuid-1/bump +POST /api/release-tracks/release-track--uuid-1/snapshots/latest/release { "version": "1.0" } ``` @@ -1024,7 +1024,7 @@ POST /api/release-tracks/release-track--uuid-virtual/snapshots/create GET /api/release-tracks/release-track--uuid-virtual/snapshots/:modified # Tag as Enterprise v14.0 -POST /api/release-tracks/release-track--uuid-virtual/snapshots/:modified/bump +POST /api/release-tracks/release-track--uuid-virtual/snapshots/:modified/release { "version": "14.0" } ``` @@ -1132,7 +1132,7 @@ GET /api/release-tracks/:id/snapshots/:modified?format=workbench GET /api/release-tracks/:id/snapshots/:modified?format=bundle # Tag only when satisfied -POST /api/release-tracks/:id/snapshots/:modified/bump +POST /api/release-tracks/:id/snapshots/:modified/release ``` ### 2. Use Scheduled Snapshots for Consistency diff --git a/docs/user/release-tracks/workflow-examples.md b/docs/user/release-tracks/workflow-examples.md index 8dec6910..d3e0c0fc 100644 --- a/docs/user/release-tracks/workflow-examples.md +++ b/docs/user/release-tracks/workflow-examples.md @@ -19,8 +19,8 @@ POST /api/release-tracks/release--123/meta # Creates: snapshot 3, x_mitre_version: null # 4. Ready for first release - tag as v1.0 -POST /api/release-tracks/release--123/bump -{ "type": "major" } +POST /api/release-tracks/release--123/snapshots/latest/release +{ "increment": "major" } # Updates: snapshot 3, x_mitre_version: "1.0" (IN-PLACE) # 5. Continue development @@ -29,8 +29,8 @@ POST /api/release-tracks/release--123/contents # Creates: snapshot 4, x_mitre_version: null # 6. Minor release -POST /api/release-tracks/release--123/bump -{ "type": "minor" } +POST /api/release-tracks/release--123/snapshots/latest/release +{ "increment": "minor" } # Updates: snapshot 4, x_mitre_version: "1.1" (IN-PLACE) # 7. More changes @@ -39,8 +39,8 @@ POST /api/release-tracks/release--123/contents # Creates: snapshot 5, x_mitre_version: null # 8. Another minor release -POST /api/release-tracks/release--123/bump -{ "type": "minor" } +POST /api/release-tracks/release--123/snapshots/latest/release +{ "increment": "minor" } # Updates: snapshot 5, x_mitre_version: "1.2" (IN-PLACE) ``` @@ -64,10 +64,10 @@ POST /api/collections/collection--456/contents # snapshot 4 POST /api/collections/collection--456/contents # snapshot 5 # Only tag snapshots 2 and 5 as releases -POST /api/collections/collection--456/modified//bump +POST /api/collections/collection--456/modified//snapshots/latest/release { "version": "1.0" } -POST /api/collections/collection--456/bump # Latest = snapshot 5 +POST /api/collections/collection--456/snapshots/latest/release # Latest = snapshot 5 { "version": "1.1" } ``` @@ -86,12 +86,12 @@ This mirrors Git's ability to tag any commit, not just the latest. ```bash # Tag latest snapshot -POST /api/collections/collection--789/bump +POST /api/collections/collection--789/snapshots/latest/release { "version": "1.0" } # Success: snapshot tagged as v1.0 -# Attempt to bump the same snapshot again -POST /api/collections/collection--789/bump +# Attempt to release the same snapshot again +POST /api/collections/collection--789/snapshots/latest/release { "version": "1.1" } # Error: AlreadyReleasedError - "This snapshot has already been tagged as version 1.0" @@ -100,8 +100,8 @@ POST /api/collections/collection--789/contents { "x_mitre_contents": [...] } # Creates new snapshot -# Now bump the new snapshot -POST /api/collections/collection--789/bump +# Now release the new snapshot +POST /api/collections/collection--789/snapshots/latest/release { "version": "1.1" } # Success: new snapshot tagged as v1.1 ``` From 341d80dc213f6c9db0ed6196de73c1b3c930bb3c Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:57:21 -0400 Subject: [PATCH 19/55] feat(release-tracks): complete virtual release workflow Scope virtual composition and draft creation beneath the virtual namespace, remove the standalone composition preview, and share persisted snapshot release previews across track types. Add chronological virtual release summaries, functional domain filters, relationship-complete bundle exports, regression coverage, OpenAPI updates, and lifecycle documentation. --- .../definitions/components/release-tracks.yml | 4 +- app/api/definitions/openapi.yml | 11 +- .../paths/release-tracks-paths.yml | 47 ++--- app/controllers/release-tracks-controller.js | 16 +- app/lib/event-constants.js | 4 + .../release-tracks/release-track-schemas.js | 4 +- app/lib/stix-bundle-relationships.js | 58 ++++++ app/repository/relationships-repository.js | 6 +- .../release-track-dynamic.repository.js | 19 ++ app/routes/release-tracks-routes.js | 12 +- app/services/release-tracks/export-service.js | 40 ++++- .../release-tracks/release-tracks-service.js | 4 - .../release-tracks/versioning-service.js | 103 ++++++++++- .../release-tracks/virtual-track-service.js | 153 ++++++++-------- app/services/stix/attack-objects-service.js | 17 ++ app/services/stix/relationships-service.js | 22 +++ app/services/stix/stix-bundles-service.js | 40 +---- .../release-tracks-backrefs.spec.js | 4 +- .../release-tracks-bundle.spec.js | 60 +++++++ .../release-tracks-release.spec.js | 165 +++++++++++++++++- .../release-tracks/releases-by-object.spec.js | 4 +- .../virtual-domain-filters.spec.js | 160 +++++++++++++++++ docs/developer/TODO.md | 108 +++++++++++- .../developer/release-tracks/bundle-export.md | 35 +++- docs/developer/release-tracks/entities.md | 3 +- .../release-tracks/implementation-notes.md | 29 +++ docs/user/release-tracks/api-reference.md | 95 ++++++---- docs/user/release-tracks/release-workflow.md | 4 +- docs/user/release-tracks/virtual-tracks.md | 100 ++++++----- 29 files changed, 1043 insertions(+), 284 deletions(-) create mode 100644 app/lib/stix-bundle-relationships.js create mode 100644 app/tests/api/release-tracks/virtual-domain-filters.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 9f815f47..6687d054 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -336,7 +336,7 @@ components: type: array items: type: string - description: 'Only include objects from these ATT&CK domains' + description: 'Only include exact pinned object revisions whose x_mitre_domains intersects these ATT&CK domains. Primary matrices fall back to external_references.external_id. Accepts enterprise/mobile/ics and their -attack forms.' version-history-entry: type: object @@ -437,7 +437,7 @@ components: description: 'When the track metadata was last updated' snapshot_schedule: nullable: true - description: 'Automated snapshot schedule (virtual tracks only)' + description: 'Stored snapshot schedule metadata for virtual tracks. Automated execution is not yet implemented.' $ref: '#/components/schemas/snapshot-schedule' tagged-release-reference: diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index e81fd13c..c8933474 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -385,14 +385,11 @@ paths: /api/release-tracks/{id}/objects/{objectRef}/versions: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1objects~1{objectRef}~1versions' - /api/release-tracks/{id}/composition: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1composition' + /api/release-tracks/{id}/virtual/composition: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1composition' - /api/release-tracks/{id}/snapshots/create: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1create' - - /api/release-tracks/{id}/snapshots/preview: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1preview' + /api/release-tracks/{id}/virtual/snapshots/create: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1snapshots~1create' /api/release-tracks/{id}/snapshots: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 78933c9f..3d3526dc 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -405,6 +405,9 @@ paths: Plan without persisting. Summary is the default; workbench and bundle render the complete would-be release. `increment` and `version` are mutually exclusive; omitting both defaults to a minor increment. + Standard summaries show staged-to-members promotion. Virtual summaries + compare the persisted draft with its chronologically preceding tagged + release; composition is never recomputed. tags: - 'Release Tracks' parameters: @@ -774,12 +777,13 @@ paths: # ============================================================================= # Virtual track operations # ============================================================================= - /api/release-tracks/{id}/composition: + /api/release-tracks/{id}/virtual/composition: put: summary: 'Update virtual track composition' operationId: 'release-tracks-composition-update' description: | - Update which component tracks a virtual track aggregates. + Update which component tracks a virtual track aggregates. This + operation is available only for tracks whose type is `virtual`. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -792,15 +796,18 @@ paths: responses: '200': description: 'Composition updated successfully' - '501': - description: 'Not yet implemented' + '400': + description: 'Track is not virtual or composition is invalid' - /api/release-tracks/{id}/snapshots/create: + /api/release-tracks/{id}/virtual/snapshots/create: post: summary: 'Create a virtual track snapshot' operationId: 'release-tracks-virtual-snapshot-create' description: | - Resolve component tracks and create a new virtual snapshot. + Resolve the configured component releases and persist a new virtual + draft snapshot. The draft must subsequently be reviewed and explicitly + released through the shared snapshot release endpoints. This operation + is available only for tracks whose type is `virtual`. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -813,28 +820,8 @@ paths: responses: '201': description: 'Virtual snapshot created successfully' - '501': - description: 'Not yet implemented' - - /api/release-tracks/{id}/snapshots/preview: - get: - summary: 'Preview a virtual track snapshot' - operationId: 'release-tracks-virtual-snapshot-preview' - description: | - Compute what a virtual snapshot would contain without persisting it. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - responses: - '200': - description: 'Virtual snapshot preview generated' - '501': - description: 'Not yet implemented' + '400': + description: 'Track is not virtual or cannot resolve its composition' # ============================================================================= # Snapshot-specific operations @@ -1224,7 +1211,9 @@ paths: operationId: 'release-tracks-preview-release-by-modified' description: | Plan without persisting. `increment` and `version` are mutually - exclusive; omitting both defaults to a minor increment. + exclusive; omitting both defaults to a minor increment. For a virtual + draft, compare against the latest tagged snapshot whose modified + timestamp precedes this selected snapshot; never recompute composition. tags: - 'Release Tracks' parameters: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index f7aa1447..e652124f 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -971,7 +971,7 @@ exports.listObjectVersions = async function listObjectVersions(req, res, next) { // Virtual track operations // ============================================================================= -/** PUT /api/release-tracks/:id/composition */ +/** PUT /api/release-tracks/:id/virtual/composition */ exports.updateComposition = async function updateComposition(req, res, next) { try { const bodyResult = updateCompositionBodySchema.safeParse(req.body); @@ -997,7 +997,7 @@ exports.updateComposition = async function updateComposition(req, res, next) { } }; -/** POST /api/release-tracks/:id/snapshots/create */ +/** POST /api/release-tracks/:id/virtual/snapshots/create */ exports.createVirtualSnapshot = async function createVirtualSnapshot(req, res, next) { try { const bodyResult = createVirtualSnapshotBodySchema.safeParse(req.body || {}); @@ -1021,15 +1021,3 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(req, res, n return next(err); } }; - -/** GET /api/release-tracks/:id/snapshots/preview */ -exports.previewVirtualSnapshot = async function previewVirtualSnapshot(req, res, next) { - try { - const result = await releaseTracksService.previewVirtualSnapshot(req.params.id); - logger.debug(`Success: Generated virtual snapshot preview for track ${req.params.id}`); - return res.status(200).send(result); - } catch (err) { - logger.error('Failed to preview virtual snapshot: ' + err); - return next(err); - } -}; diff --git a/app/lib/event-constants.js b/app/lib/event-constants.js index fdfe22d9..a83a18b5 100644 --- a/app/lib/event-constants.js +++ b/app/lib/event-constants.js @@ -159,6 +159,10 @@ module.exports = Object.freeze({ // Validation VALIDATION_BYPASS_CHECK_REQUESTED: 'validation-bypass::check-requested', + // Cross-service reads used by release-track composition/export + ATTACK_OBJECT_REVISIONS_REQUESTED: 'attack-object::revisions-requested', + BUNDLE_RELATIONSHIPS_REQUESTED: 'relationship::bundle-requested', + // Release Tracks // Emitted after any persisted change to a release track's current (latest) // snapshot. Payload: { trackId, snapshot } where snapshot is the track's diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 0ecd1a7d..b6a51b52 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -408,10 +408,10 @@ const updateConfigBodySchema = z.object({ member_sync: memberSyncConfigSchema.optional(), }); -/** PUT /release-tracks/:id/composition */ +/** PUT /release-tracks/:id/virtual/composition */ const updateCompositionBodySchema = compositionSchema; -/** POST /release-tracks/:id/snapshots/create */ +/** POST /release-tracks/:id/virtual/snapshots/create */ const createVirtualSnapshotBodySchema = z .object({ description: z.string().optional(), diff --git a/app/lib/stix-bundle-relationships.js b/app/lib/stix-bundle-relationships.js new file mode 100644 index 00000000..28dc1493 --- /dev/null +++ b/app/lib/stix-bundle-relationships.js @@ -0,0 +1,58 @@ +'use strict'; + +/** + * Relationship patterns that are intentionally excluded from published + * ATT&CK bundles. + */ +const DEPRECATED_PATTERNS = Object.freeze([ + { + type: 'relationship', + conditions: { + relationship_type: 'detects', + sourceTypePrefix: 'x-mitre-data-component--', + }, + reason: 'Data components cannot detect techniques in v17+ (only detection strategies can)', + }, +]); + +function isDeprecatedPattern(stixObject) { + return DEPRECATED_PATTERNS.some((pattern) => { + if (stixObject.type !== pattern.type) return false; + + return Object.entries(pattern.conditions).every(([key, value]) => { + if (key === 'sourceTypePrefix') { + return stixObject.source_ref?.startsWith(value); + } + return stixObject[key] === value; + }); + }); +} + +function relationshipIsActive(relationship) { + return !relationship.stix.x_mitre_deprecated && !relationship.stix.revoked; +} + +/** + * Return relationships that are publishable and whose endpoints are both + * present in the selected object set. + * + * @param {Array} relationships - Lean relationship documents + * @param {Set|Map} selectedObjects - Selected STIX IDs + * @returns {Array} Publishable relationship documents + */ +function selectRelationshipsForBundle(relationships, selectedObjects) { + return relationships.filter( + (relationship) => + relationshipIsActive(relationship) && + !isDeprecatedPattern(relationship.stix) && + selectedObjects.has(relationship.stix.source_ref) && + selectedObjects.has(relationship.stix.target_ref), + ); +} + +module.exports = { + DEPRECATED_PATTERNS, + isDeprecatedPattern, + relationshipIsActive, + selectRelationshipsForBundle, +}; diff --git a/app/repository/relationships-repository.js b/app/repository/relationships-repository.js index 24e1ef73..a4af9094 100644 --- a/app/repository/relationships-repository.js +++ b/app/repository/relationships-repository.js @@ -83,7 +83,6 @@ class RelationshipsRepository extends BaseRepository { async retrieveAllForBundle(options) { try { - // Build query exactly as original - NO domain filter const query = {}; if (!options.includeRevoked) { query['stix.revoked'] = { $in: [null, false] }; @@ -96,8 +95,11 @@ class RelationshipsRepository extends BaseRepository { ? { $in: options.state } : options.state; } + if (Array.isArray(options.objectRefs)) { + query['stix.source_ref'] = { $in: options.objectRefs }; + query['stix.target_ref'] = { $in: options.objectRefs }; + } - // Use exact same aggregation as original const aggregation = [ { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 80af68e9..6ddb1a98 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -81,6 +81,25 @@ class ReleaseTrackDynamicRepository { } } + async getLatestTaggedSnapshotBefore(trackId, modified) { + try { + const Model = this._getModel(trackId); + return await Model.findOne({ + id: trackId, + version: { $type: 'string' }, + modified: { $lt: modified }, + }) + .sort({ modified: -1 }) + .lean() + .exec(); + } catch (err) { + if (err.name === 'CastError') { + throw new BadlyFormattedParameterError({ parameterName: 'modified' }); + } + throw new DatabaseError(err); + } + } + async getSnapshotByVersion(trackId, version) { try { const Model = this._getModel(trackId); diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index f25fb08e..d2f93e64 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -232,15 +232,7 @@ router ); router - .route('/release-tracks/:id/snapshots/preview') - .get( - authn.authenticate, - authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), - releaseTracksController.previewVirtualSnapshot, - ); - -router - .route('/release-tracks/:id/snapshots/create') + .route('/release-tracks/:id/virtual/snapshots/create') .post( authn.authenticate, authz.requireRole(authz.editorOrHigher), @@ -309,7 +301,7 @@ router // ============================================================================= router - .route('/release-tracks/:id/composition') + .route('/release-tracks/:id/virtual/composition') .put( authn.authenticate, authz.requireRole(authz.editorOrHigher), diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index f5a2cef4..7d9cdec0 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -21,6 +21,9 @@ const config = require('../../config/config'); const types = require('../../lib/types'); const logger = require('../../lib/logger'); const linkById = require('../../lib/linkById'); +const EventBus = require('../../lib/event-bus'); +const Events = require('../../lib/event-constants'); +const { selectRelationshipsForBundle } = require('../../lib/stix-bundle-relationships'); const { bundleTransformSchema, workbenchTransformSchema, @@ -208,6 +211,31 @@ async function fetchSupportingObjects(documents) { return supportingObjects; } +/** + * Fetch the latest publishable relationships connecting selected bundle + * objects. Relationship revisions remain indirect export-time content rather + * than snapshot members. + * + * @param {Array} documents - Hydrated selected object documents + * @returns {Promise>} + */ +async function fetchRelationships(documents) { + const selectedIds = new Set(documents.map((document) => document.stix.id)); + if (selectedIds.size === 0) return []; + + const results = await EventBus.emit(Events.BUNDLE_RELATIONSHIPS_REQUESTED, { + objectRefs: [...selectedIds], + }); + const relationships = results?.[0]; + if (!relationships) { + throw new Error('Unable to retrieve relationships for release-track bundle export'); + } + + return selectRelationshipsForBundle(relationships, selectedIds).filter( + (relationship) => !selectedIds.has(relationship.stix.id), + ); +} + /** * Convert LinkById tags (e.g. "(LinkById: T1234)") in descriptions to * markdown citations, preferring objects already in the export before @@ -284,9 +312,10 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd * 1. Select tier entries — members always; staged/candidates via * options.include, narrowed by options.state * 2. Hydrate entries into full documents - * 3. Append referenced identities and marking definitions - * 4. Convert LinkById tags to markdown citations - * 5. Assemble the bundle (STIX version conformance + optional TOC) via the + * 3. Append current relationships whose endpoints are both selected + * 4. Append referenced identities and marking definitions + * 5. Convert LinkById tags to markdown citations + * 6. Assemble the bundle (STIX version conformance + optional TOC) via the * Zod transform schema * * @param {Object} snapshot - The raw snapshot document from the dynamic repo @@ -302,8 +331,9 @@ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options if (format === 'bundle') { const entries = collectBundleEntries(snapshot, options); const hydratedObjects = await exports.hydrateMembers(entries); - const supportingObjects = await fetchSupportingObjects(hydratedObjects); - const allObjects = [...hydratedObjects, ...supportingObjects]; + const relationships = await fetchRelationships(hydratedObjects); + const supportingObjects = await fetchSupportingObjects([...hydratedObjects, ...relationships]); + const allObjects = [...hydratedObjects, ...relationships, ...supportingObjects]; await convertLinkByIdTags(allObjects); return exports.formatAsBundle(snapshot, allObjects, { diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 03a9ec78..4aa61e4b 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -366,10 +366,6 @@ exports.createVirtualSnapshot = function createVirtualSnapshot(trackId, options) return virtualTrackService.createVirtualSnapshot(trackId, options); }; -exports.previewVirtualSnapshot = function previewVirtualSnapshot(trackId) { - return virtualTrackService.previewVirtualSnapshot(trackId); -}; - // ----------------------------------------------------------------------------- // Object versions (Phase 2 → standard-track-service) // ----------------------------------------------------------------------------- diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index ef9cb2df..c80a42ae 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -32,6 +32,52 @@ function tierCounts(snapshot) { }; } +function memberRevisions(snapshot) { + const revisionsByObject = new Map(); + for (const member of snapshot?.members || []) { + const revisions = revisionsByObject.get(member.object_ref) || new Set(); + revisions.add(iso(member.object_modified)); + revisionsByObject.set(member.object_ref, revisions); + } + return revisionsByObject; +} + +function sameRevisions(left, right) { + if (left.size !== right.size) return false; + for (const revision of left) { + if (!right.has(revision)) return false; + } + return true; +} + +function virtualReleaseChanges(previousSnapshot, draftSnapshot) { + const previous = memberRevisions(previousSnapshot); + const draft = memberRevisions(draftSnapshot); + let newCount = 0; + let updatedCount = 0; + let removedCount = 0; + + for (const [objectRef, revisions] of draft) { + const previousRevisions = previous.get(objectRef); + if (!previousRevisions) { + newCount++; + } else if (!sameRevisions(revisions, previousRevisions)) { + updatedCount++; + } + } + + for (const objectRef of previous.keys()) { + if (!draft.has(objectRef)) removedCount++; + } + + return { + new_count: newCount, + updated_count: updatedCount, + removed_count: removedCount, + quarantined_count: (draftSnapshot.quarantine || []).length, + }; +} + /** * Build the complete release plan without reading or writing external state. * @@ -40,9 +86,17 @@ function tierCounts(snapshot) { * @param {Array} versionHistory * @param {Object} options * @param {Date} now + * @param {Object|null} previousTaggedSnapshot * @returns {Object} */ -function planRelease(trackId, sourceSnapshot, versionHistory, options = {}, now = new Date()) { +function planRelease( + trackId, + sourceSnapshot, + versionHistory, + options = {}, + now = new Date(), + previousTaggedSnapshot = null, +) { if (sourceSnapshot.version != null) { throw new AlreadyReleasedError(sourceSnapshot.version); } @@ -56,7 +110,12 @@ function planRelease(trackId, sourceSnapshot, versionHistory, options = {}, now ); versionUtils.validateVersionProgression(version, versionHistory); - const before = tierCounts(snapshot); + const isVirtual = snapshot.type === 'virtual'; + const before = isVirtual + ? previousTaggedSnapshot + ? tierCounts(previousTaggedSnapshot) + : { members_count: 0, quarantine_count: 0 } + : tierCounts(snapshot); const staged = snapshot.type === 'standard' ? snapshot.staged || [] : []; const existingMembers = snapshot.members || []; let mergedMembers = existingMembers; @@ -97,6 +156,11 @@ function planRelease(trackId, sourceSnapshot, versionHistory, options = {}, now ...(snapshot.type === 'standard' ? { staged: [] } : {}), }; const after = tierCounts(afterSnapshot); + const changes = isVirtual + ? virtualReleaseChanges(previousTaggedSnapshot, afterSnapshot) + : { + promoted_count: blockingError ? 0 : staged.length, + }; const versionHistoryEntry = { version, tagged_at: now, @@ -129,19 +193,39 @@ function planRelease(trackId, sourceSnapshot, versionHistory, options = {}, now source_snapshot_modified: iso(sourceSnapshot.modified), version, releasable: !blockingError, + ...(isVirtual + ? { + previous_release: previousTaggedSnapshot + ? { + version: previousTaggedSnapshot.version, + modified: iso(previousTaggedSnapshot.modified), + } + : null, + } + : {}), before, after: blockingError ? before : after, - changes: { - promoted_count: blockingError ? 0 : staged.length, - }, + changes, conflicts: blockingError?.conflicts || [], }, }; } async function planLoadedSnapshot(trackId, snapshot, options) { - const versionHistory = await releaseHistoryService.getTrackWideVersionHistory(trackId); - return planRelease(trackId, snapshot, versionHistory, options); + const [versionHistory, previousTaggedSnapshot] = await Promise.all([ + releaseHistoryService.getTrackWideVersionHistory(trackId), + snapshot.type === 'virtual' + ? dynamicRepo.getLatestTaggedSnapshotBefore(trackId, snapshot.modified) + : Promise.resolve(null), + ]); + return planRelease( + trackId, + snapshot, + versionHistory, + options, + new Date(), + previousTaggedSnapshot, + ); } async function commitPlan(plan) { @@ -177,6 +261,11 @@ async function commitPlan(plan) { } exports.planRelease = planRelease; +exports._private = { + memberRevisions, + sameRevisions, + virtualReleaseChanges, +}; exports.planLatestRelease = async function planLatestRelease(trackId, options = {}) { const snapshot = await snapshotService.getLatestSnapshot(trackId); diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 3669b6db..5b2aa7e4 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -9,7 +9,7 @@ // Virtual tracks aggregate content from multiple standard tracks by: // 1. Resolving each component track to a specific tagged snapshot // 2. Collecting members from each resolved snapshot -// 3. Applying per-component filters (object_types) +// 3. Applying per-component filters (object_types and domains) // 4. Deduplicating across all components // 5. Persisting the result as a new draft snapshot // @@ -20,6 +20,8 @@ const snapshotService = require('./snapshot-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); const deduplicationStrategies = require('../../lib/release-tracks/deduplication-strategies'); +const EventBus = require('../../lib/event-bus'); +const Events = require('../../lib/event-constants'); const logger = require('../../lib/logger'); const { BadRequestError, @@ -139,15 +141,46 @@ async function resolveComponentSnapshot(component) { } /** - * Apply object_types filter to a list of member entries. + * Normalize public domain filter names to their STIX x_mitre_domains values. + * + * @param {string} domain + * @returns {string} + */ +function normalizeDomain(domain) { + return domain.endsWith('-attack') ? domain : `${domain}-attack`; +} + +/** + * Read explicit domains, with the established matrix fallback used by the + * legacy bundle exporter. Primary matrices identify their domain through the + * ATT&CK external reference rather than x_mitre_domains. + * + * @param {Object} stixObject + * @returns {Array} + */ +function getObjectDomains(stixObject) { + if (Array.isArray(stixObject.x_mitre_domains)) { + return stixObject.x_mitre_domains; + } + if (stixObject.type === 'x-mitre-matrix') { + return (stixObject.external_references || []) + .map((reference) => reference.external_id) + .filter((externalId) => typeof externalId === 'string' && externalId.endsWith('-attack')); + } + return []; +} + +/** + * Apply object type and domain filters to a list of member entries. * Filters by extracting the STIX type prefix from the object_ref * (e.g., "attack-pattern" from "attack-pattern--uuid"). * * @param {Array} members - Member entries with object_ref * @param {Object} [filters] - { object_types?: string[], domains?: string[] } + * @param {Map>} domainsByVersion - Exact revision key → domains * @returns {Array} Filtered members */ -function applyFilters(members, filters) { +function applyFilters(members, filters, domainsByVersion) { if (!filters) return members; let filtered = members; @@ -160,21 +193,58 @@ function applyFilters(members, filters) { }); } - // Note: domains filtering requires fetching full STIX objects, which is - // deferred to Phase 6 (export-service). For now, domains filter is a no-op - // logged as a warning. if (filters.domains && filters.domains.length > 0) { - logger.warn( - 'VirtualTrackService: domains filter is not yet implemented (requires Phase 6 export infrastructure)', - ); + const allowedDomains = new Set(filters.domains.map(normalizeDomain)); + filtered = filtered.filter((member) => { + const key = `${member.object_ref}::${new Date(member.object_modified).getTime()}`; + const objectDomains = domainsByVersion.get(key) || []; + return objectDomains.some((domain) => allowedDomains.has(normalizeDomain(domain))); + }); } return filtered; } /** - * Core composition resolution logic shared by createVirtualSnapshot and - * previewVirtualSnapshot. + * Hydrate domains for the exact pinned revisions needed by domain filters. + * + * @param {Array} componentTracks + * @param {Array} resolutions + * @returns {Promise>>} + */ +async function hydrateDomains(componentTracks, resolutions) { + const entries = []; + const seen = new Set(); + + for (let i = 0; i < componentTracks.length; i++) { + if (!componentTracks[i].filters?.domains?.length) continue; + + for (const member of resolutions[i].members || []) { + const key = `${member.object_ref}::${new Date(member.object_modified).getTime()}`; + if (seen.has(key)) continue; + seen.add(key); + entries.push(member); + } + } + + if (entries.length === 0) return new Map(); + + const results = await EventBus.emit(Events.ATTACK_OBJECT_REVISIONS_REQUESTED, { entries }); + const documents = results?.[0]; + if (!documents) { + throw new Error('Unable to hydrate ATT&CK object revisions for virtual domain filtering'); + } + + return new Map( + documents.map((document) => [ + `${document.stix.id}::${new Date(document.stix.modified).getTime()}`, + getObjectDomains(document.stix), + ]), + ); +} + +/** + * Resolve the current virtual composition into concrete member revisions. * * @param {Object} snapshot - The current virtual track snapshot * @param {Map} registryMap - track_id → registry entry @@ -194,6 +264,7 @@ async function resolveComposition(snapshot, registryMap) { const resolutions = await Promise.all( componentTracks.map((component) => resolveComponentSnapshot(component)), ); + const domainsByVersion = await hydrateDomains(componentTracks, resolutions); for (let i = 0; i < componentTracks.length; i++) { const component = componentTracks[i]; @@ -205,7 +276,7 @@ async function resolveComposition(snapshot, registryMap) { const totalObjectsInSource = sourceMembers.length; // Apply filters - const filteredMembers = applyFilters(sourceMembers, component.filters); + const filteredMembers = applyFilters(sourceMembers, component.filters, domainsByVersion); const objectsAfterFilter = filteredMembers.length; // Annotate each member with source metadata for deduplication @@ -360,61 +431,3 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op ); return snapshot; }; - -/** - * Preview what a virtual snapshot would contain without persisting. - * - * Runs the same resolution and deduplication logic as createVirtualSnapshot - * but returns the results without saving a new snapshot. - * - * @param {string} trackId - * @returns {Promise} Preview object with resolution details - */ -exports.previewVirtualSnapshot = async function previewVirtualSnapshot(trackId) { - const source = await snapshotService.getLatestSnapshot(trackId); - assertVirtualTrack(source); - - const composition = source.composition; - if (!composition || !composition.component_tracks || composition.component_tracks.length === 0) { - throw new BadRequestError({ - message: 'Cannot preview virtual snapshot: no component tracks configured', - details: 'Update the composition before previewing a snapshot', - }); - } - - // Validate component tracks - const registryMap = await validateComponentTracks(composition.component_tracks); - - // Resolve composition (same logic, but we don't persist) - const { members, quarantined, compositionResolution } = await resolveComposition( - source, - registryMap, - ); - - // Build comparison to the latest tagged version (if any) - const existingMembers = source.members || []; - const existingMemberRefs = new Set(existingMembers.map((m) => m.object_ref)); - const newMemberRefs = new Set(members.map((m) => m.object_ref)); - - const newObjects = members.filter((m) => !existingMemberRefs.has(m.object_ref)); - const removedObjects = existingMembers.filter((m) => !newMemberRefs.has(m.object_ref)); - const updatedObjects = members.filter((m) => { - const existing = existingMembers.find((e) => e.object_ref === m.object_ref); - if (!existing) return false; - return new Date(m.object_modified).getTime() !== new Date(existing.object_modified).getTime(); - }); - - return { - track_id: trackId, - preview: true, - composition_resolution: compositionResolution, - members_count: members.length, - quarantined_count: quarantined.length, - comparison_to_current: { - current_members_count: existingMembers.length, - new_objects: newObjects.length, - updated_objects: updatedObjects.length, - removed_objects: removedObjects.length, - }, - }; -}; diff --git a/app/services/stix/attack-objects-service.js b/app/services/stix/attack-objects-service.js index d7e97a3e..51917ffc 100644 --- a/app/services/stix/attack-objects-service.js +++ b/app/services/stix/attack-objects-service.js @@ -204,9 +204,26 @@ class AttackObjectsService extends BaseService { AttackObjectsService.handleReleaseTrackContentsChanged, ); + EventBus.on( + Events.ATTACK_OBJECT_REVISIONS_REQUESTED, + AttackObjectsService.handleRevisionsRequested, + ); + logger.info('AttackObjectsService: Event listeners initialized'); } + /** + * Hydrate exact ATT&CK object revisions for cross-service consumers. + * + * @param {Object} payload + * @param {Array<{object_ref: string, object_modified: string|Date}>} payload.entries + * @returns {Promise>} + */ + static async handleRevisionsRequested({ entries }) { + if (!entries || entries.length === 0) return []; + return attackObjectsRepository.findManyByIdAndModified(entries); + } + /** * Reconcile workspace.release_tracks backrefs on attackObjects documents * when a release track's contents change. Covers every STIX type stored in diff --git a/app/services/stix/relationships-service.js b/app/services/stix/relationships-service.js index 68eb89d3..2e377219 100644 --- a/app/services/stix/relationships-service.js +++ b/app/services/stix/relationships-service.js @@ -61,9 +61,31 @@ class RelationshipsService extends BaseService { this.handleReleaseTrackContentsChanged.bind(this), ); + EventBus.on( + EventConstants.BUNDLE_RELATIONSHIPS_REQUESTED, + this.handleBundleRelationshipsRequested.bind(this), + ); + logger.info('RelationshipsService: Event listeners initialized'); } + /** + * Return the latest active relationship revisions whose endpoints are both + * in the requested bundle object set. + * + * @param {Object} payload + * @param {Array} payload.objectRefs + * @returns {Promise>} + */ + static async handleBundleRelationshipsRequested({ objectRefs }) { + if (!objectRefs || objectRefs.length === 0) return []; + return relationshipsRepository.retrieveAllForBundle({ + includeRevoked: false, + includeDeprecated: false, + objectRefs, + }); + } + /** * Reconcile workspace.release_tracks backrefs on relationship documents * when a release track's contents change. Relationships live in their own diff --git a/app/services/stix/stix-bundles-service.js b/app/services/stix/stix-bundles-service.js index 04433cc3..abd0750b 100644 --- a/app/services/stix/stix-bundles-service.js +++ b/app/services/stix/stix-bundles-service.js @@ -4,6 +4,7 @@ const uuid = require('uuid'); const config = require('../../config/config'); const { BaseService } = require('../meta-classes'); const linkById = require('../../lib/linkById'); +const bundleRelationships = require('../../lib/stix-bundle-relationships'); const logger = require('../../lib/logger'); const { requiresAttackId } = require('../../lib/attack-id-generator'); const stixConformance = require('../../lib/stix-conformance'); @@ -125,16 +126,7 @@ class StixBundlesService extends BaseService { * - SRO * Reason: Data components no longer detect techniques; detection strategies do */ - static DEPRECATED_PATTERNS = [ - { - type: 'relationship', - conditions: { - relationship_type: 'detects', - sourceTypePrefix: 'x-mitre-data-component--', - }, - reason: 'Data components cannot detect techniques in v17+ (only detection strategies can)', - }, - ]; + static DEPRECATED_PATTERNS = bundleRelationships.DEPRECATED_PATTERNS; /** * Checks if a STIX object matches any deprecated pattern and should be excluded. @@ -142,31 +134,7 @@ class StixBundlesService extends BaseService { * @returns {boolean} True if the object matches a deprecated pattern */ static isDeprecatedPattern(stixObject) { - for (const pattern of StixBundlesService.DEPRECATED_PATTERNS) { - if (stixObject.type !== pattern.type) { - continue; - } - - // Check all conditions for this pattern - let matchesAllConditions = true; - for (const [key, value] of Object.entries(pattern.conditions)) { - if (key === 'sourceTypePrefix') { - // Special handling for source_ref prefix matching - if (!stixObject.source_ref?.startsWith(value)) { - matchesAllConditions = false; - break; - } - } else if (stixObject[key] !== value) { - matchesAllConditions = false; - break; - } - } - - if (matchesAllConditions) { - return true; - } - } - return false; + return bundleRelationships.isDeprecatedPattern(stixObject); } // ============================ @@ -247,7 +215,7 @@ class StixBundlesService extends BaseService { * @returns {boolean} True if the relationship is active */ static relationshipIsActive(relationship) { - return !relationship.stix.x_mitre_deprecated && !relationship.stix.revoked; + return bundleRelationships.relationshipIsActive(relationship); } /** diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index def1dff1..9bb9dfe2 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -223,7 +223,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { type: 'virtual', }); await request(app) - .put(`/api/release-tracks/${virtual.id}/composition`) + .put(`/api/release-tracks/${virtual.id}/virtual/composition`) .send({ component_tracks: [ { track_id: componentTrackId, resolution_strategy: 'latest_tagged', priority: 0 }, @@ -232,7 +232,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); - await postObject(`/api/release-tracks/${virtual.id}/snapshots/create`, {}, 201); + await postObject(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 201); // The object now carries one entry per referencing track, with types const retrieved = await getTechniqueVersion(technique); diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index 3a4ce529..8aca6aae 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -11,6 +11,8 @@ * Covered behavior: * - Default bundle contains members only, plus referenced identities and * marking definitions (self-contained bundle) + * - Active relationships whose endpoints are both selected are added + * dynamically; relationships with an endpoint outside the export are not * - `include` adds staged and/or candidate tiers (comma-separated or * repeated, singular or plural tier names) * - `state` narrows the included staged/candidate entries by workflow @@ -48,6 +50,9 @@ describe('Release Tracks Bundle Export API', function () { let memberObject; let linkedMemberObject; + let relationshipSource; + let includedRelationship; + let excludedRelationship; let linkedAttackId; let linkedAttackUrl; let candidateWip; @@ -152,6 +157,44 @@ describe('Release Tracks Bundle Export API', function () { ); candidateReviewed = await postObject('/api/techniques', buildTechnique('Candidate Reviewed')); stagedObject = await postObject('/api/techniques', buildTechnique('Staged Technique')); + relationshipSource = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Bundle Relationship Group', + description: 'Group used to verify dynamic relationship inclusion.', + spec_version: '2.1', + type: 'intrusion-set', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + includedRelationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: relationshipSource.stix.id, + target_ref: memberObject.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + excludedRelationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: relationshipSource.stix.id, + target_ref: candidateWip.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); const track = await postAction( '/api/release-tracks/new', @@ -178,6 +221,7 @@ describe('Release Tracks Bundle Export API', function () { x_mitre_contents: [ { obj_ref: memberObject.stix.id, obj_modified: memberObject.stix.modified }, { obj_ref: linkedMemberObject.stix.id, obj_modified: linkedMemberObject.stix.modified }, + { obj_ref: relationshipSource.stix.id, obj_modified: relationshipSource.stix.modified }, ], }); @@ -257,11 +301,27 @@ describe('Release Tracks Bundle Export API', function () { expect(toc.object_marking_refs).toContain(staticMarkingDefinitionId); const contentRefs = toc.x_mitre_contents.map((entry) => entry.object_ref); expect(contentRefs).toContain(memberObject.stix.id); + expect(contentRefs).toContain(includedRelationship.stix.id); expect(contentRefs).toContain(organizationIdentityId); expect(contentRefs).not.toContain(staticMarkingDefinitionId); expect(contentRefs).not.toContain(toc.id); }); + it('adds only relationships whose endpoints are both selected for the bundle', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, + ); + const ids = bundleObjectIds(bundle); + + expect(ids).toContain(includedRelationship.stix.id); + expect(ids).not.toContain(excludedRelationship.stix.id); + + const snapshot = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest`); + expect(snapshot.members.map((member) => member.object_ref)).not.toContain( + includedRelationship.stix.id, + ); + }); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&includeToc=false omits the TOC', async function () { const bundle = await getBundle( `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 825ced47..5b8bccc4 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -8,8 +8,37 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const versioningService = require('../../../services/release-tracks/versioning-service'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; +const virtualObjectRefs = [ + 'attack-pattern--00000000-0000-4000-8000-000000000101', + 'attack-pattern--00000000-0000-4000-8000-000000000102', + 'attack-pattern--00000000-0000-4000-8000-000000000103', + 'attack-pattern--00000000-0000-4000-8000-000000000104', +]; + +function snapshotBase(snapshot) { + const clone = { ...snapshot }; + delete clone._id; + delete clone.__v; + return clone; +} + +function memberEntry(objectRef, modified) { + return { object_ref: objectRef, object_modified: modified }; +} + +function quarantineEntry(objectRef, modified, sourceTrackId) { + return { + object_ref: objectRef, + object_modified: modified, + source_track_id: sourceTrackId, + source_track_name: 'Virtual Release Source', + source_snapshot_version: '1.0', + conflict_reason: 'Conflicting component revisions', + }; +} function buildTechnique(name, previous) { const timestamp = previous @@ -154,13 +183,143 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version).toBe('3.0'); }); - it('orients virtual previews around members and quarantine', async function () { + it('compares the latest virtual draft with its preceding tagged release', async function () { const track = await createTrack('Virtual Release Preview', 'virtual'); + const created = new Date(track.modified); + const taggedModified = new Date(created.getTime() + 1000); + const draftModified = new Date(created.getTime() + 2000); + const oldRevision = new Date(created.getTime() - 2000); + const newRevision = new Date(created.getTime() - 1000); + + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: taggedModified, + version: '1.0', + members: [ + memberEntry(virtualObjectRefs[0], oldRevision), + memberEntry(virtualObjectRefs[1], oldRevision), + ], + quarantine: [quarantineEntry(virtualObjectRefs[3], oldRevision, track.id)], + }); + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: draftModified, + version: null, + members: [ + memberEntry(virtualObjectRefs[0], newRevision), + memberEntry(virtualObjectRefs[2], newRevision), + ], + quarantine: [], + }); + const preview = await get(`/api/release-tracks/${track.id}/snapshots/latest/release/preview`); - expect(preview.body.type).toBe('virtual'); - expect(preview.body.before).toEqual({ members_count: 0, quarantine_count: 0 }); + expect(preview.body).toMatchObject({ + type: 'virtual', + source_snapshot_modified: draftModified.toISOString(), + version: '1.1', + previous_release: { + version: '1.0', + modified: taggedModified.toISOString(), + }, + before: { members_count: 2, quarantine_count: 1 }, + after: { members_count: 2, quarantine_count: 0 }, + changes: { + new_count: 1, + updated_count: 1, + removed_count: 1, + quarantined_count: 0, + }, + }); expect(preview.body.before).not.toHaveProperty('staged_count'); expect(preview.body.before).not.toHaveProperty('candidates_count'); + + const unchanged = await get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(draftModified.toISOString())}`, + ); + expect(unchanged.body.version).toBeNull(); + }); + + it('compares a historical virtual draft with the tagged release that preceded it', async function () { + const track = await createTrack('Historical Virtual Release Preview', 'virtual'); + const created = new Date(track.modified); + const firstTaggedModified = new Date(created.getTime() + 1000); + const historicalDraftModified = new Date(created.getTime() + 2000); + const laterTaggedModified = new Date(created.getTime() + 3000); + const oldRevision = new Date(created.getTime() - 2000); + const newRevision = new Date(created.getTime() - 1000); + + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: firstTaggedModified, + version: '1.0', + members: [memberEntry(virtualObjectRefs[0], oldRevision)], + }); + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: historicalDraftModified, + version: null, + members: [memberEntry(virtualObjectRefs[0], newRevision)], + }); + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: laterTaggedModified, + version: '2.0', + members: [memberEntry(virtualObjectRefs[3], newRevision)], + }); + + const preview = await get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(historicalDraftModified.toISOString())}/release/preview?version=3.0`, + ); + expect(preview.body.previous_release).toEqual({ + version: '1.0', + modified: firstTaggedModified.toISOString(), + }); + expect(preview.body.before).toEqual({ members_count: 1, quarantine_count: 0 }); + expect(preview.body.after).toEqual({ members_count: 1, quarantine_count: 0 }); + expect(preview.body.changes).toEqual({ + new_count: 0, + updated_count: 1, + removed_count: 0, + quarantined_count: 0, + }); + }); + + it('exposes virtual-only draft operations under the explicit virtual namespace', async function () { + const standard = await createTrack('Virtual Namespace Guard'); + + await request(app) + .put(`/api/release-tracks/${standard.id}/virtual/composition`) + .send({ + component_tracks: [ + { + track_id: standard.id, + resolution_strategy: 'latest_tagged', + }, + ], + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(400); + + await post(`/api/release-tracks/${standard.id}/virtual/snapshots/create`, {}, 400); + // The removed path now falls through to the generic :modified retrieval + // route, where "preview" is rejected as a malformed timestamp. + await get(`/api/release-tracks/${standard.id}/snapshots/preview`, 400); + await post(`/api/release-tracks/${standard.id}/snapshots/create`, {}, 405); + + await request(app) + .put(`/api/release-tracks/${standard.id}/composition`) + .send({ + component_tracks: [ + { + track_id: standard.id, + resolution_strategy: 'latest_tagged', + }, + ], + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(404); }); it('reports blocking promotion conflicts in summaries and rejects materialization', async function () { diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index 6fb1ac04..f152f89b 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -107,10 +107,10 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { 201, ); virtualTrack = virtual.id; - await put(`/api/release-tracks/${virtualTrack}/composition`, { + await put(`/api/release-tracks/${virtualTrack}/virtual/composition`, { component_tracks: [{ track_id: trackB, resolution_strategy: 'latest_tagged', priority: 0 }], }); - await post(`/api/release-tracks/${virtualTrack}/snapshots/create`, {}, 201); + await post(`/api/release-tracks/${virtualTrack}/virtual/snapshots/create`, {}, 201); await releaseLatest(virtualTrack); }); diff --git a/app/tests/api/release-tracks/virtual-domain-filters.spec.js b/app/tests/api/release-tracks/virtual-domain-filters.spec.js new file mode 100644 index 00000000..f158ee29 --- /dev/null +++ b/app/tests/api/release-tracks/virtual-domain-filters.spec.js @@ -0,0 +1,160 @@ +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const { cloneForCreate } = require('../../shared/clone-for-create'); + +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Virtual Release Track Domain Filters API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + function buildMitigation(name, domains) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'course-of-action', + labels: ['test'], + x_mitre_version: '1.0', + x_mitre_domains: domains, + object_marking_refs: [staticMarkingDefinitionId], + }, + }; + } + + function buildMatrix(name, externalDomain) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'x-mitre-matrix', + external_references: [{ source_name: 'test-source', external_id: externalDomain }], + object_marking_refs: [staticMarkingDefinitionId], + x_mitre_version: '1.0', + }, + }; + } + + async function createVirtualSnapshot(name, componentTrackId, domains) { + const virtual = await post('/api/release-tracks/new', { + name, + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: componentTrackId, + resolution_strategy: 'latest_tagged', + priority: 0, + filters: { domains }, + }, + ], + deduplication: { strategy: 'prioritize_latest_object' }, + }, + }); + return post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}); + } + + it('filters exact pinned revisions by normalized ATT&CK domains', async function () { + const enterprise = await post( + '/api/mitigations', + buildMitigation('Enterprise Domain Member', ['enterprise-attack']), + ); + const ics = await post( + '/api/mitigations', + buildMitigation('ICS Domain Member', ['ics-attack']), + ); + const shared = await post( + '/api/mitigations', + buildMitigation('Shared Domain Member', ['enterprise-attack', 'ics-attack']), + ); + const noDomain = await post('/api/mitigations', buildMitigation('No Domain Member', undefined)); + const enterpriseMatrix = await post( + '/api/matrices', + buildMatrix('Domainless Enterprise Matrix', 'enterprise-attack'), + ); + + const component = await post('/api/release-tracks/new', { + name: 'Domain Filter Component', + type: 'standard', + }); + await post( + `/api/release-tracks/${component.id}/contents`, + { + x_mitre_contents: [enterprise, ics, shared, noDomain, enterpriseMatrix].map((object) => ({ + obj_ref: object.stix.id, + obj_modified: object.stix.modified, + })), + }, + 200, + ); + await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}, 200); + + // A newer revision has a different domain, but virtual composition must + // evaluate the exact revision pinned in the tagged component snapshot. + const newerEnterpriseRevision = cloneForCreate(enterprise); + newerEnterpriseRevision.stix.modified = new Date(Date.now() + 1000).toISOString(); + newerEnterpriseRevision.stix.x_mitre_domains = ['ics-attack']; + await post('/api/mitigations', newerEnterpriseRevision); + + const enterpriseSnapshot = await createVirtualSnapshot( + 'Enterprise Domain Virtual', + component.id, + ['enterprise'], + ); + const enterpriseIds = enterpriseSnapshot.members.map((member) => member.object_ref); + expect(enterpriseIds).toEqual( + expect.arrayContaining([enterprise.stix.id, shared.stix.id, enterpriseMatrix.stix.id]), + ); + expect(enterpriseIds).not.toContain(ics.stix.id); + expect(enterpriseIds).not.toContain(noDomain.stix.id); + + const icsSnapshot = await createVirtualSnapshot('ICS Domain Virtual', component.id, [ + 'ics-attack', + ]); + const icsIds = icsSnapshot.members.map((member) => member.object_ref); + expect(icsIds).toEqual(expect.arrayContaining([ics.stix.id, shared.stix.id])); + expect(icsIds).not.toContain(enterprise.stix.id); + expect(icsIds).not.toContain(enterpriseMatrix.stix.id); + expect(icsIds).not.toContain(noDomain.stix.id); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 5e057ca4..e43fed14 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,111 @@ # Release Track TODOs +## Consolidate virtual draft creation and shared release previews + +- [x] Move virtual-only composition and draft-creation operations under an + explicit `/virtual` capability namespace. +- [x] Remove the standalone virtual snapshot-preview endpoint without an + alias. +- [x] Enhance shared virtual release summaries to compare the persisted draft + with its preceding tagged release without recomputing composition. +- [x] Add regression coverage for route removal, type enforcement, latest and + historical virtual previews, and release-preview non-persistence. +- [x] Update OpenAPI, user/developer documentation, Bruno, and the + `internalattack` Python client. +- [x] Run focused regression specs, then the complete `npm test` suite. +- [x] Review the final diff and propose a conventional commit message. + +Verification result (2026-07-29): + +- Focused release, back-reference, release-by-object, and virtual-domain specs + pass; backend lint passes. +- The first complete run encountered two unrelated full-suite flakes in Assets + and Campaigns; both passed in isolation. The required second complete + `npm test` run passed. +- The `internalattack` focused suite passes (29), its complete suite passes + (246), and changed-file Ruff and pre-commit checks pass. +- Proposed commit: + `feat(release-tracks): clarify virtual draft and release lifecycle` + +## Bootstrap faster-release core, defense, and virtual tracks + +- [x] Reconcile the clarified ownership partition with the current release-track + and virtual-composition API. +- [x] Add regression coverage for functional virtual domain filters and + relationship-complete snapshot bundle exports. +- [x] Implement virtual `filters.domains` using the established ATT&CK domain + inference rules. +- [x] Reuse/extract existing bundle relationship logic so snapshot + `format=bundle` exports dynamically include valid secondary relationships. +- [x] Inventory and report any additional release-track no-op placeholders. +- [x] Update user/developer docs and OpenAPI for the effective contract change; + Bruno has no new or changed request parameter to mirror. +- [x] Run focused release-track regression specs, then the complete `npm test` + suite. +- [x] Scan all three ATT&CK v19.1 bundles and construct a disjoint exact-revision + partition for Enterprise Core, ICS Core, Mobile Core, and Defense. +- [x] Assign the shared identity and marking definitions to Enterprise Core + using the representations supported by release-track snapshots. +- [x] Preflight exact track names and refuse conflicting duplicate tracks. +- [x] Create and verify the four v19.1-pinned standard tracks. +- [x] Create and verify the three domain-filtered virtual track definitions. +- [x] Verify that every in-scope v19.1 object is owned by exactly one standard + track and record intentional relationship/collection exclusions. CTI owns + `course-of-action`; ICS Core owns `x-mitre-asset`. +- [x] Defer materializing virtual snapshots until the component standard tracks + have tagged releases; no release/tag action was authorized in this bootstrap. +- [x] Review the final repository diff and propose a conventional commit + message. + +Operational result (2026-07-28): + +- Standard tracks: Enterprise Core + (`release-track--48be5319-2f98-435a-ba36-5533236a991a`, 875 members), + ICS Core (`release-track--73147f31-2598-42a3-9cb4-125d458c4490`, 149), + Mobile Core (`release-track--ae6df6f6-3856-4d54-af40-22db856baa2d`, 206), + Defense (`release-track--84cb1147-9dba-445f-948e-6eecc51fa7e8`, 3,151), + and CTI (`release-track--469b126a-6081-462e-8b4c-709cdbb4eac4`, 1,575). +- CTI now includes 60 campaigns, 358 courses of action, 194 intrusion sets, + 866 malware objects, and 97 tools, pinned to the latest database revisions. +- Virtual definitions: Enterprise + (`release-track--a42a6f32-80c6-43a7-b1e7-26ef0814d0cb`), ICS + (`release-track--83ede842-58c8-42ce-a3fb-c38c5dd0e74c`), and Mobile + (`release-track--05615c60-bca8-4074-b8d3-b537eed52d30`). Each composes all + five standard tracks with `latest_tagged`, `prioritize_latest_object`, and + its domain filter. +- Verified 5,928 unique v19.1 owned object IDs form a disjoint partition; + relationships remain indirect, collections are generated at export, and + marking definitions are supporting metadata. +- Focused domain-filter and bundle-export specs pass (1 and 15 tests); + lint passes; the complete suite passes (OpenAPI 2, config 21, API 909, + middleware 24). +- Proposed commit: + `feat(release-tracks): filter virtual tracks and export relationships` + +## Bootstrap CTI faster-release tracks + +- [x] Read the local environment mapping and release-track documentation. +- [x] Inspect the internalattack release-track client and reference script. +- [x] Scan the ATT&CK v19.1 ICS and Mobile bundles and report every object type. +- [x] Preflight the production-mirroring Workbench API and existing tracks. +- [x] Create the CTI standard track with the latest intrusion-set, malware, + tool, and campaign revisions as members. +- [x] Verify the persisted CTI snapshot, object-type coverage, exact latest + revision pins, and counts. +- [x] Record operational results and propose a conventional commit message for + the committable scratchpad update. + +Operational result (2026-07-28): + +- Created standard track `CTI` + (`release-track--469b126a-6081-462e-8b4c-709cdbb4eac4`). +- Initially pinned 1,217 exact latest revisions as members: 60 campaigns, 194 + intrusion sets, 866 malware objects, and 97 tools. The clarified ownership + bootstrap subsequently added 358 courses of action for 1,575 total members. +- Verified the persisted snapshot, registry count, and all 1,575 member + backrefs; candidates and staged are empty. +- Proposed commit: `docs(release-tracks): record CTI bootstrap run` + ## Harden release version selection - [x] Reject simultaneous `increment` and `version` selectors inside the @@ -459,7 +565,7 @@ the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. ## Small Fixes -- [ ] **Composition schema mismatch: `priority`.** `PUT /api/release-tracks/:id/composition` — the Zod schema (`componentTrackSchema`) marks `priority` optional, but the mongoose snapshot schema requires it, so omitting it passes validation and then fails the save with a 500 (`DatabaseError`) instead of a 400. Align the schemas (either default `priority` or make it required in Zod). Found 2026-07-15 while testing virtual-track backrefs. +- [ ] **Composition schema mismatch: `priority`.** `PUT /api/release-tracks/:id/virtual/composition` — the Zod schema (`componentTrackSchema`) marks `priority` optional, but the mongoose snapshot schema requires it, so omitting it passes validation and then fails the save with a 500 (`DatabaseError`) instead of a 400. Align the schemas (either default `priority` or make it required in Zod). Found 2026-07-15 while testing virtual-track backrefs. - [ ] **`deleteSnapshot` lacks a tagged-release guard.** `DELETE /api/release-tracks/:id/snapshots/:modified` (`snapshot-service.deleteSnapshot`) deletes any snapshot, including tagged releases — contradicting the "immutable once set" versioning rule. Should 409 on `version != null` (a squash implementation must also filter `version: null`; see Snapshot Retention section). Found 2026-07-15 while designing squash. diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 385888c8..28cadaaa 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -103,16 +103,21 @@ Implemented in 2. **Hydration** — the selected `{object_ref, object_modified}` pins are batch-fetched per STIX type via each repository's `findManyByIdAndModified`. -3. **Supporting objects** — referenced identities and marking definitions +3. **Relationships** — the relationship service fetches the latest active + relationship revisions whose `source_ref` and `target_ref` are both among + the selected objects. Deprecated data-component `detects` relationships + are excluded. Relationships remain indirect export-time content; they are + not added to the snapshot tiers. +4. **Supporting objects** — referenced identities and marking definitions that are not themselves tier entries are fetched and appended. -4. **LinkById conversion** — same behavior as the legacy exporter, preferring +5. **LinkById conversion** — same behavior as the legacy exporter, preferring objects already in the export before falling back to a database lookup. -5. **Assembly** (Zod transform) — notes are dropped, objects are conformed to +6. **Assembly** (Zod transform) — notes are dropped, objects are conformed to `stixVersion` via the shared `lib/stix-conformance.js` helpers, and the bundle envelope is emitted (with `spec_version: "2.0"` only when `stixVersion=2.0` — STIX 2.1 removed `spec_version` from the bundle object). -6. **TOC** — unless `includeToc=false`, an `x-mitre-collection` object is +7. **TOC** — unless `includeToc=false`, an `x-mitre-collection` object is prepended. Unlike the legacy exporter (which hardcoded per-domain metadata) and the ephemeral endpoint (which uses ephemeral defaults), the TOC is derived from the release track itself: @@ -129,6 +134,28 @@ Because snapshot contents are explicitly curated, the export intentionally does **not** apply the legacy attack-id / deprecated / revoked filters — if a revision is in the snapshot, it is exported. +#### Relationship consistency boundary + +Snapshot SDOs are reproducible because each member records an exact +`object_modified` revision. Relationships are intentionally different: the +bundle resolves their latest active revisions when it is requested. This +keeps relationships secondary and automatically reflects new links between +released objects, but it creates several tradeoffs: + +- exporting the same tagged snapshot at different times can produce different + relationship objects or TOC contents; +- relationship revisions are not represented in snapshot history, + release-track backrefs, or composition audit metadata; +- revoking a relationship can remove it from an older snapshot export, while + creating a relationship can add it to that export; +- each bundle request performs a relationship query, although the query is + constrained to relationships whose two endpoints are already selected. + +Consumers that require byte-for-byte or graph-level reproducibility must +archive the emitted bundle. A future model that pins relationship revisions +in a separate, generated manifest could preserve the indirect ownership model +while making repeat exports deterministic. + ### Where validation happens Query parameters are validated in the controller with Zod diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 1b4d21c9..db661c56 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -323,8 +323,7 @@ Virtual release tracks compute their contents by aggregating objects from compon // Optional: filters to limit which objects are included filters: { object_types: ["intrusion-set"], - domains: ["enterprise"], - stix_pattern: {} // Advanced STIX filtering + domains: ["enterprise"] } }, { diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 002c93f1..73b1a97c 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -58,6 +58,35 @@ policies remain responsible only for different revisions of one object. - Large collections (>10k objects) may need pagination - Consider caching for `release/preview` on large collections +### Virtual draft creation and release planning + +Virtual-only operations are deliberately scoped beneath +`/api/release-tracks/:id/virtual`: + +- `PUT /virtual/composition` clones a draft with revised composition rules. +- `POST /virtual/snapshots/create` resolves tagged component snapshots and + persists the concrete members, quarantine, and immutable + `composition_resolution`. + +There is no side-effect-free virtual snapshot-creation preview. Once a virtual +draft is persisted, it uses the same retrieval and release endpoints as a +standard draft. Release planning never resolves composition. + +For virtual summary previews, `versioning-service` loads the latest tagged +snapshot whose `modified` timestamp is strictly earlier than the selected +draft. This chronological lookup matters for historical drafts: a release +tagged later in the track must not become the comparison baseline. The pure +planner compares member IDs and exact revision timestamps and reports: + +- `new_count`: IDs present only in the draft; +- `updated_count`: IDs present in both with different revision sets; +- `removed_count`: IDs present only in the preceding release; +- `quarantined_count`: entries currently quarantined in the draft. + +The first virtual release uses zero-valued `before` counts and +`previous_release: null`. Workbench and bundle previews render the same frozen +planned snapshot, and the commit path tags that snapshot in place. + ### Snapshot history reads Snapshot history is exposed as a nested collection at diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index e2f76119..722e3e3c 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -105,9 +105,8 @@ GET /api/release-tracks/:id/objects/:objectRef/versions ### Virtual Release Tracks (Additional) ``` -PUT /api/release-tracks/:id/composition -POST /api/release-tracks/:id/snapshots/create -GET /api/release-tracks/:id/snapshots/preview +PUT /api/release-tracks/:id/virtual/composition +POST /api/release-tracks/:id/virtual/snapshots/create ``` --- @@ -882,6 +881,40 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview `format=bundle` returns its publication-ready STIX bundle. Thus “dry run” is not a separate command: it is a release preview with the desired format. +For a standard track, `before` is the selected draft before staged members are +promoted and `after` is the would-be tagged result. For a virtual track, the +contents were already resolved and frozen when the draft was explicitly +created. Its release summary therefore compares that persisted draft with the +most recent tagged snapshot that precedes it: + +```json +{ + "track_id": "release-track--virtual", + "type": "virtual", + "source_snapshot_modified": "2024-07-15T10:00:00.000Z", + "version": "14.0", + "releasable": true, + "previous_release": { + "version": "13.1", + "modified": "2024-01-15T10:00:00.000Z" + }, + "before": { "members_count": 850, "quarantine_count": 2 }, + "after": { "members_count": 870, "quarantine_count": 0 }, + "changes": { + "new_count": 30, + "updated_count": 12, + "removed_count": 10, + "quarantined_count": 0 + }, + "conflicts": [] +} +``` + +For the first virtual release, `previous_release` is `null` and the `before` +counts are zero. Historical draft previews compare against the tagged release +that chronologically preceded the selected draft, not a later release. Release +preview and release never re-resolve virtual composition. + --- ## Version Pin Management @@ -1040,17 +1073,17 @@ POST /api/release-tracks/new "composition": { "component_tracks": [ { - "track_id": "GroupsMonthly--uuid", + "track_id": "release-track--uuid", "resolution_strategy": "latest_tagged", + "priority": 0, "filters": { - "object_types": ["intrusion-set"] + "object_types": ["intrusion-set"], + "domains": ["enterprise"] } } ], "deduplication": { - "strategy": "prefer_latest_modified", - "tier_resolution": "highest_tier", - "status_resolution": "highest_status" + "strategy": "prioritize_latest_object" } }, "snapshot_schedule": { @@ -1060,10 +1093,18 @@ POST /api/release-tracks/new } ``` +`filters.domains` matches the exact pinned revision's `x_mitre_domains`. +Short names (`enterprise`, `ics`, `mobile`) and STIX names ending in +`-attack` are equivalent. Objects without a matching domain are excluded. +For primary matrices, which omit `x_mitre_domains` in published ATT&CK data, +the domain is read from `external_references[].external_id`. +`snapshot_schedule` is stored as metadata only; automated execution is not +yet implemented. + ### Update Virtual Track Composition ``` -PUT /api/release-tracks/:id/composition +PUT /api/release-tracks/:id/virtual/composition ``` **Request Body:** @@ -1084,12 +1125,13 @@ PUT /api/release-tracks/:id/composition } ``` -**Note:** Updating composition creates a new draft snapshot with the new composition rules. +**Note:** Updating composition creates a new draft snapshot containing the new +composition rules. It does not resolve component contents. ### Create Virtual Snapshot ``` -POST /api/release-tracks/:id/snapshots/create +POST /api/release-tracks/:id/virtual/snapshots/create ``` **Request Body:** @@ -1128,32 +1170,11 @@ POST /api/release-tracks/:id/snapshots/create } ``` -### Preview Virtual Snapshot - -Preview what a snapshot would contain without creating it: - -``` -GET /api/release-tracks/:id/snapshots/preview -``` - -**Response:** - -```json -{ - "preview": { - "would_resolve_to": { - "component_snapshots": [...], - "total_objects": 870 - }, - "comparison_to_latest_tagged": { - "current_version": "13.1", - "new_objects": 12, - "updated_objects": 45, - "removed_objects": 3 - } - } -} -``` +The response is the persisted draft. Review it through the shared snapshot +retrieval endpoints, then use the shared release-preview and release endpoints +to tag it. There is no separate virtual snapshot-creation preview: the release +preview is the authoritative comparison and representation of the persisted +draft that would be tagged. --- diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 94298761..6ddcf015 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -1014,9 +1014,10 @@ See [virtual-tracks.md](virtual-tracks.md) for complete virtual track documentat - Creates draft snapshot with resolved composition - Team receives notification to review -4. Review and tag +4. Review, preview, and tag - Team reviews which component versions were included - Verifies object counts and composition + - Previews the draft against its preceding tagged release - Tags snapshot when satisfied ``` @@ -1067,6 +1068,7 @@ July 15 (scheduled): July 16 (manual): - Team reviews draft - Verifies composition + - Previews the release delta and publication artifact - Tags as Enterprise v14.0 ``` diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 758b3eb0..f75c91bb 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -202,16 +202,24 @@ filters: { // Only include specific object types object_types: ["intrusion-set", "malware"], - // Only include objects with specific domains (if applicable) - domains: ["enterprise", "mobile"], - - // Only include objects matching STIX filter pattern (advanced) - stix_pattern: { - "x_mitre_platforms": { "$in": ["Windows", "macOS"] } - } + // Match the pinned revision's x_mitre_domains values. Both public names + // ("enterprise") and STIX names ("enterprise-attack") are accepted. + domains: ["enterprise", "mobile"] } ``` +Domain filters hydrate the exact revisions pinned by the component's tagged +snapshot; they do not inspect the latest database revision. An object with +multiple matching domains is included in each corresponding virtual track. +Objects without `x_mitre_domains` are excluded when a domain filter is set. +The primary Enterprise, ICS, and Mobile matrices are the exception: published +ATT&CK data identifies their domain through +`external_references[].external_id`, so virtual filtering uses that established +matrix fallback. + +`stix_pattern` is not part of the current request schema and is not +implemented. + ### Deduplication Strategies When multiple component tracks contain the same object (same `stix.id`), a conflict occurs during the sync operation. The virtual track's deduplication strategy determines how to resolve the conflict. Four strategies are available: @@ -400,7 +408,7 @@ Virtual track snapshots are created either **manually** or **on schedule**. #### Manual Snapshot ```bash -POST /api/release-tracks/:id/snapshots/create +POST /api/release-tracks/:id/virtual/snapshots/create ``` **Request:** @@ -490,7 +498,11 @@ snapshot_schedule: { } ``` -**Scheduler integration:** +The configuration is currently persisted as registry metadata only. No +release-track scheduler consumes it yet, so `cron` and `dates` schedules do +not create snapshots automatically. + +**Planned scheduler integration:** ```javascript scheduler.register({ type: "virtual-track-snapshot", @@ -522,9 +534,23 @@ GET /api/release-tracks/:id/snapshots/:modified?format=workbench&include=all **Response includes:** - All objects that will be in the release - Composition resolution details (which component versions were used) -- Statistics and diff from previous tagged release +- The exact persisted members and quarantine tiers + +### 3. Release Preview -### 3. Snapshot Tagging +Preview the selected draft against its preceding tagged release: + +```bash +GET /api/release-tracks/:id/snapshots/:modified/release/preview +``` + +The summary reports the next version, previous tagged release, type-oriented +before/after counts, and new, updated, removed, and quarantined object counts. +Use `format=workbench` for the literal would-be tagged snapshot or +`format=bundle` for its publication artifact. Previewing does not persist and +never re-resolves composition. + +### 4. Snapshot Tagging Once reviewed, explicitly tag the draft snapshot: @@ -576,7 +602,7 @@ POST /api/release-tracks/:id/snapshots/:modified/release 4. Add entry to version_history 5. Snapshot is now immutable -### 4. Snapshot Export +### 5. Snapshot Export Export virtual track snapshot as STIX bundle: @@ -704,7 +730,7 @@ for (const component of composition.component_tracks) { **User experience:** ```bash -POST /api/release-tracks/release-track--uuid-virtual/snapshots/create +POST /api/release-tracks/release-track--uuid-virtual/virtual/snapshots/create # Error response: { @@ -777,7 +803,7 @@ POST /api/release-tracks/new ### Update Composition ```bash -PUT /api/release-tracks/:id/composition +PUT /api/release-tracks/:id/virtual/composition ``` **Request:** @@ -802,7 +828,7 @@ PUT /api/release-tracks/:id/composition ### Create Virtual Snapshot ```bash -POST /api/release-tracks/:id/snapshots/create +POST /api/release-tracks/:id/virtual/snapshots/create ``` **Request:** @@ -812,44 +838,30 @@ POST /api/release-tracks/:id/snapshots/create } ``` -### Preview Virtual Snapshot - -Preview what a snapshot would contain without creating it: +### Tag Virtual Snapshot ```bash -GET /api/release-tracks/:id/snapshots/preview +POST /api/release-tracks/:id/snapshots/:modified/release ``` -**Response:** +**Request:** ```json { - "preview": { - "would_resolve_to": { - "component_snapshots": [...], - "total_objects": 870 - }, - "comparison_to_latest_tagged": { - "current_version": "13.1", - "new_objects": 12, - "updated_objects": 45, - "removed_objects": 3 - } - } + "increment": "major" } ``` -### Tag Virtual Snapshot +Release preview uses the same shared path as standard tracks: ```bash -POST /api/release-tracks/:id/snapshots/:modified/release +GET /api/release-tracks/:id/snapshots/:modified/release/preview ``` -**Request:** -```json -{ - "increment": "major" -} -``` +Virtual composition is not recomputed during preview or release. The summary +compares the selected persisted draft with the tagged release that immediately +preceded it, reporting members/quarantine counts and new, updated, removed, and +quarantined object counts. Use `format=workbench` or `format=bundle` to inspect +the literal snapshot or publication artifact that would be tagged. ### Get Virtual Track with Resolved Content @@ -1018,7 +1030,7 @@ POST /api/release-tracks/new ```bash # Manually trigger first snapshot -POST /api/release-tracks/release-track--uuid-virtual/snapshots/create +POST /api/release-tracks/release-track--uuid-virtual/virtual/snapshots/create # Review draft snapshot GET /api/release-tracks/release-track--uuid-virtual/snapshots/:modified @@ -1123,13 +1135,13 @@ Always create snapshot, review, then tag: ```bash # Create draft -POST /api/release-tracks/:id/snapshots/create +POST /api/release-tracks/:id/virtual/snapshots/create # Review GET /api/release-tracks/:id/snapshots/:modified?format=workbench -# Preview export -GET /api/release-tracks/:id/snapshots/:modified?format=bundle +# Preview release artifact +GET /api/release-tracks/:id/snapshots/:modified/release/preview?format=bundle # Tag only when satisfied POST /api/release-tracks/:id/snapshots/:modified/release From 37f521e170e1353fb637ce4f9c9de5b02a819649 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:50:35 -0400 Subject: [PATCH 20/55] fix(release-tracks): enforce virtual materialization lifecycle Invalidate materialized contents when composition changes and reject release planning until the virtual draft is rematerialized. Restrict direct contents replacement to standard tracks and document the remaining virtual-track work. --- VIRTUAL_TRACKS_TODO.md | 128 ++++++++++++++++++ .../definitions/components/release-tracks.yml | 8 ++ .../paths/release-tracks-paths.yml | 25 +++- app/exceptions/index.js | 10 ++ app/lib/error-handler.js | 2 + .../release-tracks/snapshot-service.js | 13 ++ .../release-tracks/versioning-service.js | 12 +- .../release-tracks/virtual-track-service.js | 7 +- .../release-tracks-release.spec.js | 100 ++++++++++++++ docs/developer/TODO.md | 27 ++++ docs/developer/release-tracks/entities.md | 4 +- .../release-tracks/implementation-notes.md | 10 +- docs/user/release-tracks/api-reference.md | 30 +++- docs/user/release-tracks/virtual-tracks.md | 14 +- 14 files changed, 374 insertions(+), 16 deletions(-) create mode 100644 VIRTUAL_TRACKS_TODO.md diff --git a/VIRTUAL_TRACKS_TODO.md b/VIRTUAL_TRACKS_TODO.md new file mode 100644 index 00000000..968d3ff2 --- /dev/null +++ b/VIRTUAL_TRACKS_TODO.md @@ -0,0 +1,128 @@ +# Virtual Release Tracks Completion Backlog + +This backlog records the 2026-07-29 documentation-to-implementation audit of +virtual release tracks. Items are ordered by integrity risk and implementation +dependency. A checked item must include regression coverage and any necessary +OpenAPI, user/developer documentation, client, and Bruno updates. + +## P0 — Snapshot lifecycle integrity + +- [x] Make composition changes invalidate the previous materialization: + - clear inherited `members`, `quarantine`, and `composition_resolution`; + - expose that the resulting virtual draft is awaiting materialization; + - require `POST /api/release-tracks/:id/virtual/snapshots/create` before the + draft can be previewed or tagged as a release. +- [x] Reject generic member replacement for virtual tracks: + - `POST /api/release-tracks/:id/contents`; + - `POST /api/release-tracks/:id/snapshots/:modified/contents`. + Virtual membership must only be produced by composition resolution. +- [ ] Implement the documented quarantine-resolution workflow, including + `POST /api/release-tracks/:id/quarantine/promote`, or remove the quarantine + strategy from the public contract until conflicts can be resolved. + +## P1 — Composition validation and deterministic resolution + +- [ ] Make request validation strict so misspelled keys such as + `filters.domain` return 400 instead of silently disabling filtering. +- [ ] Validate component selectors according to `resolution_strategy`: + - `specific_version` requires `version` and rejects `snapshot`; + - `specific_snapshot` requires `snapshot` and rejects `version`; + - `latest_tagged` rejects both selector fields. +- [ ] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, + and examples; reject duplicate priorities at the request boundary. +- [ ] Validate component existence, standard-track type, duplicate track IDs, + and duplicate priorities when a virtual track is initially created, not only + when composition is later updated or materialized. +- [ ] Validate `snapshot_schedule` by mode: + - `manual` rejects `cron` and `dates`; + - `cron` requires `cron` and rejects `dates`; + - `dates` requires at least one date and rejects `cron`. +- [ ] Constrain or document accepted `filters.object_types` values and add + direct regression coverage for exact-revision filtering. + +## P1 — Deduplication correctness + +- [ ] Treat the same exact object revision contributed by multiple components + as one duplicate, not a conflicting revision. +- [ ] Ensure the `quarantine` strategy only quarantines genuinely different + revisions of the same object. +- [ ] Attribute each surviving revision to one deterministic component so + `objects_contributed` totals cannot exceed `summary.total_objects`. +- [ ] Add dedicated tests for all four strategies: + `prioritize_latest_object`, `prioritize_latest_snapshot`, + `prioritize_higher_priority`, and `quarantine`. + +## P1 — Release provenance + +- [ ] Populate virtual release `version_history[].component_versions` from the + materialized snapshot's immutable `composition_resolution`. +- [ ] Define and test the provenance shape in Mongoose, OpenAPI, and user and + developer documentation. + +## P2 — Scheduled materialization + +- [ ] Connect virtual `snapshot_schedule` metadata to the existing task + scheduler. +- [ ] Implement manual, cron, and explicit-date scheduling semantics. +- [ ] Define failure behavior when a component has no matching tagged + snapshot, including automation-run audit records and retry policy. +- [ ] Add scheduler integration tests and operational documentation. + +## P2 — Contract decisions + +- [ ] Decide whether virtual tracks can compose virtual tracks. The + implementation currently rejects nesting while portions of the + documentation say standard or virtual components are supported. +- [ ] Decide whether to implement the documented native-members/hybrid model. + Prefer a dedicated standard component track unless a demonstrated use case + requires a second membership authority inside virtual tracks. +- [ ] Decide whether to implement `resolve=true` and `resolved_content`. + Remove these claims from documentation if eager materialization remains the + only supported model. +- [ ] Implement caching and component-release notifications only if measured + scale or an approved product workflow requires them; otherwise describe them + as future considerations rather than current capabilities. + +## Documentation corrections + +- [ ] Replace `stix.type = "virtual"` with the top-level snapshot + `type: "virtual"`. +- [ ] Remove the nonexistent snapshot-level `snapshot_id`; retain + `version_history[].snapshot_id`. +- [ ] Correct response envelopes and the virtual-create response example. +- [ ] Align `composition_resolution` examples with fields actually generated, + or implement the documented `by_type`, `by_tier`, and native statistics. +- [ ] Align documented error envelopes with centralized error-handler output. +- [ ] Include required `priority` values in every composition example. +- [ ] Clearly distinguish configured composition from a materialized draft and + describe scheduled behavior as unavailable until scheduler execution exists. + +## Verified complete + +- [x] `filters.domains` hydrates and evaluates exact pinned revisions. +- [x] Public domain names and STIX `*-attack` names are normalized. +- [x] Multiple domain values are supported. +- [x] Objects without domain metadata are excluded when a domain filter is set. +- [x] Primary Enterprise, ICS, and Mobile matrices use their ATT&CK external ID + as the established domain fallback. +- [x] Virtual tracks resolve only tagged snapshots and consume only component + `members`. +- [x] Virtual tracks maintain independent draft/release history and use the + shared snapshot retrieval and release endpoints after materialization. + +## Current implementation slice + +- [x] Add failing lifecycle and type-boundary regression tests. +- [x] Invalidate inherited materialization when composition changes. +- [x] Reject release previews and release commits for unmaterialized virtual + drafts. +- [x] Reject generic contents replacement for virtual tracks. +- [x] Update OpenAPI, user/developer docs, and Bruno. +- [x] Run focused specs followed by the complete `npm test` suite. + +Verification completed 2026-07-29: + +- Focused virtual/release regression suite: 46 passing. +- Lint: passing. +- Full test suite: 960 passing (OpenAPI 2, config 21, API 913, + middleware 24). diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 6687d054..e6b99d1c 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -70,6 +70,14 @@ components: nullable: true description: 'Component track references (virtual tracks only)' $ref: '#/components/schemas/composition' + composition_resolution: + type: object + nullable: true + description: | + Immutable component-resolution provenance for a materialized virtual + draft. Null or absent means the virtual composition is configured + but has not been materialized and therefore cannot be previewed or + released. config: $ref: '#/components/schemas/track-config' version_history: diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 3d3526dc..9422bbe4 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -324,7 +324,9 @@ paths: summary: 'Update member contents on the latest snapshot' operationId: 'release-tracks-update-contents-latest' description: | - Replace the members tier with new contents (x_mitre_contents format). + Replace the members tier of a standard track with new contents + (x_mitre_contents format). Virtual membership can only be produced by + POST /api/release-tracks/{id}/virtual/snapshots/create. Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. Creates a new snapshot clone. @@ -340,6 +342,8 @@ paths: responses: '200': description: 'Contents updated successfully' + '400': + description: 'Track is virtual or the contents request is invalid' /api/release-tracks/{id}/clone: post: @@ -407,7 +411,8 @@ paths: mutually exclusive; omitting both defaults to a minor increment. Standard summaries show staged-to-members promotion. Virtual summaries compare the persisted draft with its chronologically preceding tagged - release; composition is never recomputed. + release; composition is never recomputed. A virtual draft whose + composition has not been materialized returns 409. tags: - 'Release Tracks' parameters: @@ -468,6 +473,8 @@ paths: responses: '200': description: 'Release preview generated' + '409': + description: 'Virtual draft is unmaterialized or release is blocked by a conflict' '501': description: 'Requested format is not yet implemented' @@ -784,6 +791,8 @@ paths: description: | Update which component tracks a virtual track aggregates. This operation is available only for tracks whose type is `virtual`. + The new pending draft has empty members and quarantine tiers and a null + composition_resolution. Materialize it before release preview or commit. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -1119,7 +1128,9 @@ paths: summary: 'Update contents on a specific snapshot' operationId: 'release-tracks-update-contents-by-modified' description: | - Update member contents on a historical snapshot. + Update member contents on a historical standard-track snapshot. + Virtual membership can only be produced by + POST /api/release-tracks/{id}/virtual/snapshots/create. Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. Request body validated via Zod in controller. @@ -1139,6 +1150,8 @@ paths: responses: '200': description: 'Contents updated successfully' + '400': + description: 'Track is virtual or the contents request is invalid' /api/release-tracks/{id}/snapshots/{modified}/clone: post: @@ -1172,7 +1185,8 @@ paths: Immutably tag the snapshot selected by the modified timestamp using the same version-selection contract as the latest release operation: supply `increment` or `version`, never both; omit both for a minor - increment. + increment. Virtual drafts must have composition_resolution from a + successful materialization. tags: - 'Release Tracks' parameters: @@ -1214,6 +1228,7 @@ paths: exclusive; omitting both defaults to a minor increment. For a virtual draft, compare against the latest tagged snapshot whose modified timestamp precedes this selected snapshot; never recompute composition. + An unmaterialized virtual draft returns 409. tags: - 'Release Tracks' parameters: @@ -1272,5 +1287,7 @@ paths: responses: '200': description: 'Release preview generated' + '409': + description: 'Virtual draft is unmaterialized or release is blocked by a conflict' '501': description: 'Requested format is not yet implemented' diff --git a/app/exceptions/index.js b/app/exceptions/index.js index d4c74479..ad9a2977 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -341,6 +341,15 @@ class InvalidComponentTypeError extends CustomError { } } +class VirtualSnapshotNotMaterializedError extends CustomError { + constructor(trackId, options) { + super( + `Virtual release track ${trackId} has not been materialized from its composition`, + options, + ); + } +} + class TrackNotFoundError extends CustomError { constructor(trackId, options) { super(`Release track ${trackId} not found`, options); @@ -384,6 +393,7 @@ module.exports = { ReleaseConflictError, NoTaggedSnapshotsError, InvalidComponentTypeError, + VirtualSnapshotNotMaterializedError, TrackNotFoundError, MemberPinnedRevisionError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 248d6886..b0701431 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -41,6 +41,7 @@ const { ReleaseConflictError, NoTaggedSnapshotsError, InvalidComponentTypeError, + VirtualSnapshotNotMaterializedError, TrackNotFoundError, MemberPinnedRevisionError, ObjectHasValidationIssuesError, @@ -133,6 +134,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof AlreadyReleasedError || err instanceof TaggedSnapshotDeletionError || err instanceof ReleaseConflictError || + err instanceof VirtualSnapshotNotMaterializedError || err instanceof MemberPinnedRevisionError || err instanceof ObjectHasValidationIssuesError || err instanceof ActiveOrganizationIdentityDeleteError diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 0700b3c5..4ad5f62c 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -25,6 +25,7 @@ const { TrackNotFoundError, NotFoundError, TaggedSnapshotDeletionError, + BadRequestError, } = require('../../exceptions'); // ============================================================================= @@ -52,6 +53,16 @@ function normalizeTierSummary(summary) { }; } +function assertStandardTrack(snapshot) { + if (snapshot.type !== 'standard') { + throw new BadRequestError({ + message: 'Direct contents updates are only available for standard release tracks', + details: + 'Virtual members are computed from component tracks; create a virtual snapshot to update them', + }); + } +} + /** * Recompute and persist denormalized registry counters from actual snapshot data. * @@ -473,6 +484,7 @@ exports.updateMetadataByModified = async function updateMetadataByModified( // eslint-disable-next-line no-unused-vars exports.updateContents = async function updateContents(trackId, contents, _userId) { const source = await exports.getLatestSnapshot(trackId); + assertStandardTrack(source); const members = contents.x_mitre_contents.map((c) => ({ object_ref: c.obj_ref, object_modified: c.obj_modified === 'latest' ? new Date() : new Date(c.obj_modified), @@ -497,6 +509,7 @@ exports.updateContentsByModified = async function updateContentsByModified( _userId, ) { const source = await exports.getSnapshotByModified(trackId, modified); + assertStandardTrack(source); const members = contents.x_mitre_contents.map((c) => ({ object_ref: c.obj_ref, object_modified: c.obj_modified === 'latest' ? new Date() : new Date(c.obj_modified), diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index c80a42ae..a4f66cb6 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -11,7 +11,11 @@ const conflictResolution = require('../../lib/release-tracks/conflict-resolution const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const releaseHistoryService = require('./release-history-service'); const logger = require('../../lib/logger'); -const { AlreadyReleasedError, ReleaseConflictError } = require('../../exceptions'); +const { + AlreadyReleasedError, + ReleaseConflictError, + VirtualSnapshotNotMaterializedError, +} = require('../../exceptions'); function iso(value) { return new Date(value).toISOString(); @@ -100,6 +104,12 @@ function planRelease( if (sourceSnapshot.version != null) { throw new AlreadyReleasedError(sourceSnapshot.version); } + if (sourceSnapshot.type === 'virtual' && sourceSnapshot.composition_resolution == null) { + throw new VirtualSnapshotNotMaterializedError(trackId, { + details: + 'Create a persisted draft with POST /api/release-tracks/:id/virtual/snapshots/create before previewing or releasing it', + }); + } const normalized = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot); const snapshot = normalized.snapshot; diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 5b2aa7e4..fd2879c1 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -370,7 +370,12 @@ exports.updateComposition = async function updateComposition(trackId, compositio // Validate all component tracks await validateComponentTracks(composition.component_tracks); - const snapshot = await snapshotService.cloneSnapshot(trackId, source, { composition }); + const snapshot = await snapshotService.cloneSnapshot(trackId, source, { + composition, + members: [], + quarantine: [], + composition_resolution: null, + }); logger.verbose( `VirtualTrackService: Updated composition for track "${trackId}" ` + diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 5b8bccc4..228a0195 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -40,6 +40,14 @@ function quarantineEntry(objectRef, modified, sourceTrackId) { }; } +function compositionResolution(modified) { + return { + resolved_at: modified, + component_snapshots: [], + summary: { total_objects: 0, quarantined_objects: 0 }, + }; +} + function buildTechnique(name, previous) { const timestamp = previous ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() @@ -210,6 +218,7 @@ describe('Release-track release planning and commit API', function () { memberEntry(virtualObjectRefs[2], newRevision), ], quarantine: [], + composition_resolution: compositionResolution(draftModified), }); const preview = await get(`/api/release-tracks/${track.id}/snapshots/latest/release/preview`); @@ -259,6 +268,7 @@ describe('Release-track release planning and commit API', function () { modified: historicalDraftModified, version: null, members: [memberEntry(virtualObjectRefs[0], newRevision)], + composition_resolution: compositionResolution(historicalDraftModified), }); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), @@ -322,6 +332,96 @@ describe('Release-track release planning and commit API', function () { .expect(404); }); + it('requires virtual composition to be materialized before preview or release', async function () { + const member = ( + await post('/api/techniques', buildTechnique('Virtual Materialization Member'), 201) + ).body; + const component = await createTrack('Virtual Materialization Component'); + await post(`/api/release-tracks/${component.id}/contents`, { + x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], + }); + await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}); + + const virtual = ( + await post( + '/api/release-tracks/new', + { + name: 'Virtual Materialization Lifecycle', + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: component.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + }, + }, + 201, + ) + ).body; + const materialized = ( + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 201) + ).body; + expect(materialized.members).toHaveLength(1); + expect(materialized.composition_resolution).toBeDefined(); + + const compositionDraft = await request(app) + .put(`/api/release-tracks/${virtual.id}/virtual/composition`) + .send({ + component_tracks: [ + { + track_id: component.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + expect(compositionDraft.body.members).toEqual([]); + expect(compositionDraft.body.quarantine).toEqual([]); + expect(compositionDraft.body.composition_resolution).toBeNull(); + + await get(`/api/release-tracks/${virtual.id}/snapshots/latest/release/preview`, 409); + await post(`/api/release-tracks/${virtual.id}/snapshots/latest/release`, {}, 409); + + const rematerialized = ( + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 201) + ).body; + expect(rematerialized.members).toHaveLength(1); + expect(rematerialized.composition_resolution).toBeDefined(); + + const preview = await get(`/api/release-tracks/${virtual.id}/snapshots/latest/release/preview`); + expect(preview.body.releasable).toBe(true); + }); + + it('rejects generic contents replacement for virtual tracks', async function () { + const virtual = await createTrack('Virtual Contents Guard', 'virtual'); + const contents = { + x_mitre_contents: [ + { + obj_ref: virtualObjectRefs[0], + obj_modified: new Date().toISOString(), + }, + ], + }; + + await post(`/api/release-tracks/${virtual.id}/contents`, contents, 400); + await post( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}/contents`, + contents, + 400, + ); + + const latest = await get(`/api/release-tracks/${virtual.id}/snapshots/latest`); + expect(latest.body.modified).toBe(virtual.modified); + expect(latest.body.members).toEqual([]); + }); + it('reports blocking promotion conflicts in summaries and rejects materialization', async function () { const revisionA = (await post('/api/techniques', buildTechnique('Release Conflict A'), 201)) .body; diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index e43fed14..e4fd0fc4 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,32 @@ # Release Track TODOs +## Harden virtual materialization lifecycle + +- [x] Record the complete virtual-track audit in `VIRTUAL_TRACKS_TODO.md`. +- [x] Add regression coverage for stale composition state, unmaterialized + release attempts, and virtual use of standard contents endpoints. +- [x] Clear inherited materialized state when virtual composition changes. +- [x] Require a materialized virtual draft for release preview and commit. +- [x] Restrict generic contents replacement to standard tracks. +- [x] Update OpenAPI, user/developer documentation, and Bruno. +- [x] Run focused regression specs, then the complete `npm test` suite. +- [x] Review the final diff and propose a conventional commit message. + +Verification result (2026-07-29): + +- Focused release, back-reference, release-by-object, and virtual-domain specs + pass (46); backend lint passes. +- The complete suite passes (OpenAPI 2, config 21, API 913, middleware 24). +- Proposed commit: + + ```text + fix(release-tracks): enforce virtual materialization lifecycle + + Invalidate materialized contents when composition changes and reject release + planning until the virtual draft is rematerialized. Restrict direct contents + replacement to standard tracks and document the remaining virtual-track work. + ``` + ## Consolidate virtual draft creation and shared release previews - [x] Move virtual-only composition and draft-creation operations under an diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index db661c56..eba00bb9 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -342,7 +342,9 @@ Virtual release tracks compute their contents by aggregating objects from compon } }, - // Composition resolution - computed at snapshot creation time, immutable + // Composition resolution - computed at snapshot creation time, immutable. + // Null/absent means composition is configured but awaiting materialization; + // that draft cannot be previewed or tagged as a release. composition_resolution: { resolved_at: "2024-03-01T10:00:00Z", diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 73b1a97c..56d65bc2 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -63,14 +63,20 @@ policies remain responsible only for different revisions of one object. Virtual-only operations are deliberately scoped beneath `/api/release-tracks/:id/virtual`: -- `PUT /virtual/composition` clones a draft with revised composition rules. +- `PUT /virtual/composition` clones a pending draft with revised composition + rules, empty members/quarantine tiers, and + `composition_resolution: null`. Clearing all three prevents a materialized + result from surviving a change to the rules that produced it. - `POST /virtual/snapshots/create` resolves tagged component snapshots and persists the concrete members, quarantine, and immutable `composition_resolution`. There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a -standard draft. Release planning never resolves composition. +standard draft. Release planning never resolves composition and rejects a +virtual draft without `composition_resolution` with `409 Conflict`. Generic +latest and historical `/contents` mutations are standard-only; virtual +membership has composition resolution as its sole authority. For virtual summary previews, `versioning-service` loads the latest tagged snapshot whose `modified` timestamp is strictly earlier than the selected diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 722e3e3c..57675920 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -463,6 +463,11 @@ POST /api/release-tracks/:id/contents Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** The main workflow for enrolling new member objects into `x_mitre_contents` is through the candidate-staging promotion cycle described in [versioning.md](./versioning.md). +This operation is available only for standard tracks. Virtual membership is +computed from component releases and can only be updated by materializing a +virtual draft with `POST /api/release-tracks/:id/virtual/snapshots/create`. +Using either contents endpoint with a virtual track returns `400 Bad Request`. + **Request Body:** ```json @@ -499,6 +504,11 @@ Use `"version": "2.4"` instead of `increment` to select an explicit handled. Use the `:modified` release endpoint when a caller needs to pin the operation to a specific snapshot. +For virtual tracks, the selected draft must have a non-null +`composition_resolution`. An initial or composition-update draft is pending +until the virtual snapshot creation endpoint materializes it; preview and +release return `409 Conflict` before then. + ### Clone Release Track From Latest Bootstraps a new `release-track` instance from an existing snapshot. @@ -585,6 +595,8 @@ Creates new snapshot with updated member objects. **This is intended for retroac **Request Body:** Same as [Update Contents](#update-contents) for latest snapshot. +Like the latest form, this operation is restricted to standard tracks. + ### Release/Tag Specific Snapshot Converts a specific draft snapshot to a tagged release. Tags snapshot in-place (does not create new snapshot). @@ -884,8 +896,10 @@ not a separate command: it is a release preview with the desired format. For a standard track, `before` is the selected draft before staged members are promoted and `after` is the would-be tagged result. For a virtual track, the contents were already resolved and frozen when the draft was explicitly -created. Its release summary therefore compares that persisted draft with the -most recent tagged snapshot that precedes it: +created. A virtual draft without `composition_resolution` returns +`409 Conflict` instead of previewing stale or empty members. A materialized +draft's release summary compares that persisted draft with the most recent +tagged snapshot that precedes it: ```json { @@ -1125,8 +1139,13 @@ PUT /api/release-tracks/:id/virtual/composition } ``` -**Note:** Updating composition creates a new draft snapshot containing the new -composition rules. It does not resolve component contents. +**Note:** Updating composition creates a pending draft containing the new +rules. To prevent stale materialization from being released, the draft has +empty `members` and `quarantine` arrays and +`composition_resolution: null`. It cannot be previewed or tagged as a release +until `POST /api/release-tracks/:id/virtual/snapshots/create` materializes the +configured composition. Release preview and release return `409 Conflict` +while the draft is pending. ### Create Virtual Snapshot @@ -1174,7 +1193,8 @@ The response is the persisted draft. Review it through the shared snapshot retrieval endpoints, then use the shared release-preview and release endpoints to tag it. There is no separate virtual snapshot-creation preview: the release preview is the authoritative comparison and representation of the persisted -draft that would be tagged. +draft that would be tagged. A non-null `composition_resolution` is the +readiness marker for those shared release operations. --- diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index f75c91bb..567d0e6f 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -823,7 +823,11 @@ PUT /api/release-tracks/:id/virtual/composition } ``` -**Note:** Updating composition creates a new draft snapshot with the new composition rules. +**Note:** Updating composition creates a pending draft with the new rules and +invalidates any previously materialized contents. The draft has empty +`members` and `quarantine` arrays and `composition_resolution: null`. Run the +virtual snapshot creation operation before attempting release preview or +tagging; those release operations return `409 Conflict` for a pending draft. ### Create Virtual Snapshot @@ -861,7 +865,9 @@ Virtual composition is not recomputed during preview or release. The summary compares the selected persisted draft with the tagged release that immediately preceded it, reporting members/quarantine counts and new, updated, removed, and quarantined object counts. Use `format=workbench` or `format=bundle` to inspect -the literal snapshot or publication artifact that would be tagged. +the literal snapshot or publication artifact that would be tagged. The draft +must have a non-null `composition_resolution`, proving that its members and +quarantine tiers were materialized from its current composition. ### Get Virtual Track with Resolved Content @@ -1147,6 +1153,10 @@ GET /api/release-tracks/:id/snapshots/:modified/release/preview?format=bundle POST /api/release-tracks/:id/snapshots/:modified/release ``` +If composition changes after materialization, repeat the create step. Direct +member replacement through either standard-track `/contents` endpoint is +rejected for virtual tracks. + ### 2. Use Scheduled Snapshots for Consistency Define snapshot schedule up front: From ab36cdb655130a699e99aa44ecba5948f1b7de8f Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:33:19 -0400 Subject: [PATCH 21/55] feat(release-tracks): resolve virtual quarantine conflicts Add an explicitly virtual-scoped endpoint for selecting an exact quarantined revision into a new draft. Preserve materialization provenance, reconcile back-references, and update the API contract and regression coverage. --- app/api/definitions/openapi.yml | 3 + .../paths/release-tracks-paths.yml | 33 +++ app/controllers/release-tracks-controller.js | 26 +++ .../release-tracks/release-track-schemas.js | 9 + app/routes/release-tracks-routes.js | 8 + .../release-tracks/release-tracks-service.js | 4 + .../release-tracks/virtual-track-service.js | 54 +++++ .../release-tracks/virtual-quarantine.spec.js | 213 ++++++++++++++++++ docs/developer/release-tracks/entities.md | 2 + .../release-tracks/implementation-notes.md | 11 + docs/user/release-tracks/api-reference.md | 30 +++ docs/user/release-tracks/virtual-tracks.md | 24 +- 12 files changed, 411 insertions(+), 6 deletions(-) create mode 100644 app/tests/api/release-tracks/virtual-quarantine.spec.js diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index c8933474..dec2b7a3 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -391,6 +391,9 @@ paths: /api/release-tracks/{id}/virtual/snapshots/create: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1snapshots~1create' + /api/release-tracks/{id}/virtual/quarantine/promote: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1quarantine~1promote' + /api/release-tracks/{id}/snapshots: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 9422bbe4..7387f020 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -832,6 +832,39 @@ paths: '400': description: 'Track is not virtual or cannot resolve its composition' + /api/release-tracks/{id}/virtual/quarantine/promote: + post: + summary: 'Promote one quarantined revision to virtual members' + operationId: 'release-tracks-virtual-quarantine-promote' + description: | + Resolve a conflict in the latest virtual snapshot by selecting one + exact quarantined object revision. The operation clones the latest + snapshot into a new draft, replaces any existing member revision for + that object, and removes every quarantined alternative with the same + object_ref. The source snapshot and its composition_resolution remain + unchanged. Request body is strictly validated via Zod in the controller. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + '200': + description: 'Quarantined revision promoted in a new virtual draft' + '400': + description: 'Track is not virtual or the request body is invalid' + '404': + description: 'The selected exact revision is not quarantined' + # ============================================================================= # Snapshot-specific operations # ============================================================================= diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index e652124f..3bc462c5 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -50,6 +50,7 @@ const { updateConfigBodySchema, updateCompositionBodySchema, createVirtualSnapshotBodySchema, + promoteQuarantinedObjectBodySchema, xMitreVersionSchema, } = require('../lib/release-tracks/release-track-schemas'); @@ -1021,3 +1022,28 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(req, res, n return next(err); } }; + +/** POST /api/release-tracks/:id/virtual/quarantine/promote */ +exports.promoteQuarantinedObject = async function promoteQuarantinedObject(req, res, next) { + try { + const bodyResult = promoteQuarantinedObjectBodySchema.safeParse(req.body); + if (!bodyResult.success) { + return next( + new BadRequestError({ + message: 'Invalid quarantine promotion request', + details: bodyResult.error.errors, + }), + ); + } + + const result = await releaseTracksService.promoteQuarantinedObject( + req.params.id, + bodyResult.data, + ); + logger.debug(`Success: Promoted quarantined object for track ${req.params.id}`); + return res.status(200).send(result); + } catch (err) { + logger.error('Failed to promote quarantined object: ' + err); + return next(err); + } +}; diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index b6a51b52..3bac9048 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -418,6 +418,14 @@ const createVirtualSnapshotBodySchema = z }) .optional(); +/** POST /release-tracks/:id/virtual/quarantine/promote */ +const promoteQuarantinedObjectBodySchema = z + .object({ + object_ref: stixIdentifierSchema, + object_modified: z.iso.datetime(), + }) + .strict(); + // ============================================================================= // Exports // ============================================================================= @@ -478,6 +486,7 @@ module.exports = { updateConfigBodySchema, updateCompositionBodySchema, createVirtualSnapshotBodySchema, + promoteQuarantinedObjectBodySchema, // Reusable sub-schemas componentTrackSchema, diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index d2f93e64..263caf51 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -239,6 +239,14 @@ router releaseTracksController.createVirtualSnapshot, ); +router + .route('/release-tracks/:id/virtual/quarantine/promote') + .post( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.promoteQuarantinedObject, + ); + // ============================================================================= // Snapshot-specific operations (parameterised by :modified) // ============================================================================= diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 4aa61e4b..a28c923d 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -366,6 +366,10 @@ exports.createVirtualSnapshot = function createVirtualSnapshot(trackId, options) return virtualTrackService.createVirtualSnapshot(trackId, options); }; +exports.promoteQuarantinedObject = function promoteQuarantinedObject(trackId, selection) { + return virtualTrackService.promoteQuarantinedObject(trackId, selection); +}; + // ----------------------------------------------------------------------------- // Object versions (Phase 2 → standard-track-service) // ----------------------------------------------------------------------------- diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index fd2879c1..9762fe67 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -28,6 +28,7 @@ const { TrackNotFoundError, NoTaggedSnapshotsError, InvalidComponentTypeError, + NotFoundError, } = require('../../exceptions'); // ============================================================================= @@ -436,3 +437,56 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op ); return snapshot; }; + +/** + * Resolve one quarantined object by selecting its exact revision. + * + * The latest virtual snapshot is cloned into a new draft. The selected + * revision becomes the sole member entry for its object_ref, and every + * quarantined alternative for that object_ref is removed. The original + * composition_resolution remains unchanged as materialization provenance. + * + * @param {string} trackId + * @param {Object} selection - { object_ref, object_modified } + * @returns {Promise} The new draft snapshot + */ +exports.promoteQuarantinedObject = async function promoteQuarantinedObject(trackId, selection) { + const source = await snapshotService.getLatestSnapshot(trackId); + assertVirtualTrack(source); + + const selectedTime = new Date(selection.object_modified).getTime(); + const selected = (source.quarantine || []).find( + (entry) => + entry.object_ref === selection.object_ref && + new Date(entry.object_modified).getTime() === selectedTime, + ); + + if (!selected) { + throw new NotFoundError({ + details: + `Revision '${selection.object_modified}' of '${selection.object_ref}' ` + + `was not found in the latest snapshot's quarantine tier`, + }); + } + + const members = (source.members || []) + .filter((entry) => entry.object_ref !== selected.object_ref) + .concat({ + object_ref: selected.object_ref, + object_modified: selected.object_modified, + }); + const quarantine = (source.quarantine || []).filter( + (entry) => entry.object_ref !== selected.object_ref, + ); + + const snapshot = await snapshotService.cloneSnapshot(trackId, source, { + members, + quarantine, + }); + + logger.verbose( + `VirtualTrackService: Promoted quarantined revision "${selected.object_ref}" ` + + `at ${new Date(selected.object_modified).toISOString()} in track "${trackId}"`, + ); + return snapshot; +}; diff --git a/app/tests/api/release-tracks/virtual-quarantine.spec.js b/app/tests/api/release-tracks/virtual-quarantine.spec.js new file mode 100644 index 00000000..34f91d15 --- /dev/null +++ b/app/tests/api/release-tracks/virtual-quarantine.spec.js @@ -0,0 +1,213 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Virtual release-track quarantine API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, status = 200) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + async function get(path, status = 200) { + const response = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + async function createTrack(name, type = 'standard', composition) { + return post('/api/release-tracks/new', { name, type, composition }, 201); + } + + async function createReleasedComponent(name, member) { + const track = await createTrack(name); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [ + { + obj_ref: member.stix.id, + obj_modified: member.stix.modified, + }, + ], + }); + await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}); + return track; + } + + async function getTechniqueVersion(technique) { + return get( + `/api/techniques/${technique.stix.id}/modified/${encodeURIComponent(technique.stix.modified)}`, + ); + } + + function entryForTrack(object, trackId) { + return (object.workspace.release_tracks || []).find((entry) => entry.id === trackId); + } + + it('validates quarantine promotion requests and enforces virtual track type', async function () { + const technique = await post('/api/techniques', buildTechnique('Quarantine Guard'), 201); + const standard = await createTrack('Quarantine Standard Guard'); + const body = { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }; + + await post(`/api/release-tracks/${standard.id}/virtual/quarantine/promote`, body, 400); + await post( + `/api/release-tracks/${standard.id}/virtual/quarantine/promote`, + { object_ref: technique.stix.id }, + 400, + ); + }); + + it('promotes one exact revision and removes its quarantined alternatives in a new draft', async function () { + const revisionA = await post('/api/techniques', buildTechnique('Quarantine Resolution A'), 201); + const revisionB = await post( + '/api/techniques', + buildTechnique('Quarantine Resolution B', revisionA), + 201, + ); + const componentA = await createReleasedComponent('Quarantine Component A', revisionA); + const componentB = await createReleasedComponent('Quarantine Component B', revisionB); + const virtual = await createTrack('Quarantine Resolution Virtual', 'virtual', { + component_tracks: [ + { + track_id: componentA.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + { + track_id: componentB.id, + resolution_strategy: 'latest_tagged', + priority: 2, + }, + ], + deduplication: { strategy: 'quarantine' }, + }); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + 201, + ); + + expect(materialized.members).toEqual([]); + expect(materialized.quarantine).toHaveLength(2); + expect(materialized.quarantine.map((entry) => entry.object_modified).sort()).toEqual( + [revisionA.stix.modified, revisionB.stix.modified].sort(), + ); + + const materializedResolution = materialized.composition_resolution; + const revisionABefore = await getTechniqueVersion(revisionA); + const revisionBBefore = await getTechniqueVersion(revisionB); + expect(entryForTrack(revisionABefore, virtual.id)).toMatchObject({ + type: 'virtual', + tier: 'quarantine', + }); + expect(entryForTrack(revisionBBefore, virtual.id)).toMatchObject({ + type: 'virtual', + tier: 'quarantine', + }); + + await post( + `/api/release-tracks/${virtual.id}/virtual/quarantine/promote`, + { + object_ref: revisionA.stix.id, + object_modified: new Date(new Date(revisionB.stix.modified).getTime() + 1000).toISOString(), + }, + 404, + ); + const unchanged = await get(`/api/release-tracks/${virtual.id}/snapshots/latest`); + expect(unchanged.modified).toBe(materialized.modified); + + const promoted = await post(`/api/release-tracks/${virtual.id}/virtual/quarantine/promote`, { + object_ref: revisionB.stix.id, + object_modified: revisionB.stix.modified, + }); + + expect(promoted.modified).not.toBe(materialized.modified); + expect(promoted.version).toBeNull(); + expect(promoted.members).toEqual([ + { + object_ref: revisionB.stix.id, + object_modified: revisionB.stix.modified, + }, + ]); + expect(promoted.quarantine).toEqual([]); + expect(promoted.composition_resolution).toEqual(materializedResolution); + + const historical = await get( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(materialized.modified)}`, + ); + expect(historical.members).toEqual([]); + expect(historical.quarantine).toHaveLength(2); + + const revisionAAfter = await getTechniqueVersion(revisionA); + const revisionBAfter = await getTechniqueVersion(revisionB); + expect(entryForTrack(revisionAAfter, virtual.id)).toBeUndefined(); + expect(entryForTrack(revisionBAfter, virtual.id)).toEqual({ + id: virtual.id, + type: 'virtual', + tier: 'members', + status: 'reviewed', + }); + + const preview = await get(`/api/release-tracks/${virtual.id}/snapshots/latest/release/preview`); + expect(preview).toMatchObject({ + type: 'virtual', + releasable: true, + after: { members_count: 1, quarantine_count: 0 }, + changes: { quarantined_count: 0 }, + }); + }); +}); diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index eba00bb9..41126779 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -463,3 +463,5 @@ Virtual release tracks compute their contents by aggregating objects from compon - All snapshots start as **drafts** and must be explicitly tagged - Component tracks must exist and have at least one tagged release - Each component track must have a unique **priority** value (no duplicates) +- Quarantine promotion selects an exact revision in a new draft and preserves + the source snapshot's immutable `composition_resolution` diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 56d65bc2..6665093e 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -70,6 +70,9 @@ Virtual-only operations are deliberately scoped beneath - `POST /virtual/snapshots/create` resolves tagged component snapshots and persists the concrete members, quarantine, and immutable `composition_resolution`. +- `POST /virtual/quarantine/promote` clones the latest virtual snapshot, + selects one exact quarantined revision for members, and removes all + quarantined alternatives for that object. There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a @@ -78,6 +81,14 @@ virtual draft without `composition_resolution` with `409 Conflict`. Generic latest and historical `/contents` mutations are standard-only; virtual membership has composition resolution as its sole authority. +Quarantine promotion is a snapshot mutation, not a composition +re-resolution. It preserves `composition_resolution` so that field continues +to describe the immutable component inputs and deduplication result that +created the source draft. The preceding snapshot retains every quarantined +source alternative; the new draft records the operator's choice through its +exact member revision. Normal clone behavior reconciles latest-snapshot object +back-references after the move. + For virtual summary previews, `versioning-service` loads the latest tagged snapshot whose `modified` timestamp is strictly earlier than the selected draft. This chronological lookup matters for historical drafts: a release diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 57675920..07744434 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -107,6 +107,7 @@ GET /api/release-tracks/:id/objects/:objectRef/versions ``` PUT /api/release-tracks/:id/virtual/composition POST /api/release-tracks/:id/virtual/snapshots/create +POST /api/release-tracks/:id/virtual/quarantine/promote ``` --- @@ -1196,6 +1197,35 @@ preview is the authoritative comparison and representation of the persisted draft that would be tagged. A non-null `composition_resolution` is the readiness marker for those shared release operations. +### Promote a Quarantined Virtual Revision + +``` +POST /api/release-tracks/:id/virtual/quarantine/promote +``` + +Select one exact quarantined revision for membership in the latest virtual +snapshot: + +```json +{ + "object_ref": "attack-pattern--11111111-1111-4111-8111-111111111111", + "object_modified": "2024-02-01T10:00:00Z" +} +``` + +The selected `(object_ref, object_modified)` pair must exist in the latest +snapshot's `quarantine` tier. A successful request creates a new draft, +replaces any existing member revision for that object with the selected +revision, and removes every quarantined alternative with the same +`object_ref`. The materialized source snapshot remains unchanged and +retrievable by its `modified` timestamp. Its `composition_resolution` is +carried forward unchanged as the immutable record of the original component +resolution. + +The endpoint returns `400 Bad Request` for standard tracks or malformed +requests and `404 Not Found` when the exact selected revision is not +quarantined. + --- ## Query Variations diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 567d0e6f..47a2b679 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -365,7 +365,10 @@ deduplication: { **Use case:** "Conflicts require human review; don't automatically choose a version" -**Follow-up workflow:** Users review the quarantined objects and manually promote one version to `members` during a future snapshot update. The quarantined objects remain in the virtual track until manual intervention occurs. +**Follow-up workflow:** Users review the quarantined objects and manually +promote one exact version to `members`. Promotion creates a new draft and +removes every quarantined alternative for that object. Other quarantined +objects remain until separately resolved. ### Virtual Track Two-Tier System @@ -921,21 +924,30 @@ GET /api/release-tracks/:id/snapshots/latest?include=quarantine **Manually promote a quarantined object to members:** ```bash -POST /api/release-tracks/:id/quarantine/promote +POST /api/release-tracks/:id/virtual/quarantine/promote ``` **Request:** ```json { - "object_ref": "intrusion-set--APT1", + "object_ref": "intrusion-set--11111111-1111-4111-8111-111111111111", "object_modified": "2024-02-01T10:00:00Z" } ``` **Effect:** -- Moves the specified version from `quarantine` to `members` -- Removes other versions of the same object from `quarantine` -- Next snapshot tagging will include this object in the release +- Requires the exact `(object_ref, object_modified)` pair to be quarantined +- Creates a new draft with the selected revision in `members` +- Replaces any prior member revision with the same `object_ref` +- Removes every version of the same object from `quarantine` +- Leaves the materialized source snapshot and its composition-resolution + provenance unchanged +- Reconciles object back-references to the new latest snapshot +- Allows the next snapshot tagging operation to include the selected revision + +Malformed requests and attempts against standard tracks return `400 Bad +Request`. Selecting a revision that is not quarantined returns `404 Not Found` +without creating a snapshot. ## Hybrid Model: Virtual Track + Native Objects From a50a78a8aee44be1018712a8f26618cfe55a7268 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:59:03 -0400 Subject: [PATCH 22/55] fix(release-tracks): validate virtual composition contracts Reject unknown composition properties and enforce strategy-specific component selectors across virtual-track creation and updates. Align OpenAPI, documentation, frontend guidance, and Bruno examples. --- VIRTUAL_TRACKS_TODO.md | 128 ---- .../definitions/components/release-tracks.yml | 75 ++- .../paths/release-tracks-paths.yml | 15 +- .../release-tracks/release-track-schemas.js | 57 +- .../virtual-composition-validation.spec.js | 146 +++++ docs/README.md | 4 + docs/developer/FRONTEND_TODO.md | 549 ++++++++++++++++++ docs/developer/TODO.md | 211 ++++++- docs/developer/release-tracks/entities.md | 10 +- .../release-tracks/implementation-notes.md | 8 + docs/user/release-tracks/api-reference.md | 10 + docs/user/release-tracks/virtual-tracks.md | 18 +- 12 files changed, 1066 insertions(+), 165 deletions(-) delete mode 100644 VIRTUAL_TRACKS_TODO.md create mode 100644 app/tests/api/release-tracks/virtual-composition-validation.spec.js create mode 100644 docs/developer/FRONTEND_TODO.md diff --git a/VIRTUAL_TRACKS_TODO.md b/VIRTUAL_TRACKS_TODO.md deleted file mode 100644 index 968d3ff2..00000000 --- a/VIRTUAL_TRACKS_TODO.md +++ /dev/null @@ -1,128 +0,0 @@ -# Virtual Release Tracks Completion Backlog - -This backlog records the 2026-07-29 documentation-to-implementation audit of -virtual release tracks. Items are ordered by integrity risk and implementation -dependency. A checked item must include regression coverage and any necessary -OpenAPI, user/developer documentation, client, and Bruno updates. - -## P0 — Snapshot lifecycle integrity - -- [x] Make composition changes invalidate the previous materialization: - - clear inherited `members`, `quarantine`, and `composition_resolution`; - - expose that the resulting virtual draft is awaiting materialization; - - require `POST /api/release-tracks/:id/virtual/snapshots/create` before the - draft can be previewed or tagged as a release. -- [x] Reject generic member replacement for virtual tracks: - - `POST /api/release-tracks/:id/contents`; - - `POST /api/release-tracks/:id/snapshots/:modified/contents`. - Virtual membership must only be produced by composition resolution. -- [ ] Implement the documented quarantine-resolution workflow, including - `POST /api/release-tracks/:id/quarantine/promote`, or remove the quarantine - strategy from the public contract until conflicts can be resolved. - -## P1 — Composition validation and deterministic resolution - -- [ ] Make request validation strict so misspelled keys such as - `filters.domain` return 400 instead of silently disabling filtering. -- [ ] Validate component selectors according to `resolution_strategy`: - - `specific_version` requires `version` and rejects `snapshot`; - - `specific_snapshot` requires `snapshot` and rejects `version`; - - `latest_tagged` rejects both selector fields. -- [ ] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, - and examples; reject duplicate priorities at the request boundary. -- [ ] Validate component existence, standard-track type, duplicate track IDs, - and duplicate priorities when a virtual track is initially created, not only - when composition is later updated or materialized. -- [ ] Validate `snapshot_schedule` by mode: - - `manual` rejects `cron` and `dates`; - - `cron` requires `cron` and rejects `dates`; - - `dates` requires at least one date and rejects `cron`. -- [ ] Constrain or document accepted `filters.object_types` values and add - direct regression coverage for exact-revision filtering. - -## P1 — Deduplication correctness - -- [ ] Treat the same exact object revision contributed by multiple components - as one duplicate, not a conflicting revision. -- [ ] Ensure the `quarantine` strategy only quarantines genuinely different - revisions of the same object. -- [ ] Attribute each surviving revision to one deterministic component so - `objects_contributed` totals cannot exceed `summary.total_objects`. -- [ ] Add dedicated tests for all four strategies: - `prioritize_latest_object`, `prioritize_latest_snapshot`, - `prioritize_higher_priority`, and `quarantine`. - -## P1 — Release provenance - -- [ ] Populate virtual release `version_history[].component_versions` from the - materialized snapshot's immutable `composition_resolution`. -- [ ] Define and test the provenance shape in Mongoose, OpenAPI, and user and - developer documentation. - -## P2 — Scheduled materialization - -- [ ] Connect virtual `snapshot_schedule` metadata to the existing task - scheduler. -- [ ] Implement manual, cron, and explicit-date scheduling semantics. -- [ ] Define failure behavior when a component has no matching tagged - snapshot, including automation-run audit records and retry policy. -- [ ] Add scheduler integration tests and operational documentation. - -## P2 — Contract decisions - -- [ ] Decide whether virtual tracks can compose virtual tracks. The - implementation currently rejects nesting while portions of the - documentation say standard or virtual components are supported. -- [ ] Decide whether to implement the documented native-members/hybrid model. - Prefer a dedicated standard component track unless a demonstrated use case - requires a second membership authority inside virtual tracks. -- [ ] Decide whether to implement `resolve=true` and `resolved_content`. - Remove these claims from documentation if eager materialization remains the - only supported model. -- [ ] Implement caching and component-release notifications only if measured - scale or an approved product workflow requires them; otherwise describe them - as future considerations rather than current capabilities. - -## Documentation corrections - -- [ ] Replace `stix.type = "virtual"` with the top-level snapshot - `type: "virtual"`. -- [ ] Remove the nonexistent snapshot-level `snapshot_id`; retain - `version_history[].snapshot_id`. -- [ ] Correct response envelopes and the virtual-create response example. -- [ ] Align `composition_resolution` examples with fields actually generated, - or implement the documented `by_type`, `by_tier`, and native statistics. -- [ ] Align documented error envelopes with centralized error-handler output. -- [ ] Include required `priority` values in every composition example. -- [ ] Clearly distinguish configured composition from a materialized draft and - describe scheduled behavior as unavailable until scheduler execution exists. - -## Verified complete - -- [x] `filters.domains` hydrates and evaluates exact pinned revisions. -- [x] Public domain names and STIX `*-attack` names are normalized. -- [x] Multiple domain values are supported. -- [x] Objects without domain metadata are excluded when a domain filter is set. -- [x] Primary Enterprise, ICS, and Mobile matrices use their ATT&CK external ID - as the established domain fallback. -- [x] Virtual tracks resolve only tagged snapshots and consume only component - `members`. -- [x] Virtual tracks maintain independent draft/release history and use the - shared snapshot retrieval and release endpoints after materialization. - -## Current implementation slice - -- [x] Add failing lifecycle and type-boundary regression tests. -- [x] Invalidate inherited materialization when composition changes. -- [x] Reject release previews and release commits for unmaterialized virtual - drafts. -- [x] Reject generic contents replacement for virtual tracks. -- [x] Update OpenAPI, user/developer docs, and Bruno. -- [x] Run focused specs followed by the complete `npm test` suite. - -Verification completed 2026-07-29: - -- Focused virtual/release regression suite: 46 passing. -- Lint: passing. -- Full test suite: 960 passing (OpenAPI 2, config 21, API 913, - middleware 24). diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index e6b99d1c..06dc855e 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -288,24 +288,42 @@ components: composition: type: object description: 'Virtual track composition (references to component standard tracks)' + additionalProperties: false + required: + - component_tracks properties: component_tracks: type: array + minItems: 1 items: $ref: '#/components/schemas/component-track' - deduplication_strategy: - type: string - enum: - - prioritize_latest_object - - prioritize_latest_snapshot - - prioritize_higher_priority - - quarantine - description: 'How to resolve duplicate objects across components' - default: 'prioritize_latest_object' + deduplication: + type: object + additionalProperties: false + required: + - strategy + properties: + strategy: + type: string + enum: + - prioritize_latest_object + - prioritize_latest_snapshot + - prioritize_higher_priority + - quarantine + description: 'How to resolve duplicate objects across components' + default: 'prioritize_latest_object' component-track: type: object - description: 'Reference to a component track in a virtual track composition' + description: | + Reference to a component standard track. Selector fields are determined + by resolution_strategy: latest_tagged rejects version and snapshot; + specific_version requires only version; specific_snapshot requires only + snapshot. Unknown properties are rejected. + additionalProperties: false + required: + - track_id + - resolution_strategy properties: track_id: type: string @@ -324,16 +342,15 @@ components: default: 'latest_tagged' version: type: string - nullable: true description: 'Specific version to pin to (when resolution_strategy is specific_version)' - snapshot_modified: + snapshot: type: string format: date-time - nullable: true description: 'Specific snapshot to pin to (when resolution_strategy is specific_snapshot)' filters: type: object description: 'Optional filters to apply to component members' + additionalProperties: false properties: object_types: type: array @@ -345,6 +362,38 @@ components: items: type: string description: 'Only include exact pinned object revisions whose x_mitre_domains intersects these ATT&CK domains. Primary matrices fall back to external_references.external_id. Accepts enterprise/mobile/ics and their -attack forms.' + oneOf: + - title: 'Latest tagged release' + properties: + resolution_strategy: + enum: + - latest_tagged + not: + anyOf: + - required: + - version + - required: + - snapshot + - title: 'Specific release version' + required: + - version + properties: + resolution_strategy: + enum: + - specific_version + not: + required: + - snapshot + - title: 'Specific tagged snapshot' + required: + - snapshot + properties: + resolution_strategy: + enum: + - specific_snapshot + not: + required: + - version version-history-entry: type: object diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 7387f020..b5e0ecf7 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -223,7 +223,9 @@ paths: operationId: 'release-tracks-create' description: | Create a new standard or virtual release track with an initial empty draft snapshot. - Request body is validated via Zod (not OpenAPI). See controller for schema. + Request body is validated via Zod (not OpenAPI). Virtual composition + objects are strict, and component selectors must match their + resolution_strategy. tags: - 'Release Tracks' # Request body validation moved to Zod in controller @@ -793,7 +795,10 @@ paths: operation is available only for tracks whose type is `virtual`. The new pending draft has empty members and quarantine tiers and a null composition_resolution. Materialize it before release preview or commit. - Request body validated via Zod in controller. + Request body is strictly validated via Zod. Unknown composition, + component, filter, and deduplication keys are rejected. latest_tagged + rejects selector fields; specific_version requires version; + specific_snapshot requires snapshot. tags: - 'Release Tracks' parameters: @@ -802,6 +807,12 @@ paths: required: true schema: type: string + requestBody: + required: true + content: + application/json: + schema: + type: object responses: '200': description: 'Composition updated successfully' diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 3bac9048..f3794027 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -263,28 +263,53 @@ const snapshotScheduleSchema = z.object({ dates: z.array(z.iso.datetime()).optional(), }); -const componentTrackSchema = z.object({ +const componentTrackFiltersSchema = z + .object({ + object_types: z.array(z.string()).optional(), + domains: z.array(z.string()).optional(), + }) + .strict(); + +const componentTrackBaseShape = { track_id: releaseTrackIdSchema, - resolution_strategy: resolutionStrategySchema, priority: z.number().int().min(0).optional(), - version: xMitreVersionSchema.optional(), - snapshot: z.iso.datetime().optional(), - filters: z + filters: componentTrackFiltersSchema.optional(), +}; + +const componentTrackSchema = z.discriminatedUnion('resolution_strategy', [ + z .object({ - object_types: z.array(z.string()).optional(), - domains: z.array(z.string()).optional(), + ...componentTrackBaseShape, + resolution_strategy: z.literal('latest_tagged'), }) - .optional(), -}); - -const compositionSchema = z.object({ - component_tracks: z.array(componentTrackSchema).min(1), - deduplication: z + .strict(), + z .object({ - strategy: deduplicationStrategySchema, + ...componentTrackBaseShape, + resolution_strategy: z.literal('specific_version'), + version: xMitreVersionSchema, }) - .optional(), -}); + .strict(), + z + .object({ + ...componentTrackBaseShape, + resolution_strategy: z.literal('specific_snapshot'), + snapshot: z.iso.datetime(), + }) + .strict(), +]); + +const compositionSchema = z + .object({ + component_tracks: z.array(componentTrackSchema).min(1), + deduplication: z + .object({ + strategy: deduplicationStrategySchema, + }) + .strict() + .optional(), + }) + .strict(); const createTrackBodySchema = z.object({ name: trackNameSchema, diff --git a/app/tests/api/release-tracks/virtual-composition-validation.spec.js b/app/tests/api/release-tracks/virtual-composition-validation.spec.js new file mode 100644 index 00000000..1e53176d --- /dev/null +++ b/app/tests/api/release-tracks/virtual-composition-validation.spec.js @@ -0,0 +1,146 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +describe('Virtual release-track composition validation API', function () { + let app; + let passportCookie; + let componentTrack; + let virtualTrack; + let createSequence = 0; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + + componentTrack = await post('/api/release-tracks/new', { + name: 'Composition Validation Component', + type: 'standard', + }); + virtualTrack = await post('/api/release-tracks/new', { + name: 'Composition Validation Virtual', + type: 'virtual', + }); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + async function putComposition(composition, status = 200) { + const response = await request(app) + .put(`/api/release-tracks/${virtualTrack.id}/virtual/composition`) + .send(composition) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + function component(resolutionStrategy, overrides = {}) { + return { + track_id: componentTrack.id, + resolution_strategy: resolutionStrategy, + priority: 1, + ...overrides, + }; + } + + function composition(componentTrack, overrides = {}) { + return { + component_tracks: [componentTrack], + deduplication: { strategy: 'prioritize_latest_object' }, + ...overrides, + }; + } + + async function createVirtual(compositionBody, status = 201) { + createSequence += 1; + return post( + '/api/release-tracks/new', + { + name: `Strict Composition Create ${createSequence}`, + type: 'virtual', + composition: compositionBody, + }, + status, + ); + } + + it('rejects unknown composition keys instead of silently stripping them', async function () { + const invalidCompositions = [ + composition(component('latest_tagged'), { unexpected: true }), + composition(component('latest_tagged', { unexpected: true })), + composition( + component('latest_tagged', { + filters: { domains: ['enterprise'], domain: 'enterprise' }, + }), + ), + composition(component('latest_tagged'), { + deduplication: { + strategy: 'prioritize_latest_object', + fallback: 'quarantine', + }, + }), + ]; + + for (const invalidComposition of invalidCompositions) { + await createVirtual(invalidComposition, 400); + await putComposition(invalidComposition, 400); + } + }); + + it('requires and restricts selectors according to resolution_strategy', async function () { + const timestamp = '2024-02-01T10:00:00.000Z'; + const invalidComponents = [ + component('latest_tagged', { version: '1.0' }), + component('latest_tagged', { snapshot: timestamp }), + component('specific_version'), + component('specific_version', { snapshot: timestamp }), + component('specific_version', { version: '1.0', snapshot: timestamp }), + component('specific_snapshot'), + component('specific_snapshot', { version: '1.0' }), + component('specific_snapshot', { version: '1.0', snapshot: timestamp }), + ]; + + for (const invalidComponent of invalidComponents) { + await createVirtual(composition(invalidComponent), 400); + await putComposition(composition(invalidComponent), 400); + } + }); + + it('accepts only the selector defined by each resolution strategy', async function () { + const timestamp = '2024-02-01T10:00:00.000Z'; + const validComponents = [ + component('latest_tagged'), + component('specific_version', { version: '1.0' }), + component('specific_snapshot', { snapshot: timestamp }), + ]; + + for (const validComponent of validComponents) { + const created = await createVirtual(composition(validComponent)); + expect(created.composition.component_tracks[0]).toMatchObject(validComponent); + + const updated = await putComposition(composition(validComponent)); + expect(updated.composition.component_tracks[0]).toMatchObject(validComponent); + } + }); +}); diff --git a/docs/README.md b/docs/README.md index d3b4cfdc..1ceae506 100644 --- a/docs/README.md +++ b/docs/README.md @@ -40,6 +40,10 @@ Architecture, patterns, and implementation details for contributors. ### Release Tracks (Internals) +- [Implementation Backlog](developer/TODO.md): Active release-track work and + completed implementation records +- [Frontend Handoff](developer/FRONTEND_TODO.md): Backend contract changes + requiring downstream Angular updates - [Entities](developer/release-tracks/entities.md): Database schemas and data models - [Backref Reconciliation](developer/release-tracks/backref-reconciliation.md): How `workspace.release_tracks` backrefs stay in sync with snapshots - [Member Sync Strategies](developer/release-tracks/member-sync-strategies.md): Automatic tracking of member object revisions diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md new file mode 100644 index 00000000..376dbb29 --- /dev/null +++ b/docs/developer/FRONTEND_TODO.md @@ -0,0 +1,549 @@ +# Release Tracks Frontend TODO + +This developer handoff tracks backend release-track changes that require +corresponding work in the Angular frontend. It is intentionally task-oriented, +but each task also explains why the change matters so that it can be +implemented without reconstructing the backend design history. + +The current server contract is defined by: + +- [Release-track OpenAPI paths](../../app/api/definitions/paths/release-tracks-paths.yml) +- [Release-track API reference](../user/release-tracks/api-reference.md) +- [Virtual-track guide](../user/release-tracks/virtual-tracks.md) +- [Bruno release-track requests](https://gitlab.mitre.org/attack-strategy/bruno/-/tree/main/workbench/Release%20Tracks?ref_type=heads) +- [`internalattack` Python client](https://gitlab.mitre.org/attack-strategy/internalattack-python) + +## Endpoint conventions + +Keep these rules in mind while updating the connector: + +- Operations shared by standard and virtual tracks do not include a type + namespace. Snapshot retrieval and release operations are shared. +- New virtual-only operations include `/virtual/` in the path. +- The current OpenAPI document is authoritative. Some older standard-only + workflow routes, such as `/candidates`, `/staged`, and `/contents`, predate + the namespace convention and do not currently include `/standard/`. +- A release preview is a read-only `GET`. A release commit is a `POST`. + +## P0 — Align the Angular connector with the current routes + +### [ ] Use only the explicit snapshot-retrieval endpoints + +The release-track resource path no longer doubles as an implicit request for +the latest snapshot. `GET /api/release-tracks/:id` was removed before the +feature was officially released, so there is no compatibility alias. + +Update the frontend to use: + +```text +GET /api/release-tracks/:id/snapshots +GET /api/release-tracks/:id/snapshots/latest +GET /api/release-tracks/:id/snapshots/:modified +``` + +Required work: + +- Keep `getLatestSnapshot()` on `/snapshots/latest`. +- Update the integration test that still calls `GET /release-tracks/:id`. +- Remove any fallback that interprets a full snapshot or `version_history` as + the snapshot-list response. +- Remove the unsupported `releases=only` option from track-list and + snapshot-retrieval types. Tagged-state filtering now belongs on the snapshot + history endpoint as `tagged=true`. + +Done when: + +- No Angular code calls `GET /api/release-tracks/:id`. +- Connector and integration tests assert the three explicit retrieval paths. + +### [ ] Move virtual-only operations under `/virtual/` + +Virtual composition and materialization are now visibly scoped in the URL: + +```text +PUT /api/release-tracks/:id/virtual/composition +POST /api/release-tracks/:id/virtual/snapshots/create +``` + +The Angular connector still calls the older `/composition` and +`/snapshots/create` paths. Update those paths and their tests. + +Also remove: + +```text +GET /api/release-tracks/:id/snapshots/preview +``` + +That endpoint no longer exists. It recomputed a hypothetical virtual +composition without persisting it, which overlapped confusingly with release +preview. Creating a virtual draft is now an explicit operation. The current +`onDraft()` flow should therefore open a confirmation dialog and call the +create endpoint directly instead of first calling `previewVirtualSnapshot()`. + +Done when: + +- `updateComposition()` calls `/virtual/composition`. +- `createVirtualSnapshot()` calls `/virtual/snapshots/create`. +- `previewVirtualSnapshot()` and its UI/test fixtures are removed. +- Creating a virtual draft still asks for confirmation, but does not depend on + a nonexistent preview payload. + +## P0 — Replace “bump” with the release contract + +### [ ] Rename bump-oriented frontend symbols and user-facing text + +“Release” is now the operation name throughout the API. The server has no +`/bump` routes, and retaining bump terminology in Angular makes logs, types, +tests, and UI copy disagree with the public contract. + +Suggested renames: + +```text +BumpPayload -> ReleasePayload +previewBump() -> previewRelease() +bumpByLatest() -> releaseLatest() +bumpByModified() -> releaseSnapshot() +bumpRelease() -> releaseSnapshot() +``` + +Update comments, test names, log messages, and errors such as “Failed to +preview release track bump” at the same time. + +Done when: + +- “bump” is absent from the release-track connector, models, components, and + tests unless it appears in a historical explanation. +- Angular method names distinguish previewing from committing a release. + +### [ ] Replace the old release request body + +The current frontend `BumpPayload` is obsolete: + +```ts +{ + type?: 'major' | 'minor'; + version?: string; + dry_run?: boolean; +} +``` + +The release body is now: + +```ts +{ + increment?: 'major' | 'minor'; + version?: string; // exact MAJOR.MINOR, for example "14.1" +} +``` + +The rules are: + +- `increment` and `version` are mutually exclusive. +- Supplying both returns `400 Bad Request`. +- Omitting both asks the server for the default minor increment. +- `dry_run` was removed. Use a release-preview representation instead. +- `expected_snapshot_modified` is not required. Choosing `latest` means the + caller accepts whichever snapshot is latest when the server handles the + request; choosing `:modified` explicitly pins the target. + +Use the same selector in the preview query and the release request body. For +example: + +```text +GET .../release/preview?format=summary&increment=major +POST .../release +Body: { "increment": "major" } +``` + +Done when: + +- Angular never sends `type`, `dry_run`, or + `expected_snapshot_modified` in a release request. +- The UI can select `major`, `minor`, or an explicitly entered `MAJOR.MINOR` + version and validates mutual exclusivity before calling the server. + +## P0 — Correct the release-preview flow + +### [ ] Treat release preview as a `GET` with representation-specific output + +Both track types use the same preview routes: + +```text +GET /api/release-tracks/:id/snapshots/latest/release/preview +GET /api/release-tracks/:id/snapshots/:modified/release/preview +``` + +The supported preview formats are: + +- `summary` — a before/after delta; this is the default +- `workbench` — the complete snapshot that would be persisted +- `bundle` — the publication-ready STIX bundle +- `filesystemstore` — reserved but currently returns `501 Not Implemented` + +The existing Angular flow requests `format=workbench` and then reads summary +fields such as `next_version_minor` and `staged_count`. Those are different +representations and must not be mixed. + +Recommended interaction: + +1. Ask the operator to choose `major`, `minor`, or an exact version. +2. Request `format=summary` using that selector. +3. Render the returned `version`, `before`, `after`, `changes`, and + `conflicts`. +4. Optionally let the operator inspect `format=workbench` or `format=bundle` + using the same selector. +5. Commit the release using the same selector only after confirmation. + +The summary contains the planned `version`; it does not return separate +`next_version_minor` and `next_version_major` fields. + +Done when: + +- The confirmation dialog is driven by a `summary` response. +- Workbench and bundle previews are treated as literal payloads, not deltas. +- The preview and commit always use the same target snapshot and version + selector. + +### [ ] Render standard and virtual summaries differently + +Standard and virtual snapshots share the preview endpoint, but their deltas +answer different questions. + +For a standard track: + +- `before` is the selected draft. +- `after` is the would-be release after staged objects become members. +- Counts are oriented around `members_count`, `staged_count`, and + `candidates_count`. + +For a virtual track: + +- `before` is the tagged release immediately preceding the selected draft. +- `after` is the selected, already-materialized virtual draft. +- Counts are oriented around `members_count` and `quarantine_count`. +- `previous_release` identifies the comparison baseline when one exists. +- `changes` can include `new_count`, `updated_count`, `removed_count`, and + `quarantined_count`. + +Virtual release does not promote staged objects because virtual tracks do not +have a staged tier. Avoid showing zero-valued staged/candidate statistics as +though they described the virtual workflow. + +Done when: + +- Summary rendering branches on the top-level `type`. +- Standard previews explain staged-to-member promotion. +- Virtual previews explain the delta from the preceding tagged release. +- A first virtual release handles `previous_release: null` cleanly. + +## P0 — Represent virtual materialization state + +### [ ] Show when a virtual draft is awaiting materialization + +Saving a new virtual composition now invalidates the preceding materialized +contents. The new latest draft intentionally has: + +```json +{ + "type": "virtual", + "members": [], + "quarantine": [], + "composition_resolution": null +} +``` + +This does not mean that the composition resolved to an empty release. It means +the composition has changed and the operator must explicitly materialize a new +draft with: + +```text +POST /api/release-tracks/:id/virtual/snapshots/create +``` + +Until that succeeds, shared release preview and release commit endpoints return +`409 Conflict`. + +Required work: + +- Treat `type === 'virtual' && composition_resolution == null` as “awaiting + materialization.” +- Show an explanatory state rather than an ordinary empty-members view. +- Disable release actions and make “Create Draft” the primary next action. +- After a composition update, refresh both the latest snapshot and history so + the newly invalidated draft is visible. +- Surface a helpful error when a component track has no tagged snapshots. A + virtual materialization can only consume tagged `members` from its component + standard tracks. + +Done when: + +- Operators cannot accidentally interpret an unmaterialized draft as a valid + empty virtual release. +- A `409` from preview/release explains that materialization is required + instead of being swallowed as a null preview. + +### [ ] Keep standard-only mutations out of virtual-track controls + +Direct contents replacement is now explicitly rejected for virtual tracks: + +```text +POST /api/release-tracks/:id/contents +POST /api/release-tracks/:id/snapshots/:modified/contents +``` + +Virtual membership has one authority: composition materialization followed by +optional quarantine resolution. Both contents endpoints return `400 Bad +Request` for a virtual track. + +Hide direct member replacement, candidate, and staged controls when +`type === 'virtual'`. Keep them available for standard tracks on their current +routes. + +Done when: + +- A virtual-track screen does not offer actions the server will reject because + they belong to the standard workflow. + +## P0 — Consume snapshot history as summaries + +### [ ] Type and paginate the snapshot-history response + +`GET /api/release-tracks/:id/snapshots` returns a paginated envelope: + +```json +{ + "data": [], + "pagination": { + "total": 0, + "limit": 50, + "offset": 0 + } +} +``` + +Supported query parameters are: + +- `tagged=true` — tagged releases only +- `tagged=false` — untagged drafts only +- omit `tagged` — both; this is the no-filter default +- `limit` — 1 through 200, default 50 +- `offset` — zero or greater + +The current connector discards `pagination`, accepts no filters, and contains +fallback normalization for older response shapes. Replace that compatibility +logic with the explicit envelope. Otherwise tracks with more than 50 +snapshots are silently truncated. + +Done when: + +- The connector accepts `tagged`, `limit`, and `offset`. +- The component retains pagination metadata and supports paging or loading + more results. +- There is a clear UI control for all snapshots, releases only, and drafts + only. + +### [ ] Use the type-oriented snapshot summary fields + +Snapshot history entries are lightweight summaries, not full snapshots. All +entries include: + +```text +id, type, modified, version, name, description, members_count +``` + +Standard entries additionally include: + +```text +staged_count, candidates_count +``` + +Virtual entries additionally include: + +```text +quarantine_count +``` + +These counts are top-level fields. They are not nested beneath `summary`, and +the history endpoint does not return the tier arrays needed to derive object +deltas. The current `buildSnapshotHistory()` logic therefore reports zero for +several values and should not calculate “Added” or “Modified” by comparing +missing member arrays. + +Required work: + +- Define a discriminated union keyed by `type: 'standard' | 'virtual'`. +- Read the top-level count fields directly. +- Show standard counts as members/staged/candidates. +- Show virtual counts as members/quarantine. +- If the design still needs per-object added/updated/removed deltas, retrieve + an appropriate release summary or full snapshots explicitly rather than + inferring them from the lightweight list response. + +Done when: + +- Snapshot cards display accurate counts for both track types. +- No history calculation assumes the list response contains `members`, + `staged`, `candidates`, or `quarantine` arrays. + +## P1 — Add the virtual quarantine-resolution workflow + +### [ ] Let an operator select an exact quarantined revision + +The backend now provides: + +```text +POST /api/release-tracks/:id/virtual/quarantine/promote +``` + +Request body: + +```json +{ + "object_ref": "attack-pattern--11111111-1111-4111-8111-111111111111", + "object_modified": "2024-02-01T10:00:00Z" +} +``` + +The `(object_ref, object_modified)` pair must exactly match an entry in the +latest virtual snapshot's quarantine tier. On success, the server creates a +new draft, places the selected revision in `members`, removes every +quarantined alternative for that object, and preserves the original snapshot +for history and provenance. + +Suggested UI: + +- Group quarantine entries by `object_ref`. +- Show every conflicting revision, its modified timestamp, and any available + source/component context. +- Require the operator to choose one exact revision. +- Confirm that the other alternatives will be removed from the new draft. +- Refresh the latest snapshot and history after success. + +Error behavior: + +- `400` — malformed body or the target track is not virtual +- `404` — that exact revision is no longer quarantined in the latest snapshot + +A `404` is often a stale-screen condition. Refresh the latest snapshot and ask +the operator to review the current alternatives rather than retrying the old +selection automatically. + +Done when: + +- Each quarantined conflict has an explicit resolution action. +- The request always includes both the STIX ID and exact modified timestamp. +- Successful promotion updates members, quarantine, and snapshot history in + the UI. + +## P1 — Expose implemented virtual component filters + +### [ ] Add plural `filters.domains` to component-track editing + +Virtual composition can filter the exact revisions contributed by each +component track: + +```json +{ + "filters": { + "object_types": ["intrusion-set", "malware"], + "domains": ["enterprise", "mobile"] + } +} +``` + +Both public domain names (`enterprise`, `ics`, `mobile`) and their STIX names +(`enterprise-attack`, `ics-attack`, `mobile-attack`) are accepted. Prefer the +short public names in Angular controls for consistency. + +The key is plural: `domains`. Do not send `filters.domain`; that typo does not +enable filtering. The backend now rejects that typo with `400 Bad Request`. + +Objects without domain metadata are excluded when a domain filter is active. +This is expected behavior, not a partial match. + +Composition payloads are strict at every nested level. Do not preserve +frontend-only properties in the submitted composition, component, filter, or +deduplication objects. Component selector fields must also follow the selected +strategy: + +- `latest_tagged` sends neither `version` nor `snapshot`. +- `specific_version` sends `version` and omits `snapshot`. +- `specific_snapshot` sends `snapshot` and omits `version`. + +Done when: + +- Each virtual component row can select zero or more domains. +- Saved and reloaded composition preserves `filters.domains`. +- Tests assert the plural key and multi-domain payload shape. +- Changing resolution strategy clears the selector from the previous strategy. +- Submitted composition payloads contain only server-supported properties. + +## P1 — Separate snapshot and preview output-format types + +### [ ] Remove the invalid `snapshot` format and model `summary` correctly + +The frontend currently uses one `ExportFormat` enum for endpoints with +different contracts and includes `Snapshot = 'snapshot'`, which the server +rejects. + +Use separate types: + +```ts +type SnapshotOutputFormat = 'workbench' | 'bundle' | 'filesystemstore'; + +type ReleasePreviewFormat = 'summary' | 'workbench' | 'bundle' | 'filesystemstore'; +``` + +`filesystemstore` should remain disabled or clearly marked unavailable until +the server implementation exists. + +For workbench snapshot retrieval, `include` may select `members`, `staged`, +`candidates`, `quarantine`, or `all`. Ensure virtual inspection can request +quarantine and that the type does not restrict `include` to standard tiers. + +Done when: + +- Angular cannot send `format=snapshot`. +- `summary` is available only where the release-preview endpoint supports it. +- Virtual workbench retrieval can include quarantine. + +## P1 — Update frontend tests around the public contract + +### [ ] Replace stale fixtures and add standard/virtual lifecycle coverage + +Update connector, component, and integration tests together so old mock shapes +do not keep obsolete behavior alive. + +Minimum regression coverage: + +- Canonical latest retrieval uses `/snapshots/latest`. +- Snapshot history preserves `{ data, pagination }` and supports `tagged`. +- Release preview uses `GET`, `format=summary`, and query-based version + selection. +- Release commit uses `{ increment }` or `{ version }`. +- Latest and timestamp-selected releases use the same contract. +- Virtual composition and materialization use `/virtual/` paths. +- The removed virtual snapshot-preview method is absent. +- An unmaterialized virtual draft disables release and explains a `409`. +- Standard and virtual summary fixtures use their respective count fields. +- Quarantine promotion sends an exact object revision and handles stale `404` + responses. +- The frontend never offers standard-only contents/candidate/staged mutations + on a virtual track. + +## Backend changes that do not require Angular API changes + +The following changes are useful context but should not create extra connector +work: + +- Snapshot bundle exports now include valid secondary relationships + dynamically. Existing bundle download code receives a more complete bundle + without changing its request. +- Release-track object back-references are reconciled when snapshots change. + Frontend object refreshes will see the updated membership metadata without a + new endpoint. +- The server intentionally does not require + `expected_snapshot_modified`. Do not add a client-side precondition field. +- Virtual release preview and release use the same shared routes as standard + tracks after materialization; do not create separate virtual release + endpoints. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index e4fd0fc4..7e526ed6 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,8 +1,217 @@ # Release Track TODOs +## Virtual release tracks + +This section records the 2026-07-29 documentation-to-implementation audit of +virtual release tracks. Items are ordered by integrity risk and implementation +dependency. A checked item must include regression coverage and any necessary +OpenAPI, user/developer documentation, client, and Bruno updates. + +The completed P0 implementation and verification records remain in the dated +sections below. The following items constitute the active virtual-track +completion backlog. + +### P1 — Composition validation and deterministic resolution + +- [x] Make request validation strict so misspelled keys such as + `filters.domain` return 400 instead of silently disabling filtering. +- [x] Validate component selectors according to `resolution_strategy`: + - `specific_version` requires `version` and rejects `snapshot`; + - `specific_snapshot` requires `snapshot` and rejects `version`; + - `latest_tagged` rejects both selector fields. +- [ ] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, + and examples; reject duplicate priorities at the request boundary. +- [ ] Validate component existence, standard-track type, duplicate track IDs, + and duplicate priorities when a virtual track is initially created, not only + when composition is later updated or materialized. +- [ ] Validate `snapshot_schedule` by mode: + - `manual` rejects `cron` and `dates`; + - `cron` requires `cron` and rejects `dates`; + - `dates` requires at least one date and rejects `cron`. +- [ ] Constrain or document accepted `filters.object_types` values and add + direct regression coverage for exact-revision filtering. + +### P1 — Deduplication correctness + +- [ ] Treat the same exact object revision contributed by multiple components + as one duplicate, not a conflicting revision. +- [ ] Ensure the `quarantine` strategy only quarantines genuinely different + revisions of the same object. +- [ ] Attribute each surviving revision to one deterministic component so + `objects_contributed` totals cannot exceed `summary.total_objects`. +- [ ] Add dedicated tests for all four strategies: + `prioritize_latest_object`, `prioritize_latest_snapshot`, + `prioritize_higher_priority`, and `quarantine`. + +### P1 — Release provenance + +- [ ] Populate virtual release `version_history[].component_versions` from the + materialized snapshot's immutable `composition_resolution`. +- [ ] Define and test the provenance shape in Mongoose, OpenAPI, and user and + developer documentation. + +### P2 — Scheduled materialization + +- [ ] Connect virtual `snapshot_schedule` metadata to the existing task + scheduler. +- [ ] Implement manual, cron, and explicit-date scheduling semantics. +- [ ] Define failure behavior when a component has no matching tagged + snapshot, including automation-run audit records and retry policy. +- [ ] Add scheduler integration tests and operational documentation. + +### P2 — Contract decisions + +- [ ] Decide whether virtual tracks can compose virtual tracks. The + implementation currently rejects nesting while portions of the + documentation say standard or virtual components are supported. +- [ ] Decide whether to implement the documented native-members/hybrid model. + Prefer a dedicated standard component track unless a demonstrated use case + requires a second membership authority inside virtual tracks. +- [ ] Decide whether to implement `resolve=true` and `resolved_content`. + Remove these claims from documentation if eager materialization remains the + only supported model. +- [ ] Implement caching and component-release notifications only if measured + scale or an approved product workflow requires them; otherwise describe them + as future considerations rather than current capabilities. + +### Documentation corrections + +- [ ] Replace `stix.type = "virtual"` with the top-level snapshot + `type: "virtual"`. +- [ ] Remove the nonexistent snapshot-level `snapshot_id`; retain + `version_history[].snapshot_id`. +- [ ] Correct response envelopes and the virtual-create response example. +- [ ] Align `composition_resolution` examples with fields actually generated, + or implement the documented `by_type`, `by_tier`, and native statistics. +- [ ] Align documented error envelopes with centralized error-handler output. +- [ ] Include required `priority` values in every composition example. +- [ ] Clearly distinguish configured composition from a materialized draft and + describe scheduled behavior as unavailable until scheduler execution exists. + +### Verified complete + +- [x] Composition changes invalidate inherited materialized contents and + require explicit rematerialization before release. +- [x] Generic contents replacement rejects virtual tracks. +- [x] Exact-revision quarantine resolution is available at + `POST /api/release-tracks/:id/virtual/quarantine/promote`. +- [x] `filters.domains` hydrates and evaluates exact pinned revisions. +- [x] Public domain names and STIX `*-attack` names are normalized. +- [x] Multiple domain values are supported. +- [x] Objects without domain metadata are excluded when a domain filter is set. +- [x] Primary Enterprise, ICS, and Mobile matrices use their ATT&CK external ID + as the established domain fallback. +- [x] Virtual tracks resolve only tagged snapshots and consume only component + `members`. +- [x] Virtual tracks maintain independent draft/release history and use the + shared snapshot retrieval and release endpoints after materialization. + +### Current implementation slice — Strict composition contracts + +- [x] Add API regression coverage for unknown composition/filter keys on both + virtual-track creation and composition update. +- [x] Require the selector appropriate to each `resolution_strategy` and + reject selectors that do not apply to that strategy. +- [x] Make the composition, component, filter, and deduplication request + objects strict without changing persisted response shapes. +- [x] Update OpenAPI, user/developer documentation, and Bruno examples. +- [x] Run the focused regression spec, then lint and the complete `npm test` + suite. +- [x] Review the final diff and propose a conventional commit message. + +Verification result (2026-07-29): + +- The focused virtual-composition contract spec passes (3), OpenAPI validation + passes (2), and backend lint passes. +- The first complete run encountered one roaming 404 in the new spec after 917 + API tests passed. The spec passed both in isolation (3) and alongside its + preceding snapshot-history spec (10). +- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 918, + middleware 24). +- Proposed commit: + + ```text + fix(release-tracks): validate virtual composition contracts + + Reject unknown composition properties and enforce strategy-specific + component selectors across virtual-track creation and updates. Align + OpenAPI, documentation, frontend guidance, and Bruno examples. + ``` + +### Tracker consolidation + +- [x] Consolidate the virtual-track completion backlog into this section. +- [x] Preserve completed implementation evidence in the dated records below. +- [x] Move the downstream Angular handoff to + `docs/developer/FRONTEND_TODO.md`. +- [x] Remove the superseded root-level tracker files. + +## Document downstream frontend work + +- [x] Inventory the current release-track API contract and recent endpoint, + terminology, lifecycle, validation, and response-shape changes. +- [x] Inspect the Angular release-track consumers so the handoff identifies + concrete downstream work instead of restating backend implementation notes. +- [x] Create `docs/developer/FRONTEND_TODO.md` with task-oriented guidance, + contextual explanations, and acceptance criteria. +- [x] Cross-check the handoff against OpenAPI, user/developer documentation, + Bruno, and the `internalattack` client. +- [x] Review formatting and the final diff. + +Verification result (2026-07-29): + +- The handoff was cross-checked against the current OpenAPI paths, release-track + documentation, Bruno requests, `internalattack` methods, and Angular + release-track consumers. +- `git diff --check` passes. +- Proposed commit: + + ```text + docs(release-tracks): track required frontend updates + + Document the route, request, response, lifecycle, and terminology changes + that the Angular release-track client must adopt. + ``` + +## Implement virtual quarantine resolution + +- [x] Add end-to-end regression coverage for exact-revision quarantine + promotion, snapshot immutability, back-reference reconciliation, validation, + and virtual-track type enforcement. +- [x] Add `POST /api/release-tracks/:id/virtual/quarantine/promote`. +- [x] Promote the selected revision to members in a new draft and remove all + quarantined alternatives for the same object. +- [x] Preserve the immutable composition-resolution record and historical + materialized snapshot. +- [x] Update OpenAPI, user/developer documentation, and Bruno. +- [x] Run focused regression specs, then lint and the complete `npm test` suite. +- [x] Review the final diff and propose a conventional commit message. + +Verification result (2026-07-29): + +- Focused quarantine, release, back-reference, and virtual-domain specs pass + (40); backend lint passes. +- The first complete run encountered six unrelated shared-suite failures in + collection bundles, data-component pagination, and user accounts. All three + specs passed in isolation (30, 13, and 14 tests respectively). +- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 915, + middleware 24). +- The `internalattack` focused release-track suite passes (30), its complete + suite passes (247), and changed-file Ruff and pre-commit checks pass. +- Proposed commit: + + ```text + feat(release-tracks): resolve virtual quarantine conflicts + + Add an explicitly virtual-scoped endpoint for selecting an exact + quarantined revision into a new draft. Preserve materialization provenance, + reconcile back-references, and update supported clients and documentation. + ``` + ## Harden virtual materialization lifecycle -- [x] Record the complete virtual-track audit in `VIRTUAL_TRACKS_TODO.md`. +- [x] Record the complete virtual-track audit in the dedicated virtual release + tracks section of this file. - [x] Add regression coverage for stale composition state, unmaterialized release attempts, and virtual use of standard contents endpoints. - [x] Clear inherited materialized state when virtual composition changes. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 41126779..e0c60180 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -316,10 +316,6 @@ Virtual release tracks compute their contents by aggregating objects from compon resolution_strategy: "latest_tagged", // "latest_tagged" | "specific_version" | "specific_snapshot" priority: 1, // Required for prioritize_higher_priority strategy (lower number = higher priority) - // Optional: version/snapshot specification for non-latest strategies - version: "5.0", // Used with "specific_version" strategy - snapshot: "2024-02-01T10:00:00Z", // Used with "specific_snapshot" strategy - // Optional: filters to limit which objects are included filters: { object_types: ["intrusion-set"], @@ -463,5 +459,11 @@ Virtual release tracks compute their contents by aggregating objects from compon - All snapshots start as **drafts** and must be explicitly tagged - Component tracks must exist and have at least one tagged release - Each component track must have a unique **priority** value (no duplicates) +- Composition request objects are strict; unknown composition, component, + filter, and deduplication keys return `400 Bad Request` +- Selector fields form a discriminated request contract: + - `latest_tagged` rejects `version` and `snapshot` + - `specific_version` requires `version` and rejects `snapshot` + - `specific_snapshot` requires `snapshot` and rejects `version` - Quarantine promotion selects an exact revision in a new draft and preserves the source snapshot's immutable `composition_resolution` diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 6665093e..be97403c 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -74,6 +74,14 @@ Virtual-only operations are deliberately scoped beneath selects one exact quarantined revision for members, and removes all quarantined alternatives for that object. +Composition input uses strict Zod objects at the composition, component, +filter, and deduplication levels. Components form a discriminated union on +`resolution_strategy`: `latest_tagged` accepts no selector, +`specific_version` requires only `version`, and `specific_snapshot` requires +only `snapshot`. This prevents misspelled filters or irrelevant selectors from +being silently stripped before persistence. The same schema is used for +initial virtual-track creation and composition updates. + There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a standard draft. Release planning never resolves composition and rejects a diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 07744434..15f3e531 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1116,6 +1116,12 @@ the domain is read from `external_references[].external_id`. `snapshot_schedule` is stored as metadata only; automated execution is not yet implemented. +Composition, component, filter, and deduplication objects are strict. Unknown +keys, including the incorrect singular `filters.domain`, return +`400 Bad Request`. Component selectors are also strategy-specific: +`latest_tagged` rejects `version` and `snapshot`; `specific_version` requires +only `version`; and `specific_snapshot` requires only `snapshot`. + ### Update Virtual Track Composition ``` @@ -1140,6 +1146,10 @@ PUT /api/release-tracks/:id/virtual/composition } ``` +The same strict composition and selector validation applies to this update +operation. Invalid fields are rejected rather than removed from the persisted +configuration. + **Note:** Updating composition creates a pending draft containing the new rules. To prevent stale materialization from being released, the draft has empty `members` and `quarantine` arrays and diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 47a2b679..e9c5020f 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -182,6 +182,15 @@ Resolves to a specific snapshot by its `modified` timestamp. **Use case:** "Lock to exact snapshot for reproducibility" +Component selectors are strict and strategy-specific: + +- `latest_tagged` rejects both `version` and `snapshot`. +- `specific_version` requires `version` and rejects `snapshot`. +- `specific_snapshot` requires `snapshot` and rejects `version`. + +Unknown component properties are rejected with `400 Bad Request`; they are +not silently discarded. + ### Component Track Sync Rules Virtual tracks **only sync from component tracks' `members` tier** (`x_mitre_contents`). This ensures that virtual tracks only aggregate objects that have been officially released in their source tracks. @@ -218,7 +227,8 @@ ATT&CK data identifies their domain through matrix fallback. `stix_pattern` is not part of the current request schema and is not -implemented. +implemented. Filter objects are strict, so misspelled or unsupported keys such +as `domain` fail with `400 Bad Request`; use the plural `domains`. ### Deduplication Strategies @@ -826,6 +836,12 @@ PUT /api/release-tracks/:id/virtual/composition } ``` +Composition requests are strict at every nested level. Unknown composition, +component, filter, or deduplication properties return `400 Bad Request`. +Selector fields must match `resolution_strategy`: `latest_tagged` accepts +neither selector, `specific_version` requires only `version`, and +`specific_snapshot` requires only `snapshot`. + **Note:** Updating composition creates a pending draft with the new rules and invalidates any previously materialized contents. The draft has empty `members` and `quarantine` arrays and `composition_resolution: null`. Run the From 290ba19aba64834c39feb9ca6a691fd8efd701bc Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:36:46 -0400 Subject: [PATCH 23/55] fix(release-tracks): validate virtual component identities Require unique component priorities and track IDs, validate referenced standard tracks before initial virtual-track persistence, and align request, persistence, OpenAPI, documentation, and frontend contracts. --- .../definitions/components/release-tracks.yml | 6 +- .../paths/release-tracks-paths.yml | 7 +- .../release-tracks/release-track-schemas.js | 28 +++++++- .../release-track-snapshot-schema.js | 10 ++- .../release-tracks/release-tracks-service.js | 6 +- .../release-tracks/virtual-track-service.js | 22 ++++++ .../virtual-composition-validation.spec.js | 69 ++++++++++++++++++- docs/developer/FRONTEND_TODO.md | 8 +++ docs/developer/TODO.md | 53 ++++++++++++-- docs/developer/release-tracks/entities.md | 6 +- .../release-tracks/implementation-notes.md | 7 ++ docs/user/release-tracks/api-reference.md | 11 ++- docs/user/release-tracks/release-workflow.md | 6 +- docs/user/release-tracks/terminology.md | 16 +++-- docs/user/release-tracks/virtual-tracks.md | 45 ++++++++---- 15 files changed, 259 insertions(+), 41 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 06dc855e..5d9e3984 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -324,14 +324,16 @@ components: required: - track_id - resolution_strategy + - priority properties: track_id: type: string description: 'The release track ID of the component' example: 'release-track--a1b2c3d4-e5f6-7890-abcd-ef1234567890' priority: - type: number - description: 'Priority for deduplication (higher wins)' + type: integer + minimum: 0 + description: 'Required unique component priority; lower numbers have higher priority' resolution_strategy: type: string enum: diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index b5e0ecf7..e639e914 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -225,7 +225,9 @@ paths: Create a new standard or virtual release track with an initial empty draft snapshot. Request body is validated via Zod (not OpenAPI). Virtual composition objects are strict, and component selectors must match their - resolution_strategy. + resolution_strategy. Component IDs and priorities must be unique, + every priority is required, and referenced components must already + exist as standard tracks. tags: - 'Release Tracks' # Request body validation moved to Zod in controller @@ -798,7 +800,8 @@ paths: Request body is strictly validated via Zod. Unknown composition, component, filter, and deduplication keys are rejected. latest_tagged rejects selector fields; specific_version requires version; - specific_snapshot requires snapshot. + specific_snapshot requires snapshot. Component IDs and required + non-negative integer priorities must each be unique. tags: - 'Release Tracks' parameters: diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index f3794027..ae833963 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -272,7 +272,7 @@ const componentTrackFiltersSchema = z const componentTrackBaseShape = { track_id: releaseTrackIdSchema, - priority: z.number().int().min(0).optional(), + priority: z.number().int().min(0), filters: componentTrackFiltersSchema.optional(), }; @@ -309,7 +309,31 @@ const compositionSchema = z .strict() .optional(), }) - .strict(); + .strict() + .superRefine((composition, context) => { + const trackIds = new Set(); + const priorities = new Set(); + + composition.component_tracks.forEach((component, index) => { + if (trackIds.has(component.track_id)) { + context.addIssue({ + code: 'custom', + path: ['component_tracks', index, 'track_id'], + message: 'Each component track must reference a unique track', + }); + } + trackIds.add(component.track_id); + + if (priorities.has(component.priority)) { + context.addIssue({ + code: 'custom', + path: ['component_tracks', index, 'priority'], + message: 'Each component track must have a unique priority value', + }); + } + priorities.add(component.priority); + }); + }); const createTrackBodySchema = z.object({ name: trackNameSchema, diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 2d2c18a7..4fd08ae2 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -102,7 +102,15 @@ const componentTrackDefinition = { enum: ['latest_tagged', 'specific_version', 'specific_snapshot'], required: true, }, - priority: { type: Number, required: true }, + priority: { + type: Number, + required: true, + min: 0, + validate: { + validator: Number.isInteger, + message: 'Component priority must be an integer', + }, + }, version: { type: String, validate: validateVersion, diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index a28c923d..b9a922a9 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -169,7 +169,11 @@ exports.getReleasesByObject = function getReleasesByObject(objectRef, options) { return releaseHistoryService.getReleasesByObject(objectRef, options); }; -exports.createTrack = function createTrack(data) { +exports.createTrack = async function createTrack(data) { + if (data.type === 'virtual' && data.composition) { + await virtualTrackService.validateComposition(data.composition); + } + return snapshotService.createTrack(data); }; diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 9762fe67..6c419422 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -63,6 +63,16 @@ async function validateComponentTracks(componentTracks) { }); } + const invalidPriority = componentTracks.find( + (component) => !Number.isInteger(component.priority) || component.priority < 0, + ); + if (invalidPriority) { + throw new BadRequestError({ + message: 'Invalid component priority', + details: 'Each component track must have a non-negative integer priority', + }); + } + // Check for duplicate track_ids const trackIds = componentTracks.map((c) => c.track_id); const uniqueTrackIds = new Set(trackIds); @@ -99,6 +109,18 @@ async function validateComponentTracks(componentTracks) { return registryMap; } +/** + * Validate component identities and types without resolving their snapshots. + * Used before initial virtual-track persistence as well as by virtual + * operations that replace or materialize composition. + * + * @param {Object} composition + * @returns {Promise>} + */ +exports.validateComposition = async function validateComposition(composition) { + return validateComponentTracks(composition.component_tracks); +}; + /** * Resolve a component track to a specific tagged snapshot based on its * resolution strategy. diff --git a/app/tests/api/release-tracks/virtual-composition-validation.spec.js b/app/tests/api/release-tracks/virtual-composition-validation.spec.js index 1e53176d..8613152a 100644 --- a/app/tests/api/release-tracks/virtual-composition-validation.spec.js +++ b/app/tests/api/release-tracks/virtual-composition-validation.spec.js @@ -12,6 +12,7 @@ describe('Virtual release-track composition validation API', function () { let app; let passportCookie; let componentTrack; + let secondComponentTrack; let virtualTrack; let createSequence = 0; @@ -29,6 +30,10 @@ describe('Virtual release-track composition validation API', function () { name: 'Composition Validation Component', type: 'standard', }); + secondComponentTrack = await post('/api/release-tracks/new', { + name: 'Composition Validation Second Component', + type: 'standard', + }); virtualTrack = await post('/api/release-tracks/new', { name: 'Composition Validation Virtual', type: 'virtual', @@ -72,12 +77,12 @@ describe('Virtual release-track composition validation API', function () { }; } - async function createVirtual(compositionBody, status = 201) { + async function createVirtual(compositionBody, status = 201, name) { createSequence += 1; return post( '/api/release-tracks/new', { - name: `Strict Composition Create ${createSequence}`, + name: name || `Strict Composition Create ${createSequence}`, type: 'virtual', composition: compositionBody, }, @@ -85,6 +90,16 @@ describe('Virtual release-track composition validation API', function () { ); } + async function listTracks(search) { + const response = await request(app) + .get('/api/release-tracks') + .query({ search }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return response.body.data; + } + it('rejects unknown composition keys instead of silently stripping them', async function () { const invalidCompositions = [ composition(component('latest_tagged'), { unexpected: true }), @@ -143,4 +158,54 @@ describe('Virtual release-track composition validation API', function () { expect(updated.composition.component_tracks[0]).toMatchObject(validComponent); } }); + + it('requires unique component priorities and track IDs', async function () { + const invalidCompositions = [ + composition(component('latest_tagged', { priority: undefined })), + { + component_tracks: [ + component('latest_tagged'), + { + track_id: secondComponentTrack.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + }, + { + component_tracks: [component('latest_tagged'), component('latest_tagged', { priority: 2 })], + }, + ]; + + for (const invalidComposition of invalidCompositions) { + await createVirtual(invalidComposition, 400); + await putComposition(invalidComposition, 400); + } + }); + + it('validates initial component existence and standard-track type before persistence', async function () { + const missingComponentName = 'Missing Component Create'; + await createVirtual( + composition({ + track_id: 'release-track--11111111-1111-4111-8111-111111111111', + resolution_strategy: 'latest_tagged', + priority: 1, + }), + 404, + missingComponentName, + ); + expect(await listTracks(missingComponentName)).toEqual([]); + + const virtualComponentName = 'Virtual Component Create'; + await createVirtual( + composition({ + track_id: virtualTrack.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }), + 400, + virtualComponentName, + ); + expect(await listTracks(virtualComponentName)).toEqual([]); + }); }); diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 376dbb29..77476eb2 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -469,6 +469,12 @@ strategy: - `latest_tagged` sends neither `version` nor `snapshot`. - `specific_version` sends `version` and omits `snapshot`. - `specific_snapshot` sends `snapshot` and omits `version`. +- Every component sends a unique, non-negative integer `priority`; lower + numbers have higher priority. + +The server validates component identity during both creation and update. +Referenced tracks must already exist and must be standard tracks, and duplicate +component track IDs are rejected. Done when: @@ -476,6 +482,8 @@ Done when: - Saved and reloaded composition preserves `filters.domains`. - Tests assert the plural key and multi-domain payload shape. - Changing resolution strategy clears the selector from the previous strategy. +- Every component row requires a priority, and duplicate priorities or track + selections are blocked before submission. - Submitted composition payloads contain only server-supported properties. ## P1 — Separate snapshot and preview output-format types diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 7e526ed6..ba4aa54b 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -19,9 +19,9 @@ completion backlog. - `specific_version` requires `version` and rejects `snapshot`; - `specific_snapshot` requires `snapshot` and rejects `version`; - `latest_tagged` rejects both selector fields. -- [ ] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, +- [x] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, and examples; reject duplicate priorities at the request boundary. -- [ ] Validate component existence, standard-track type, duplicate track IDs, +- [x] Validate component existence, standard-track type, duplicate track IDs, and duplicate priorities when a virtual track is initially created, not only when composition is later updated or materialized. - [ ] Validate `snapshot_schedule` by mode: @@ -84,7 +84,7 @@ completion backlog. - [ ] Align `composition_resolution` examples with fields actually generated, or implement the documented `by_type`, `by_tier`, and native statistics. - [ ] Align documented error envelopes with centralized error-handler output. -- [ ] Include required `priority` values in every composition example. +- [x] Include required `priority` values in every composition example. - [ ] Clearly distinguish configured composition from a materialized draft and describe scheduled behavior as unavailable until scheduler execution exists. @@ -138,6 +138,49 @@ Verification result (2026-07-29): OpenAPI, documentation, frontend guidance, and Bruno examples. ``` +### Current implementation slice — Component identity and priority validation + +- [x] Add creation and composition-update regression coverage for required + priorities, duplicate priorities, and duplicate component track IDs. +- [x] Reject missing component tracks and virtual component tracks before an + initial virtual track is persisted. +- [x] Make component priority required and non-negative across Zod, Mongoose, + OpenAPI, user/developer documentation, and Bruno examples. +- [x] Keep service-layer component validation as a defense for non-HTTP + callers while moving deterministic duplicates to request validation. +- [x] Run the focused regression specs, then lint and the complete `npm test` + suite. +- [x] Review the final diff and propose a conventional commit message. + +Verification result (2026-07-29): + +- The focused release-track regression group passes (22), the isolated + backrefs spec passes (23), OpenAPI validation passes (2), and backend lint + passes. +- The first complete run encountered one unrelated shared-suite failure in the + backrefs manual-sync case after 919 API tests passed. The affected spec + passed in isolation (23). +- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 920, + middleware 24). +- Proposed commit: + + ```text + fix(release-tracks): validate virtual component identities + + Require unique component priorities and track IDs, validate referenced + standard tracks before initial virtual-track persistence, and align request, + persistence, OpenAPI, documentation, and frontend contracts. + ``` + +- Proposed companion Bruno commit: + + ```text + docs(release-tracks): document component priority constraints + + Document required unique priorities and standard component references for + virtual-track creation and composition updates. + ``` + ### Tracker consolidation - [x] Consolidate the virtual-track completion backlog into this section. @@ -801,7 +844,9 @@ the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. ## Small Fixes -- [ ] **Composition schema mismatch: `priority`.** `PUT /api/release-tracks/:id/virtual/composition` — the Zod schema (`componentTrackSchema`) marks `priority` optional, but the mongoose snapshot schema requires it, so omitting it passes validation and then fails the save with a 500 (`DatabaseError`) instead of a 400. Align the schemas (either default `priority` or make it required in Zod). Found 2026-07-15 while testing virtual-track backrefs. +- [x] **Composition schema mismatch: `priority`.** Resolved 2026-07-29 by + requiring a unique, non-negative integer priority in request validation, + persistence, OpenAPI, documentation, and Bruno examples. - [ ] **`deleteSnapshot` lacks a tagged-release guard.** `DELETE /api/release-tracks/:id/snapshots/:modified` (`snapshot-service.deleteSnapshot`) deletes any snapshot, including tagged releases — contradicting the "immutable once set" versioning rule. Should 409 on `version != null` (a squash implementation must also filter `version: null`; see Snapshot Retention section). Found 2026-07-15 while designing squash. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index e0c60180..7d22badb 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -314,7 +314,7 @@ Virtual release tracks compute their contents by aggregating objects from compon { track_id: "release-track--groups-monthly", resolution_strategy: "latest_tagged", // "latest_tagged" | "specific_version" | "specific_snapshot" - priority: 1, // Required for prioritize_higher_priority strategy (lower number = higher priority) + priority: 1, // Always required and unique (lower number = higher priority) // Optional: filters to limit which objects are included filters: { @@ -459,6 +459,10 @@ Virtual release tracks compute their contents by aggregating objects from compon - All snapshots start as **drafts** and must be explicitly tagged - Component tracks must exist and have at least one tagged release - Each component track must have a unique **priority** value (no duplicates) +- Priority is a required non-negative integer for every component, regardless + of deduplication strategy +- Component IDs and priorities are validated before initial virtual-track + persistence as well as during composition updates and materialization - Composition request objects are strict; unknown composition, component, filter, and deduplication keys return `400 Bad Request` - Selector fields form a discriminated request contract: diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index be97403c..142b4025 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -82,6 +82,13 @@ only `snapshot`. This prevents misspelled filters or irrelevant selectors from being silently stripped before persistence. The same schema is used for initial virtual-track creation and composition updates. +Component `priority` is always required, even when the selected deduplication +strategy does not inspect it. Zod rejects duplicate component IDs and +priorities before service delegation. The facade also asks the virtual-track +service to verify that every component exists and is a standard track before +persisting an initial virtual track; update and materialization retain the same +service-layer validation. + There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a standard draft. Release planning never resolves composition and rejects a diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 15f3e531..6e0a2f86 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1121,6 +1121,9 @@ keys, including the incorrect singular `filters.domain`, return `400 Bad Request`. Component selectors are also strategy-specific: `latest_tagged` rejects `version` and `snapshot`; `specific_version` requires only `version`; and `specific_snapshot` requires only `snapshot`. +Every component requires a unique, non-negative integer `priority`; lower +numbers have higher priority. When composition is supplied during creation, +each referenced track must already exist and must be a standard track. ### Update Virtual Track Composition @@ -1135,12 +1138,14 @@ PUT /api/release-tracks/:id/virtual/composition "component_tracks": [ { "track_id": "GroupsMonthly--uuid", - "resolution_strategy": "latest_tagged" + "resolution_strategy": "latest_tagged", + "priority": 0 }, { "track_id": "TechniquesQuarterly--uuid", "resolution_strategy": "specific_version", - "version": "2.0" + "version": "2.0", + "priority": 1 } ] } @@ -1148,7 +1153,7 @@ PUT /api/release-tracks/:id/virtual/composition The same strict composition and selector validation applies to this update operation. Invalid fields are rejected rather than removed from the persisted -configuration. +configuration. Component track IDs and priorities must each be unique. **Note:** Updating composition creates a pending draft containing the new rules. To prevent stale materialization from being released, the draft has diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 6ddcf015..54c335c3 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -1034,11 +1034,13 @@ POST /api/release-tracks/new "component_tracks": [ { "track_id": "GroupsMonthly--uuid", - "resolution_strategy": "latest_tagged" + "resolution_strategy": "latest_tagged", + "priority": 0 }, { "track_id": "TechniquesQuarterly--uuid", - "resolution_strategy": "latest_tagged" + "resolution_strategy": "latest_tagged", + "priority": 1 } ] }, diff --git a/docs/user/release-tracks/terminology.md b/docs/user/release-tracks/terminology.md index eb4bb1d3..7fe37a6d 100644 --- a/docs/user/release-tracks/terminology.md +++ b/docs/user/release-tracks/terminology.md @@ -309,17 +309,18 @@ A **virtual release track** is a special type of release track that computes its ### Component Track -A **component track** is a release track (standard or virtual) that is referenced by a virtual release track. +A **component track** is a standard release track that is referenced by a virtual release track. **Technical Definition:** - A component track is specified in a virtual track's `composition.component_tracks` array - Each component defines a `resolution_strategy` (how to select which snapshot to use) +- Each component defines a unique, non-negative integer `priority` - Each component can optionally specify `filters` (which objects to include) **Characteristics:** - Component tracks are independent - they don't know they're being referenced - Virtual tracks "pull" content from components via composition rules -- Components can be standard tracks (manage objects) or virtual tracks (aggregate) +- Components must be standard tracks; virtual-track nesting is rejected - Components must have at least one tagged snapshot for virtual track to resolve **Examples:** @@ -349,11 +350,12 @@ A **component track** is a release track (standard or virtual) that is reference { track_id: "GroupsMonthly--uuid", resolution_strategy: "latest_tagged", + priority: 0, filters: { object_types: ["intrusion-set"] } } ], deduplication: { - strategy: "prefer_latest_modified" + strategy: "prioritize_latest_object" } } ``` @@ -402,9 +404,9 @@ A **resolution strategy** determines which snapshot from a component track to us 3. **specific_snapshot** - Use a specific snapshot by timestamp **Examples:** -- `{ resolution_strategy: "latest_tagged" }` → Always gets latest -- `{ resolution_strategy: "specific_version", version: "5.0" }` → Always uses v5.0 -- `{ resolution_strategy: "specific_snapshot", snapshot: "2024-02-01T10:00:00Z" }` → Always uses that exact snapshot +- `{ resolution_strategy: "latest_tagged", priority: 0 }` → Always gets latest +- `{ resolution_strategy: "specific_version", version: "5.0", priority: 0 }` → Always uses v5.0 +- `{ resolution_strategy: "specific_snapshot", snapshot: "2024-02-01T10:00:00Z", priority: 0 }` → Always uses that exact snapshot --- @@ -459,4 +461,4 @@ A **resolution strategy** determines which snapshot from a component track to us **Object Management:** - "The current snapshot contains 3,000 member objects and 150 staged objects" - "Move these candidate objects to staged" -- "Export the member objects as a STIX bundle" \ No newline at end of file +- "Export the member objects as a STIX bundle" diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index e9c5020f..947e2ada 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -85,7 +85,7 @@ Virtual tracks are identified by `stix.type = "virtual"` in their schema. { track_id: "release-track--uuid-1", resolution_strategy: "latest_tagged", - priority: 1, // Used with prioritize_higher_priority strategy (lower number = higher priority) + priority: 1, // Required and unique; lower number = higher priority filters: { object_types: ["intrusion-set"], // Additional filters... @@ -134,7 +134,8 @@ Always resolves to the most recent **tagged snapshot** from the component track. ```javascript { track_id: "release-track--uuid-1", - resolution_strategy: "latest_tagged" + resolution_strategy: "latest_tagged", + priority: 0 } // At virtual snapshot time (e.g., March 1, 2024): @@ -154,7 +155,8 @@ Resolves to a specific semantic version from the component track. { track_id: "release-track--uuid-1", resolution_strategy: "specific_version", - version: "5.0" + version: "5.0", + priority: 0 } // At virtual snapshot time: @@ -172,7 +174,8 @@ Resolves to a specific snapshot by its `modified` timestamp. { track_id: "release-track--uuid-1", resolution_strategy: "specific_snapshot", - snapshot: "2024-02-01T10:00:00Z" + snapshot: "2024-02-01T10:00:00Z", + priority: 0 } // At virtual snapshot time: @@ -289,7 +292,7 @@ deduplication: { #### 3. `prioritize_higher_priority` -Keep the version from the component track with the higher priority (lower priority number). Each component track must have a unique priority value. +Keep the version from the component track with the higher priority (lower priority number). Every component track requires a unique, non-negative integer priority. ```javascript composition: { @@ -794,15 +797,14 @@ POST /api/release-tracks/new { "track_id": "release-track--uuid-1", "resolution_strategy": "latest_tagged", + "priority": 0, "filters": { "object_types": ["intrusion-set"] } } ], "deduplication": { - "strategy": "prefer_latest_modified", - "tier_resolution": "highest_tier", - "status_resolution": "highest_status" + "strategy": "prioritize_latest_object" } }, @@ -825,12 +827,14 @@ PUT /api/release-tracks/:id/virtual/composition "component_tracks": [ { "track_id": "release-track--uuid-1", - "resolution_strategy": "latest_tagged" + "resolution_strategy": "latest_tagged", + "priority": 0 }, { "track_id": "release-track--uuid-2", "resolution_strategy": "specific_version", - "version": "2.0" + "version": "2.0", + "priority": 1 } ] } @@ -841,6 +845,9 @@ component, filter, or deduplication properties return `400 Bad Request`. Selector fields must match `resolution_strategy`: `latest_tagged` accepts neither selector, `specific_version` requires only `version`, and `specific_snapshot` requires only `snapshot`. +Every component also requires a unique, non-negative integer `priority`. +Referenced tracks must exist and must be standard tracks; these rules are +checked during initial virtual-track creation as well as composition updates. **Note:** Updating composition creates a pending draft with the new rules and invalidates any previously materialized contents. The draft has empty @@ -977,8 +984,16 @@ Virtual tracks can optionally have **native objects** in addition to composed co // Composed from standard tracks composition: { component_tracks: [ - { track_id: "release-track--uuid-1", priority: 1 }, - { track_id: "release-track--uuid-2", priority: 2 } + { + track_id: "release-track--uuid-1", + resolution_strategy: "latest_tagged", + priority: 1 + }, + { + track_id: "release-track--uuid-2", + resolution_strategy: "latest_tagged", + priority: 2 + } ], deduplication: { strategy: "prioritize_latest_object" @@ -1045,11 +1060,13 @@ POST /api/release-tracks/new "component_tracks": [ { "track_id": "release-track--uuid-1", - "resolution_strategy": "latest_tagged" + "resolution_strategy": "latest_tagged", + "priority": 0 }, { "track_id": "release-track--uuid-2", - "resolution_strategy": "latest_tagged" + "resolution_strategy": "latest_tagged", + "priority": 1 } ] }, From 029e33a00a4ac7a2d55f9ed97e4f0b7a0894858f Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:56:32 -0400 Subject: [PATCH 24/55] fix(release-tracks): validate virtual snapshot schedules Enforce strict mode-specific virtual snapshot schedules across request, service, persistence, OpenAPI, documentation, and frontend contracts. Reject schedule metadata for standard tracks and record required cron and explicit-date scheduler work. --- .../definitions/components/release-tracks.yml | 62 +++++-- .../paths/release-tracks-paths.yml | 4 +- .../release-tracks/release-track-schemas.js | 50 +++-- .../release-track-validators.js | 19 ++ .../release-track-registry-model.js | 16 +- .../release-tracks/release-tracks-service.js | 28 ++- ...rtual-snapshot-schedule-validation.spec.js | 172 ++++++++++++++++++ docs/developer/FRONTEND_TODO.md | 38 ++++ docs/developer/TODO.md | 73 +++++++- docs/developer/release-tracks/entities.md | 40 +++- .../release-tracks/implementation-notes.md | 8 + docs/user/release-tracks/api-reference.md | 9 +- docs/user/release-tracks/virtual-tracks.md | 13 +- 13 files changed, 478 insertions(+), 54 deletions(-) create mode 100644 app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 5d9e3984..66c52f86 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -546,24 +546,46 @@ components: format: date-time snapshot-schedule: - type: object - description: 'Schedule for automated virtual track snapshot creation' - properties: - mode: - type: string - enum: - - interval + description: 'Stored schedule metadata for virtual track snapshot creation; automated execution is not yet implemented' + oneOf: + - type: object + additionalProperties: false + required: + - mode + properties: + mode: + type: string + enum: + - manual + description: 'Snapshots are created explicitly' + - type: object + additionalProperties: false + required: + - mode + - cron + properties: + mode: + type: string + enum: + - cron + cron: + type: string + description: 'Five-field UTC cron expression' + example: '0 0 1 1,7 *' + - type: object + additionalProperties: false + required: + - mode - dates - - disabled - description: 'Scheduling mode' - interval_days: - type: number - nullable: true - description: 'Days between snapshots (when mode is interval)' - dates: - type: array - nullable: true - items: - type: string - format: date-time - description: 'Specific dates for snapshots (when mode is dates)' + properties: + mode: + type: string + enum: + - dates + dates: + type: array + minItems: 1 + items: + type: string + format: date-time + description: 'Explicit UTC dates for snapshot creation' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index e639e914..9571c82c 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -227,7 +227,9 @@ paths: objects are strict, and component selectors must match their resolution_strategy. Component IDs and priorities must be unique, every priority is required, and referenced components must already - exist as standard tracks. + exist as standard tracks. Virtual snapshot schedules are strict by + mode: manual accepts no selector, cron requires cron, and dates + requires at least one date. Standard tracks reject snapshot_schedule. tags: - 'Release Tracks' # Request body validation moved to Zod in controller diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index ae833963..e1beeb0f 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -257,11 +257,25 @@ const memberSyncConfigSchema = z.object({ // ============================================================================= /** POST /release-tracks/new */ -const snapshotScheduleSchema = z.object({ - mode: z.enum(['manual', 'cron', 'dates']), - cron: cronSchema.optional(), - dates: z.array(z.iso.datetime()).optional(), -}); +const snapshotScheduleSchema = z.discriminatedUnion('mode', [ + z + .object({ + mode: z.literal('manual'), + }) + .strict(), + z + .object({ + mode: z.literal('cron'), + cron: cronSchema, + }) + .strict(), + z + .object({ + mode: z.literal('dates'), + dates: z.array(z.iso.datetime()).min(1), + }) + .strict(), +]); const componentTrackFiltersSchema = z .object({ @@ -335,14 +349,24 @@ const compositionSchema = z }); }); -const createTrackBodySchema = z.object({ - name: trackNameSchema, - description: z.string().optional(), - type: trackTypeQuerySchema.default('standard'), - object_marking_refs: z.array(stixIdentifierSchema).optional(), - composition: compositionSchema.optional(), - snapshot_schedule: snapshotScheduleSchema.optional(), -}); +const createTrackBodySchema = z + .object({ + name: trackNameSchema, + description: z.string().optional(), + type: trackTypeQuerySchema.default('standard'), + object_marking_refs: z.array(stixIdentifierSchema).optional(), + composition: compositionSchema.optional(), + snapshot_schedule: snapshotScheduleSchema.optional(), + }) + .superRefine((track, context) => { + if (track.type !== 'virtual' && track.snapshot_schedule !== undefined) { + context.addIssue({ + code: 'custom', + path: ['snapshot_schedule'], + message: 'Snapshot schedules are only available for virtual tracks', + }); + } + }); /** POST /release-tracks/new-from-bundle */ const createFromBundleBodySchema = z.object({ diff --git a/app/lib/release-tracks/release-track-validators.js b/app/lib/release-tracks/release-track-validators.js index 3890a22d..4cae4594 100644 --- a/app/lib/release-tracks/release-track-validators.js +++ b/app/lib/release-tracks/release-track-validators.js @@ -14,6 +14,7 @@ const { releaseTrackIdSchema, trackNameSchema, cronSchema, + snapshotScheduleSchema, stixIdentifierSchema, xMitreVersionSchema, createStixIdValidator, @@ -64,6 +65,23 @@ const validateCron = { message: (props) => `"${props.value}" is not a valid cron expression (expected 5 fields)`, }; +const validateSnapshotSchedule = { + validator: (value) => { + if (value === undefined || value === null) return true; + + const schedule = typeof value.toObject === 'function' ? value.toObject() : value; + const normalized = { + ...schedule, + dates: schedule.dates?.map((date) => (date instanceof Date ? date.toISOString() : date)), + }; + if (normalized.dates === undefined) delete normalized.dates; + + return snapshotScheduleSchema.safeParse(normalized).success; + }, + message: + 'Snapshot schedule fields must match mode: manual has no selector, cron requires cron, and dates requires at least one date', +}; + // ============================================================================= // Exports // ============================================================================= @@ -76,4 +94,5 @@ module.exports = { validateMarkingDefRefs, validateVersion, validateCron, + validateSnapshotSchedule, }; diff --git a/app/models/release-tracks/release-track-registry-model.js b/app/models/release-tracks/release-track-registry-model.js index 4c0c5e2c..833ad6ce 100644 --- a/app/models/release-tracks/release-track-registry-model.js +++ b/app/models/release-tracks/release-track-registry-model.js @@ -6,6 +6,7 @@ const { validateTrackName, validateVersion, validateCron, + validateSnapshotSchedule, } = require('../../lib/release-tracks/release-track-validators'); // --- Sub-schemas --- @@ -69,7 +70,20 @@ const releaseTrackRegistryDefinition = { tagged_releases: { type: [taggedReleaseSchema], default: [] }, // Virtual tracks only - snapshot_schedule: { type: snapshotScheduleSchema, default: undefined }, + snapshot_schedule: { + type: snapshotScheduleSchema, + default: undefined, + validate: { + validator: function validateRegistrySnapshotSchedule(value) { + return ( + value === undefined || + (this.type === 'virtual' && validateSnapshotSchedule.validator(value)) + ); + }, + message: + 'Snapshot schedule is only valid for virtual tracks and its fields must match its mode', + }, + }, created_at: { type: Date, required: true }, updated_at: { type: Date, required: true }, diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index b9a922a9..879b9edd 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -14,7 +14,8 @@ // Phase 6: Export, ephemeral, bundle import → export-service, ephemeral-service, bundle-import-service // ============================================================================= -const { NotImplementedError } = require('../../exceptions'); +const { BadRequestError, NotImplementedError } = require('../../exceptions'); +const { snapshotScheduleSchema } = require('../../lib/release-tracks/release-track-schemas'); const snapshotService = require('./snapshot-service'); const standardTrackService = require('./standard-track-service'); const versioningService = require('./versioning-service'); @@ -170,11 +171,30 @@ exports.getReleasesByObject = function getReleasesByObject(objectRef, options) { }; exports.createTrack = async function createTrack(data) { - if (data.type === 'virtual' && data.composition) { - await virtualTrackService.validateComposition(data.composition); + let validatedData = data; + + if (data.snapshot_schedule !== undefined) { + if (data.type !== 'virtual') { + throw new BadRequestError({ + message: 'Snapshot schedules are only available for virtual release tracks', + }); + } + + const scheduleResult = snapshotScheduleSchema.safeParse(data.snapshot_schedule); + if (!scheduleResult.success) { + throw new BadRequestError({ + message: 'Invalid snapshot schedule', + details: scheduleResult.error.errors, + }); + } + validatedData = { ...data, snapshot_schedule: scheduleResult.data }; + } + + if (validatedData.type === 'virtual' && validatedData.composition) { + await virtualTrackService.validateComposition(validatedData.composition); } - return snapshotService.createTrack(data); + return snapshotService.createTrack(validatedData); }; // Phase 6 → bundle-import-service diff --git a/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js b/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js new file mode 100644 index 00000000..4f6b532e --- /dev/null +++ b/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js @@ -0,0 +1,172 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); +const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); + +describe('Virtual release-track snapshot schedule validation API', function () { + let app; + let passportCookie; + let createSequence = 0; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function createTrack(snapshotSchedule, status = 201, type = 'virtual') { + createSequence += 1; + const name = `Schedule Validation ${createSequence}`; + const response = await request(app) + .post('/api/release-tracks/new') + .send({ + name, + type, + snapshot_schedule: snapshotSchedule, + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return { name, body: response.body }; + } + + async function getRegistryTrack(search) { + const response = await request(app) + .get('/api/release-tracks') + .query({ search }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return response.body.data[0]; + } + + it('accepts and persists the fields defined by each schedule mode', async function () { + const schedules = [ + { mode: 'manual' }, + { mode: 'cron', cron: '0 0 1 1,7 *' }, + { + mode: 'dates', + dates: ['2027-01-15T00:00:00.000Z', '2027-07-15T00:00:00.000Z'], + }, + ]; + + for (const schedule of schedules) { + const created = await createTrack(schedule); + const registryTrack = await getRegistryTrack(created.name); + expect(registryTrack.snapshot_schedule).toEqual(schedule); + } + }); + + it('rejects fields that do not apply to manual schedules', async function () { + const invalidSchedules = [ + { mode: 'manual', cron: '0 0 1 1,7 *' }, + { mode: 'manual', dates: ['2027-01-15T00:00:00.000Z'] }, + { mode: 'manual', unexpected: true }, + ]; + + for (const schedule of invalidSchedules) { + await createTrack(schedule, 400); + } + }); + + it('requires cron and rejects dates for cron schedules', async function () { + const invalidSchedules = [ + { mode: 'cron' }, + { + mode: 'cron', + cron: '0 0 1 1,7 *', + dates: ['2027-01-15T00:00:00.000Z'], + }, + { mode: 'cron', cron: '0 0 1 1,7 * 2027' }, + ]; + + for (const schedule of invalidSchedules) { + await createTrack(schedule, 400); + } + }); + + it('requires non-empty dates and rejects cron for dates schedules', async function () { + const invalidSchedules = [ + { mode: 'dates' }, + { mode: 'dates', dates: [] }, + { + mode: 'dates', + dates: ['2027-01-15T00:00:00.000Z'], + cron: '0 0 1 1,7 *', + }, + ]; + + for (const schedule of invalidSchedules) { + await createTrack(schedule, 400); + } + }); + + it('rejects snapshot schedules on standard tracks', async function () { + await createTrack({ mode: 'manual' }, 400, 'standard'); + }); + + it('repeats schedule validation for non-HTTP service callers', async function () { + const registryCountBefore = await ReleaseTrackRegistry.countDocuments(); + + const invalidTracks = [ + { + name: 'Invalid Service Schedule', + type: 'virtual', + snapshot_schedule: { mode: 'cron' }, + }, + { + name: 'Invalid Standard Service Schedule', + type: 'standard', + snapshot_schedule: { mode: 'manual' }, + }, + ]; + + for (const track of invalidTracks) { + await expect(releaseTracksService.createTrack(track)).rejects.toThrow(); + } + + expect(await ReleaseTrackRegistry.countDocuments()).toBe(registryCountBefore); + }); + + it('repeats mode validation at the persistence boundary', async function () { + const invalidRegistries = [ + { + type: 'virtual', + name: 'Invalid Persistence Schedule', + snapshot_schedule: { + mode: 'manual', + cron: '0 0 1 1,7 *', + }, + }, + { + type: 'standard', + name: 'Invalid Standard Persistence Schedule', + snapshot_schedule: { + mode: 'manual', + }, + }, + ]; + + for (const invalidRegistry of invalidRegistries) { + const registry = new ReleaseTrackRegistry({ + track_id: 'release-track--11111111-1111-4111-8111-111111111111', + created_at: new Date(), + updated_at: new Date(), + ...invalidRegistry, + }); + await expect(registry.validate()).rejects.toThrow(); + } + }); +}); diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 77476eb2..5363e97b 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -486,6 +486,44 @@ Done when: selections are blocked before submission. - Submitted composition payloads contain only server-supported properties. +## P1 — Submit mode-correct virtual snapshot schedules + +### [ ] Add conditional validation and complete the dates-mode UI + +`snapshot_schedule` is virtual-only and now has a strict discriminated +contract: + +```ts +type SnapshotSchedule = + | { mode: 'manual' } + | { mode: 'cron'; cron: string } + | { mode: 'dates'; dates: string[] }; +``` + +The modes are mutually exclusive. Do not retain hidden form values when the +mode changes: `manual` sends neither selector, `cron` sends only a valid +five-field cron expression, and `dates` sends only a nonempty array of ISO +timestamps. Standard-track payloads must omit `snapshot_schedule`. + +The current dialog already lists `dates`, but it has no date controls and +therefore submits only `{ mode: 'dates' }`, which the server rejects. The cron +control is also not conditionally required, allowing `{ mode: 'cron' }` to be +submitted. + +Schedule configuration is metadata only for now. The UI must not imply that +automatic creation is active until the P2 backend scheduler integration is +implemented. + +Done when: + +- Selecting cron makes a valid cron expression required and clears dates. +- Selecting dates exposes date controls, requires at least one value, emits + ISO timestamps, and clears cron. +- Selecting manual clears both selector fields. +- Standard-track creation never sends schedule metadata. +- Tests cover all three modes and mode switching. +- User-facing copy says scheduled execution is not yet active. + ## P1 — Separate snapshot and preview output-format types ### [ ] Remove the invalid `snapshot` format and model `summary` correctly diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index ba4aa54b..3e727a4c 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -24,7 +24,7 @@ completion backlog. - [x] Validate component existence, standard-track type, duplicate track IDs, and duplicate priorities when a virtual track is initially created, not only when composition is later updated or materialized. -- [ ] Validate `snapshot_schedule` by mode: +- [x] Validate `snapshot_schedule` by mode: - `manual` rejects `cron` and `dates`; - `cron` requires `cron` and rejects `dates`; - `dates` requires at least one date and rejects `cron`. @@ -53,11 +53,23 @@ completion backlog. ### P2 — Scheduled materialization - [ ] Connect virtual `snapshot_schedule` metadata to the existing task - scheduler. -- [ ] Implement manual, cron, and explicit-date scheduling semantics. + scheduler. This is required for virtual-track completion, not an optional + future enhancement. +- [ ] Implement `cron` execution so each matching schedule occurrence + materializes a new virtual draft through the same lifecycle and validation + used by `POST /api/release-tracks/:id/virtual/snapshots/create`. +- [ ] Implement `dates` execution so every configured timestamp materializes + exactly one virtual draft, including deterministic handling for restart + recovery, missed timestamps, and duplicate-delivery prevention. +- [ ] Preserve `manual` semantics: store no executable schedule and create + drafts only through the explicit virtual snapshot-creation endpoint. - [ ] Define failure behavior when a component has no matching tagged snapshot, including automation-run audit records and retry policy. -- [ ] Add scheduler integration tests and operational documentation. +- [ ] Add scheduler integration tests for both `cron` and `dates`, including + successful execution, restart recovery, idempotency, component-resolution + failure, and retry behavior. +- [ ] Add operational documentation covering scheduler activation, UTC + interpretation, observability, failures, and retries. ### P2 — Contract decisions @@ -181,6 +193,59 @@ Verification result (2026-07-29): virtual-track creation and composition updates. ``` +### Current implementation slice — Snapshot schedule contracts + +- [x] Add creation regressions for valid and invalid `manual`, `cron`, and + `dates` schedule payloads. +- [x] Enforce a strict mode-discriminated request contract: + - `manual` accepts only `mode`; + - `cron` requires `cron` and rejects `dates`; + - `dates` requires at least one date and rejects `cron`. +- [x] Reject `snapshot_schedule` on standard-track creation instead of silently + dropping it. +- [x] Repeat schedule invariants at the service and Mongoose boundaries for + non-HTTP callers. +- [x] Align OpenAPI, user/developer documentation, frontend guidance, the + `internalattack` test fixture, and Bruno. +- [x] Run focused regression specs, lint, and the complete `npm test` suite. +- [x] Review the final diff and propose conventional commit messages. + +Verification result (2026-07-29): + +- The focused schedule-contract spec passes (7), the focused virtual-track + regression group passes (29), OpenAPI validation passes (2), and backend + lint passes. +- The required complete `npm test` suite passes (OpenAPI 2, config 21, API + 927, middleware 24). +- The `internalattack` focused release-track suite passes (30), and its + complete suite passes (247). +- Proposed REST API commit: + + ```text + fix(release-tracks): validate virtual snapshot schedules + + Enforce strict mode-specific virtual snapshot schedules across request, + service, persistence, OpenAPI, documentation, and frontend contracts. + Reject schedule metadata for standard tracks. + ``` + +- Proposed companion Bruno commit: + + ```text + docs(release-tracks): document snapshot schedule modes + + Document the strict manual, cron, and dates schedule payloads and clarify + that automated execution is not yet implemented. + ``` + +- Proposed companion `internalattack` commit: + + ```text + test(release-tracks): align virtual composition fixture + + Include the required component priority in virtual-track creation coverage. + ``` + ### Tracker consolidation - [x] Consolidate the virtual-track completion backlog into this section. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 7d22badb..c975ee46 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -408,14 +408,11 @@ Virtual release tracks compute their contents by aggregating objects from compon } }, - // Optional: Virtual tracks can schedule automatic snapshot creation + // Optional schedule metadata. Choose exactly one mode-specific shape. + // This example uses cron; automated execution is a required P2 capability. snapshot_schedule: { - mode: "manual", // "manual" | "cron" | "dates" - cron: "0 0 1 1,7 *", // Cron expression (e.g., Jan 1 and July 1 at midnight) - dates: [ // Or specific dates - "2024-01-01T00:00:00Z", - "2024-07-01T00:00:00Z" - ] + mode: "cron", + cron: "0 0 1 1,7 *" // Jan 1 and July 1 at midnight UTC }, // Configuration @@ -439,6 +436,29 @@ Virtual release tracks compute their contents by aggregating objects from compon } ``` +The three valid `snapshot_schedule` shapes are: + +```javascript +// Explicit creation only +{ mode: "manual" } + +// Five-field UTC cron schedule +{ mode: "cron", cron: "0 0 1 1,7 *" } + +// Explicit execution dates +{ + mode: "dates", + dates: [ + "2024-01-01T00:00:00Z", + "2024-07-01T00:00:00Z" + ] +} +``` + +These are alternatives, not fields to combine in one schedule. The API +currently validates and persists all three shapes. Automated execution for +`cron` and `dates` is not implemented yet and is a required P2 deliverable. + **Key Differences from Standard Tracks:** 1. **Type Identification**: `stix.type = "virtual"` @@ -463,6 +483,12 @@ Virtual release tracks compute their contents by aggregating objects from compon of deduplication strategy - Component IDs and priorities are validated before initial virtual-track persistence as well as during composition updates and materialization +- Snapshot schedules are strict and mode-discriminated: `manual` accepts only + `mode`, `cron` requires only a five-field `cron` expression, and `dates` + requires only a nonempty `dates` array +- Standard tracks reject `snapshot_schedule`; schedules are stored as virtual + registry metadata. Automated `cron` and `dates` execution is required but + remains pending until scheduler integration exists - Composition request objects are strict; unknown composition, component, filter, and deduplication keys return `400 Bad Request` - Selector fields form a discriminated request contract: diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 142b4025..08b67dec 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -89,6 +89,14 @@ service to verify that every component exists and is a standard track before persisting an initial virtual track; update and materialization retain the same service-layer validation. +Snapshot schedules use the same strict, mode-discriminated Zod schema at the +controller and service boundaries. `manual` has no selector field, `cron` +requires a five-field cron expression, and `dates` requires a nonempty array of +ISO timestamps. Standard-track creation rejects `snapshot_schedule` instead of +silently dropping it. Mongoose repeats the mode and track-type invariants for +direct persistence callers. Schedule configuration remains registry metadata; +P2 scheduler execution is not implemented. + There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a standard draft. Release planning never resolves composition and rejects a diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 6e0a2f86..a70ea022 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1114,7 +1114,14 @@ Short names (`enterprise`, `ics`, `mobile`) and STIX names ending in For primary matrices, which omit `x_mitre_domains` in published ATT&CK data, the domain is read from `external_references[].external_id`. `snapshot_schedule` is stored as metadata only; automated execution is not -yet implemented. +yet implemented. Its shape depends on `mode`: + +- `manual` accepts only `{ "mode": "manual" }`; +- `cron` requires a five-field `cron` expression and rejects `dates`; +- `dates` requires at least one ISO timestamp and rejects `cron`. + +Unknown schedule properties return `400 Bad Request`. Standard tracks also +reject `snapshot_schedule` rather than silently ignoring it. Composition, component, filter, and deduplication objects are strict. Unknown keys, including the incorrect singular `filters.domain`, return diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 947e2ada..b9f50209 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -108,9 +108,7 @@ Virtual tracks are identified by `stix.type = "virtual"` in their schema. // Snapshot schedule configuration snapshot_schedule: { - mode: "manual", // "manual" | "cron" | "dates" - cron: "0 0 1 1,7 *", // Jan 1 and July 1 at midnight - dates: ["2024-01-01T00:00:00Z", "2024-07-01T00:00:00Z"] + mode: "manual" // "manual" | "cron" | "dates" }, // Configuration @@ -518,6 +516,15 @@ The configuration is currently persisted as registry metadata only. No release-track scheduler consumes it yet, so `cron` and `dates` schedules do not create snapshots automatically. +Schedule payloads are strict and mode-specific: + +- `manual` accepts only `{ mode: "manual" }`. +- `cron` requires `cron` and rejects `dates`. +- `dates` requires a nonempty `dates` array and rejects `cron`. + +Unknown schedule fields return `400 Bad Request`. Standard tracks do not +support `snapshot_schedule`. + **Planned scheduler integration:** ```javascript scheduler.register({ From 294e4e5b11c5189713bbb4c6dee4b5353087f4b4 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:30:31 -0400 Subject: [PATCH 25/55] fix(release-tracks): validate virtual object type filters Constrain virtual component object-type filters to the canonical Workbench STIX vocabulary across request, service, persistence, OpenAPI, and documentation boundaries. Preserve exact component snapshot revisions. --- .../definitions/components/release-tracks.yml | 23 +- .../release-tracks/release-track-schemas.js | 20 +- .../release-track-validators.js | 10 + .../release-track-snapshot-schema.js | 7 +- .../release-tracks/release-tracks-service.js | 25 +- .../virtual-object-type-filters.spec.js | 241 ++++++++++++++++++ docs/developer/FRONTEND_TODO.md | 24 ++ docs/developer/TODO.md | 50 +++- docs/developer/release-tracks/entities.md | 5 + .../release-tracks/implementation-notes.md | 8 + docs/user/release-tracks/api-reference.md | 11 + docs/user/release-tracks/virtual-tracks.md | 7 + 12 files changed, 425 insertions(+), 6 deletions(-) create mode 100644 app/tests/api/release-tracks/virtual-object-type-filters.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 66c52f86..1ae22fc8 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -356,9 +356,30 @@ components: properties: object_types: type: array + minItems: 1 + uniqueItems: true items: type: string - description: 'Only include these STIX types' + enum: + - 'x-mitre-asset' + - 'campaign' + - 'x-mitre-collection' + - 'intrusion-set' + - 'course-of-action' + - 'tool' + - 'x-mitre-tactic' + - 'malware' + - 'x-mitre-matrix' + - 'relationship' + - 'marking-definition' + - 'identity' + - 'note' + - 'x-mitre-data-source' + - 'x-mitre-data-component' + - 'attack-pattern' + - 'x-mitre-analytic' + - 'x-mitre-detection-strategy' + description: 'Only include members whose object_ref uses one of these canonical Workbench STIX type prefixes; omit the property to include all types' domains: type: array items: diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index e1beeb0f..13e30b3c 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -17,6 +17,7 @@ const { xMitreVersionSchema, createStixIdValidator, } = require('@mitre-attack/attack-data-model'); +const types = require('../types'); // ----------------------------------------------------------------------------- // Custom STIX identifier @@ -277,9 +278,23 @@ const snapshotScheduleSchema = z.discriminatedUnion('mode', [ .strict(), ]); +const releaseTrackObjectTypes = Object.freeze(Object.values(types)); +const releaseTrackObjectTypeSchema = z.enum(releaseTrackObjectTypes); +const objectTypesFilterSchema = z + .array(releaseTrackObjectTypeSchema) + .min(1) + .superRefine((objectTypes, context) => { + if (new Set(objectTypes).size !== objectTypes.length) { + context.addIssue({ + code: 'custom', + message: 'Object type filters must not contain duplicate values', + }); + } + }); + const componentTrackFiltersSchema = z .object({ - object_types: z.array(z.string()).optional(), + object_types: objectTypesFilterSchema.optional(), domains: z.array(z.string()).optional(), }) .strict(); @@ -512,6 +527,9 @@ module.exports = { // Domain schemas trackNameSchema, cronSchema, + releaseTrackObjectTypes, + releaseTrackObjectTypeSchema, + objectTypesFilterSchema, // Re-exports from @mitre-attack/attack-data-model stixIdentifierSchema, diff --git a/app/lib/release-tracks/release-track-validators.js b/app/lib/release-tracks/release-track-validators.js index 4cae4594..e8e6465e 100644 --- a/app/lib/release-tracks/release-track-validators.js +++ b/app/lib/release-tracks/release-track-validators.js @@ -15,6 +15,7 @@ const { trackNameSchema, cronSchema, snapshotScheduleSchema, + objectTypesFilterSchema, stixIdentifierSchema, xMitreVersionSchema, createStixIdValidator, @@ -82,6 +83,14 @@ const validateSnapshotSchedule = { 'Snapshot schedule fields must match mode: manual has no selector, cron requires cron, and dates requires at least one date', }; +const validateObjectTypesFilter = { + validator: (value) => + value === undefined || + (Array.isArray(value) && objectTypesFilterSchema.safeParse(value).success), + message: + 'Object type filters must be a non-empty, duplicate-free list of supported Workbench STIX types', +}; + // ============================================================================= // Exports // ============================================================================= @@ -95,4 +104,5 @@ module.exports = { validateVersion, validateCron, validateSnapshotSchedule, + validateObjectTypesFilter, }; diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 4fd08ae2..4f72d32c 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -8,6 +8,7 @@ const { validateIdentityRef, validateMarkingDefRefs, validateVersion, + validateObjectTypesFilter, } = require('../../lib/release-tracks/release-track-validators'); // ============================================================================= @@ -84,7 +85,11 @@ const quarantineEntrySchema = new mongoose.Schema(quarantineEntryDefinition, { _ // --- Composition sub-schemas (virtual tracks) --- const componentTrackFiltersDefinition = { - object_types: { type: [String], default: undefined }, + object_types: { + type: [String], + default: undefined, + validate: validateObjectTypesFilter, + }, domains: { type: [String], default: undefined }, }; const componentTrackFiltersSchema = new mongoose.Schema(componentTrackFiltersDefinition, { diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 879b9edd..cda77646 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -15,7 +15,10 @@ // ============================================================================= const { BadRequestError, NotImplementedError } = require('../../exceptions'); -const { snapshotScheduleSchema } = require('../../lib/release-tracks/release-track-schemas'); +const { + compositionSchema, + snapshotScheduleSchema, +} = require('../../lib/release-tracks/release-track-schemas'); const snapshotService = require('./snapshot-service'); const standardTrackService = require('./standard-track-service'); const versioningService = require('./versioning-service'); @@ -190,6 +193,17 @@ exports.createTrack = async function createTrack(data) { validatedData = { ...data, snapshot_schedule: scheduleResult.data }; } + if (validatedData.composition !== undefined) { + const compositionResult = compositionSchema.safeParse(validatedData.composition); + if (!compositionResult.success) { + throw new BadRequestError({ + message: 'Invalid virtual track composition', + details: compositionResult.error.errors, + }); + } + validatedData = { ...validatedData, composition: compositionResult.data }; + } + if (validatedData.type === 'virtual' && validatedData.composition) { await virtualTrackService.validateComposition(validatedData.composition); } @@ -383,7 +397,14 @@ exports.updateConfig = function updateConfig(trackId, config, userId) { // ----------------------------------------------------------------------------- exports.updateComposition = function updateComposition(trackId, composition, userId) { - return virtualTrackService.updateComposition(trackId, composition, userId); + const compositionResult = compositionSchema.safeParse(composition); + if (!compositionResult.success) { + throw new BadRequestError({ + message: 'Invalid virtual track composition', + details: compositionResult.error.errors, + }); + } + return virtualTrackService.updateComposition(trackId, compositionResult.data, userId); }; exports.createVirtualSnapshot = function createVirtualSnapshot(trackId, options) { diff --git a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js new file mode 100644 index 00000000..fb18ac67 --- /dev/null +++ b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js @@ -0,0 +1,241 @@ +'use strict'; + +const mongoose = require('mongoose'); +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const types = require('../../../lib/types'); +const login = require('../../shared/login'); +const { cloneForCreate } = require('../../shared/clone-for-create'); +const { + compositionSchema, +} = require('../../../models/release-tracks/release-track-snapshot-schema'); +const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); + +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; +const supportedObjectTypes = Object.values(types); + +const compositionBoundarySchema = new mongoose.Schema({ + composition: { type: compositionSchema, required: true }, +}); +const CompositionBoundary = + mongoose.models.VirtualObjectTypeFilterCompositionBoundary || + mongoose.model('VirtualObjectTypeFilterCompositionBoundary', compositionBoundarySchema); + +describe('Virtual release-track object-type filters API', function () { + let app; + let passportCookie; + let componentTrack; + let virtualTrack; + let createSequence = 0; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + + componentTrack = await post('/api/release-tracks/new', { + name: 'Object Type Filter Component', + type: 'standard', + }); + virtualTrack = await post('/api/release-tracks/new', { + name: 'Object Type Filter Virtual', + type: 'virtual', + }); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + async function put(path, body, status = 200) { + const response = await request(app) + .put(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + function composition(objectTypes, trackId = componentTrack.id) { + return { + component_tracks: [ + { + track_id: trackId, + resolution_strategy: 'latest_tagged', + priority: 0, + filters: { object_types: objectTypes }, + }, + ], + deduplication: { strategy: 'prioritize_latest_object' }, + }; + } + + async function createVirtual(compositionBody, status = 201) { + createSequence += 1; + return post( + '/api/release-tracks/new', + { + name: `Object Type Filter Create ${createSequence}`, + type: 'virtual', + composition: compositionBody, + }, + status, + ); + } + + function buildMitigation(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'course-of-action', + labels: ['test'], + x_mitre_version: '1.0', + object_marking_refs: [staticMarkingDefinitionId], + }, + }; + } + + function buildMatrix(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'x-mitre-matrix', + external_references: [{ source_name: 'test-source', external_id: 'enterprise-attack' }], + object_marking_refs: [staticMarkingDefinitionId], + x_mitre_version: '1.0', + }, + }; + } + + it('accepts the canonical Workbench STIX type vocabulary', async function () { + const created = await createVirtual(composition(supportedObjectTypes)); + expect(created.composition.component_tracks[0].filters.object_types).toEqual( + supportedObjectTypes, + ); + + const updated = await put( + `/api/release-tracks/${virtualTrack.id}/virtual/composition`, + composition(supportedObjectTypes), + ); + expect(updated.composition.component_tracks[0].filters.object_types).toEqual( + supportedObjectTypes, + ); + }); + + it('rejects empty, duplicate, malformed, and unsupported object-type filters', async function () { + const invalidFilters = [ + [], + ['attack-pattern', 'attack-pattern'], + ['Attack-Pattern'], + ['not-a-workbench-type'], + ]; + + for (const objectTypes of invalidFilters) { + await createVirtual(composition(objectTypes), 400); + await put( + `/api/release-tracks/${virtualTrack.id}/virtual/composition`, + composition(objectTypes), + 400, + ); + } + }); + + it('repeats the accepted-value constraint at the persistence boundary', async function () { + const invalidCompositions = [ + composition(null), + composition([]), + composition(['attack-pattern', 'attack-pattern']), + composition(['not-a-workbench-type']), + ]; + + for (const invalidComposition of invalidCompositions) { + const boundary = new CompositionBoundary({ composition: invalidComposition }); + await expect(boundary.validate()).rejects.toThrow(); + } + }); + + it('repeats the accepted-value constraint for direct service callers', async function () { + const invalidComposition = composition(['not-a-workbench-type']); + + await expect( + releaseTracksService.createTrack({ + name: 'Invalid Service Object Type Filter', + type: 'virtual', + composition: invalidComposition, + }), + ).rejects.toThrow(); + + expect(() => + releaseTracksService.updateComposition(virtualTrack.id, invalidComposition), + ).toThrow(); + }); + + it('filters members without replacing the revision pinned by the tagged component', async function () { + const mitigation = await post('/api/mitigations', buildMitigation('Pinned Type Member')); + const matrix = await post('/api/matrices', buildMatrix('Excluded Type Member')); + + await post( + `/api/release-tracks/${componentTrack.id}/contents`, + { + x_mitre_contents: [mitigation, matrix].map((object) => ({ + obj_ref: object.stix.id, + obj_modified: object.stix.modified, + })), + }, + 200, + ); + await post(`/api/release-tracks/${componentTrack.id}/snapshots/latest/release`, {}, 200); + + const newerMitigationRevision = cloneForCreate(mitigation); + newerMitigationRevision.stix.modified = new Date(Date.now() + 1000).toISOString(); + newerMitigationRevision.stix.name = 'Newer Unpinned Type Member'; + await post('/api/mitigations', newerMitigationRevision); + + const filteredTrack = await createVirtual(composition(['course-of-action'])); + const materialized = await post( + `/api/release-tracks/${filteredTrack.id}/virtual/snapshots/create`, + {}, + ); + + expect(materialized.members).toHaveLength(1); + expect(materialized.members[0].object_ref).toBe(mitigation.stix.id); + expect(new Date(materialized.members[0].object_modified).toISOString()).toBe( + mitigation.stix.modified, + ); + expect(new Date(materialized.members[0].object_modified).toISOString()).not.toBe( + newerMitigationRevision.stix.modified, + ); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 5363e97b..bffb5a5b 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -524,6 +524,30 @@ Done when: - Tests cover all three modes and mode switching. - User-facing copy says scheduled execution is not yet active. +## P1 — Align virtual component object-type filters + +### [ ] Use the complete canonical Workbench STIX type vocabulary + +The backend now validates `composition.component_tracks[].filters.object_types` +against its canonical STIX type registry. When `object_types` is present, it +must be a nonempty array of unique, case-sensitive STIX type names. Omit the +property to include all types; do not send an empty array. + +The create dialog already removes the property when the user has no +selections, and `mat-select` naturally prevents duplicates. Its current +hard-coded options are only a subset of the server vocabulary, however. They +omit `identity`, `marking-definition`, `note`, `relationship`, +`x-mitre-collection`, and `x-mitre-data-source`. + +Done when: + +- Creation and composition editing use the same complete canonical option + list. +- Clearing all selections removes `object_types` from the submitted filter. +- Unknown values loaded from stale local state are rejected or removed before + submission. +- Tests cover the complete option list and clearing the filter. + ## P1 — Separate snapshot and preview output-format types ### [ ] Remove the invalid `snapshot` format and model `summary` correctly diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 3e727a4c..10bd0f3e 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -28,7 +28,7 @@ completion backlog. - `manual` rejects `cron` and `dates`; - `cron` requires `cron` and rejects `dates`; - `dates` requires at least one date and rejects `cron`. -- [ ] Constrain or document accepted `filters.object_types` values and add +- [x] Constrain or document accepted `filters.object_types` values and add direct regression coverage for exact-revision filtering. ### P1 — Deduplication correctness @@ -246,6 +246,54 @@ Verification result (2026-07-29): Include the required component priority in virtual-track creation coverage. ``` +### Current implementation slice — Object-type filter contracts + +- [x] Define `filters.object_types` against the canonical Workbench STIX type + vocabulary instead of accepting arbitrary strings. +- [x] Reject empty arrays, duplicate values, malformed values, and unsupported + object types on both virtual-track creation and composition update. +- [x] Repeat the accepted-value constraint at the Mongoose persistence + boundary. +- [x] Add direct materialization coverage proving that object-type filtering + preserves the exact revision pinned by the tagged component snapshot rather + than resolving the latest database revision. +- [x] Align OpenAPI, user/developer documentation, frontend guidance, and + Bruno; verify whether `internalattack` needs a typed client change. +- [x] Run focused regression specs, lint, and the complete `npm test` suite. +- [x] Review the final diff and propose conventional commit messages. + +Verification result (2026-07-29): + +- The dedicated object-type contract and exact-revision materialization spec + passes (5); the combined virtual composition, domain, schedule, and + object-type filter group passes (18). +- OpenAPI validation passes (2), backend lint passes, and the required clean + `npm test` run passes (OpenAPI 2, config 21, API 923, middleware 24). +- Earlier complete runs encountered unrelated shared-suite flakes in user + account startup, analytics socket handling, and campaign/group HTTP + handling. The affected specs pass in isolation (14, 12, and 44 + respectively). +- `internalattack` already accepts composition filters as a mapping, so this + contract clarification does not require a typed client change. +- Proposed REST API commit: + + ```text + fix(release-tracks): validate virtual object type filters + + Constrain virtual component object-type filters to the canonical Workbench + STIX vocabulary across request, service, persistence, OpenAPI, and + documentation boundaries. Preserve exact component snapshot revisions. + ``` + +- Proposed Bruno commit: + + ```text + docs(release-tracks): document object type filters + + Document canonical virtual component object-type values, omission semantics, + and exact-revision behavior. + ``` + ### Tracker consolidation - [x] Consolidate the virtual-track completion backlog into this section. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index c975ee46..d9cd8cf7 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -489,6 +489,11 @@ currently validates and persists all three shapes. Automated execution for - Standard tracks reject `snapshot_schedule`; schedules are stored as virtual registry metadata. Automated `cron` and `dates` execution is required but remains pending until scheduler integration exists +- `filters.object_types` uses the canonical Workbench STIX type names from + `app/lib/types.js`. When present, it must be nonempty and duplicate-free; + omit it to include every object type. Filtering reads the type prefix from + each member's immutable `object_ref`, so it preserves the exact revision + pinned by the resolved component snapshot - Composition request objects are strict; unknown composition, component, filter, and deduplication keys return `400 Bad Request` - Selector fields form a discriminated request contract: diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 08b67dec..66c4853f 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -97,6 +97,14 @@ silently dropping it. Mongoose repeats the mode and track-type invariants for direct persistence callers. Schedule configuration remains registry metadata; P2 scheduler execution is not implemented. +Component `filters.object_types` values are constrained to the canonical +Workbench STIX vocabulary exported by `app/lib/types.js`. The request schema +requires a nonempty, duplicate-free array when the property is present, and +the Mongoose composition schema repeats that invariant. Omitting the property +means no type filter. Materialization compares each value to the type prefix +already encoded in the resolved snapshot member's `object_ref`; it never +re-resolves that member to the latest database revision. + There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a standard draft. Release planning never resolves composition and rejects a diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index a70ea022..4335e081 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1113,6 +1113,17 @@ Short names (`enterprise`, `ics`, `mobile`) and STIX names ending in `-attack` are equivalent. Objects without a matching domain are excluded. For primary matrices, which omit `x_mitre_domains` in published ATT&CK data, the domain is read from `external_references[].external_id`. + +`filters.object_types` accepts canonical Workbench STIX type names: +`attack-pattern`, `campaign`, `course-of-action`, `identity`, `intrusion-set`, +`malware`, `marking-definition`, `note`, `relationship`, `tool`, +`x-mitre-analytic`, `x-mitre-asset`, `x-mitre-collection`, +`x-mitre-data-component`, `x-mitre-data-source`, +`x-mitre-detection-strategy`, `x-mitre-matrix`, and `x-mitre-tactic`. +When present, the array must contain at least one unique value. Omit it to +include all object types. Type filtering preserves each member revision pinned +by the resolved component snapshot. + `snapshot_schedule` is stored as metadata only; automated execution is not yet implemented. Its shape depends on `mode`: diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index b9f50209..6f965c90 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -227,6 +227,13 @@ ATT&CK data identifies their domain through `external_references[].external_id`, so virtual filtering uses that established matrix fallback. +`object_types` values are case-sensitive canonical Workbench STIX type names. +When the property is present, it must contain at least one value and cannot +contain duplicates. Omit `object_types` to include every type. The filter reads +the type prefix from each resolved member's `object_ref`, so a newer database +revision cannot replace the exact revision pinned by the component release. +Unsupported values return `400 Bad Request`. + `stix_pattern` is not part of the current request schema and is not implemented. Filter objects are strict, so misspelled or unsupported keys such as `domain` fail with `400 Bad Request`; use the plural `domains`. From 080930fdfe5d1bf173deeaa7211d4db36c06813e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:57:24 -0400 Subject: [PATCH 26/55] fix(release-tracks): deduplicate virtual revisions deterministically Collapse exact component revision duplicates before resolving conflicts, attribute every surviving member to one deterministic source, and quarantine only genuinely different revisions. --- .../definitions/components/release-tracks.yml | 5 +- .../deduplication-strategies.js | 114 ++++++-- .../release-tracks/virtual-track-service.js | 20 +- .../virtual-deduplication.spec.js | 271 ++++++++++++++++++ docs/developer/FRONTEND_TODO.md | 29 ++ docs/developer/TODO.md | 65 ++++- docs/developer/release-tracks/entities.md | 5 + .../release-tracks/implementation-notes.md | 17 ++ docs/user/release-tracks/api-reference.md | 43 ++- docs/user/release-tracks/virtual-tracks.md | 21 +- 10 files changed, 537 insertions(+), 53 deletions(-) create mode 100644 app/tests/api/release-tracks/virtual-deduplication.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 1ae22fc8..fb9bc2c4 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -77,7 +77,10 @@ components: Immutable component-resolution provenance for a materialized virtual draft. Null or absent means the virtual composition is configured but has not been materialized and therefore cannot be previewed or - released. + released. Duplicate counts include STIX IDs contributed by more than + one component, while conflicts include only IDs with genuinely + different revisions. Each surviving member is attributed to exactly + one component in objects_contributed. config: $ref: '#/components/schemas/track-config' version_history: diff --git a/app/lib/release-tracks/deduplication-strategies.js b/app/lib/release-tracks/deduplication-strategies.js index de9090dc..25c4ba10 100644 --- a/app/lib/release-tracks/deduplication-strategies.js +++ b/app/lib/release-tracks/deduplication-strategies.js @@ -24,7 +24,8 @@ * Each entry: { object_ref, object_modified, _source_track_id, _source_track_name, * _source_snapshot_modified, _source_snapshot_version, _source_priority } * @param {string} strategy - One of the four deduplication strategies - * @returns {{ members: Array, quarantined: Array, report: Object }} + * @returns {{ members: Array, quarantined: Array, + * contributions: Array, report: Object }} */ exports.deduplicate = function deduplicate(allMembers, strategy) { // Group entries by object_ref to identify duplicates @@ -39,31 +40,56 @@ exports.deduplicate = function deduplicate(allMembers, strategy) { const members = []; const quarantined = []; + const contributions = []; const conflictsResolved = []; + let duplicatesFound = 0; for (const [objectRef, entries] of groups) { - if (entries.length === 1) { - // No conflict — single source - members.push(_stripSourceMeta(entries[0])); + if (entries.length > 1) { + duplicatesFound += 1; + } + + const distinctRevisions = _collapseExactRevisions(entries, strategy); + if (distinctRevisions.length === 1) { + // No conflict — one distinct revision with one selected source + _addMember(distinctRevisions[0], members, contributions); continue; } - // Conflict: same object_ref from multiple component tracks + // Conflict: same object_ref with genuinely different revisions switch (strategy) { case 'prioritize_latest_object': - _resolveByLatestObject(objectRef, entries, members, conflictsResolved); + _resolveByLatestObject( + objectRef, + distinctRevisions, + members, + contributions, + conflictsResolved, + ); break; case 'prioritize_latest_snapshot': - _resolveByLatestSnapshot(objectRef, entries, members, conflictsResolved); + _resolveByLatestSnapshot( + objectRef, + distinctRevisions, + members, + contributions, + conflictsResolved, + ); break; case 'prioritize_higher_priority': - _resolveByHigherPriority(objectRef, entries, members, conflictsResolved); + _resolveByHigherPriority( + objectRef, + distinctRevisions, + members, + contributions, + conflictsResolved, + ); break; case 'quarantine': - _resolveByQuarantine(objectRef, entries, quarantined, conflictsResolved); + _resolveByQuarantine(objectRef, distinctRevisions, quarantined, conflictsResolved); break; default: @@ -74,11 +100,11 @@ exports.deduplicate = function deduplicate(allMembers, strategy) { const report = { total_objects_before: allMembers.length, total_objects_after: members.length, - duplicates_found: conflictsResolved.length, + duplicates_found: duplicatesFound, conflicts_resolved: conflictsResolved, }; - return { members, quarantined, report }; + return { members, quarantined, contributions, report }; }; // ============================================================================= @@ -88,7 +114,7 @@ exports.deduplicate = function deduplicate(allMembers, strategy) { /** * Keep the entry with the most recent object_modified timestamp. */ -function _resolveByLatestObject(objectRef, entries, members, conflictsResolved) { +function _resolveByLatestObject(objectRef, entries, members, contributions, conflictsResolved) { let winner = entries[0]; for (let i = 1; i < entries.length; i++) { if ( @@ -98,7 +124,7 @@ function _resolveByLatestObject(objectRef, entries, members, conflictsResolved) } } - members.push(_stripSourceMeta(winner)); + _addMember(winner, members, contributions); conflictsResolved.push({ object_ref: objectRef, strategy: 'prioritize_latest_object', @@ -112,17 +138,15 @@ function _resolveByLatestObject(objectRef, entries, members, conflictsResolved) * Keep the entry from the component track whose resolved snapshot has the * most recent modified timestamp. */ -function _resolveByLatestSnapshot(objectRef, entries, members, conflictsResolved) { +function _resolveByLatestSnapshot(objectRef, entries, members, contributions, conflictsResolved) { let winner = entries[0]; for (let i = 1; i < entries.length; i++) { - const entrySnapshotTime = new Date(entries[i]._source_snapshot_modified).getTime(); - const winnerSnapshotTime = new Date(winner._source_snapshot_modified).getTime(); - if (entrySnapshotTime > winnerSnapshotTime) { + if (_preferLatestSnapshot(entries[i], winner)) { winner = entries[i]; } } - members.push(_stripSourceMeta(winner)); + _addMember(winner, members, contributions); conflictsResolved.push({ object_ref: objectRef, strategy: 'prioritize_latest_snapshot', @@ -136,15 +160,15 @@ function _resolveByLatestSnapshot(objectRef, entries, members, conflictsResolved * Keep the entry from the component track with the highest priority * (lowest priority number). */ -function _resolveByHigherPriority(objectRef, entries, members, conflictsResolved) { +function _resolveByHigherPriority(objectRef, entries, members, contributions, conflictsResolved) { let winner = entries[0]; for (let i = 1; i < entries.length; i++) { - if (entries[i]._source_priority < winner._source_priority) { + if (_preferHigherPriority(entries[i], winner)) { winner = entries[i]; } } - members.push(_stripSourceMeta(winner)); + _addMember(winner, members, contributions); conflictsResolved.push({ object_ref: objectRef, strategy: 'prioritize_higher_priority', @@ -191,3 +215,51 @@ function _stripSourceMeta(entry) { object_modified: entry.object_modified, }; } + +/** + * Collapse repeated contributions of an exact object revision to one source. + * Source ownership follows the active strategy where it can distinguish the + * sources, then falls back to the required unique component priority. + */ +function _collapseExactRevisions(entries, strategy) { + const revisions = new Map(); + + for (const entry of entries) { + const revisionKey = new Date(entry.object_modified).getTime(); + const current = revisions.get(revisionKey); + if (!current || _preferSource(entry, current, strategy)) { + revisions.set(revisionKey, entry); + } + } + + return Array.from(revisions.values()); +} + +function _preferSource(candidate, current, strategy) { + if (strategy === 'prioritize_latest_snapshot') { + return _preferLatestSnapshot(candidate, current); + } + return _preferHigherPriority(candidate, current); +} + +function _preferLatestSnapshot(candidate, current) { + const candidateTime = new Date(candidate._source_snapshot_modified).getTime(); + const currentTime = new Date(current._source_snapshot_modified).getTime(); + if (candidateTime !== currentTime) { + return candidateTime > currentTime; + } + return _preferHigherPriority(candidate, current); +} + +function _preferHigherPriority(candidate, current) { + return candidate._source_priority < current._source_priority; +} + +function _addMember(entry, members, contributions) { + members.push(_stripSourceMeta(entry)); + contributions.push({ + object_ref: entry.object_ref, + object_modified: entry.object_modified, + source_track_id: entry._source_track_id, + }); +} diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 6c419422..4ccb869c 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -331,25 +331,17 @@ async function resolveComposition(snapshot, registryMap) { } // Deduplicate across all components - const { members, quarantined, report } = deduplicationStrategies.deduplicate( + const { members, quarantined, contributions, report } = deduplicationStrategies.deduplicate( allAnnotatedMembers, strategy, ); - // Update objects_contributed per component by counting how many of each - // component's members survived deduplication + // Each surviving member is explicitly attributed to one source component + // by deduplication, including exact revisions supplied by multiple tracks. const survivorSources = new Map(); - for (const annotated of allAnnotatedMembers) { - // Check if this specific entry survived deduplication - const survived = members.some( - (m) => - m.object_ref === annotated.object_ref && - new Date(m.object_modified).getTime() === new Date(annotated.object_modified).getTime(), - ); - if (survived) { - const count = survivorSources.get(annotated._source_track_id) || 0; - survivorSources.set(annotated._source_track_id, count + 1); - } + for (const contribution of contributions) { + const count = survivorSources.get(contribution.source_track_id) || 0; + survivorSources.set(contribution.source_track_id, count + 1); } for (const meta of componentSnapshotsMeta) { diff --git a/app/tests/api/release-tracks/virtual-deduplication.spec.js b/app/tests/api/release-tracks/virtual-deduplication.spec.js new file mode 100644 index 00000000..8950a190 --- /dev/null +++ b/app/tests/api/release-tracks/virtual-deduplication.spec.js @@ -0,0 +1,271 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Virtual release-track deduplication API', function () { + let app; + let passportCookie; + let exactRevision; + let conflictRevisionA; + let conflictRevisionB; + let componentA; + let componentB; + let componentC; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + + exactRevision = await post('/api/mitigations', buildMitigation('Exact Shared Revision')); + conflictRevisionA = await post('/api/mitigations', buildMitigation('Conflict Revision A')); + conflictRevisionB = await post( + '/api/mitigations', + buildMitigation('Conflict Revision B', conflictRevisionA), + ); + + componentC = await createReleasedComponent('Deduplication Component C', [conflictRevisionB]); + await advanceSnapshotClock(); + componentA = await createReleasedComponent('Deduplication Component A', [ + exactRevision, + conflictRevisionA, + ]); + await advanceSnapshotClock(); + componentB = await createReleasedComponent('Deduplication Component B', [ + exactRevision, + conflictRevisionA, + ]); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + function buildMitigation(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'course-of-action', + labels: ['test'], + x_mitre_version: '1.0', + object_marking_refs: [staticMarkingDefinitionId], + }, + }; + } + + async function advanceSnapshotClock() { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + async function createReleasedComponent(name, members) { + const track = await post('/api/release-tracks/new', { name, type: 'standard' }); + await post( + `/api/release-tracks/${track.id}/contents`, + { + x_mitre_contents: members.map((member) => ({ + obj_ref: member.stix.id, + obj_modified: member.stix.modified, + })), + }, + 200, + ); + const release = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}, 200); + return { ...track, release }; + } + + async function materialize(strategy) { + const names = { + prioritize_latest_object: 'Dedup Latest Object', + prioritize_latest_snapshot: 'Dedup Latest Snapshot', + prioritize_higher_priority: 'Dedup Higher Priority', + quarantine: 'Dedup Quarantine', + }; + const virtual = await post('/api/release-tracks/new', { + name: names[strategy], + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: componentA.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + { + track_id: componentB.id, + resolution_strategy: 'latest_tagged', + priority: 2, + }, + { + track_id: componentC.id, + resolution_strategy: 'latest_tagged', + priority: 3, + }, + ], + deduplication: { strategy }, + }, + }); + + return post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}); + } + + function memberModified(snapshot, objectRef) { + return snapshot.members.find((member) => member.object_ref === objectRef)?.object_modified; + } + + function contributions(snapshot) { + return Object.fromEntries( + snapshot.composition_resolution.component_snapshots.map((component) => [ + component.track_id, + component.objects_contributed, + ]), + ); + } + + function expectReport(snapshot, expectedAfter) { + const resolution = snapshot.composition_resolution; + expect(resolution.deduplication).toMatchObject({ + total_objects_before: 5, + total_objects_after: expectedAfter, + duplicates_found: 2, + }); + expect(resolution.deduplication.conflicts_resolved).toHaveLength(1); + expect(resolution.deduplication.conflicts_resolved[0].object_ref).toBe( + conflictRevisionA.stix.id, + ); + expect(Object.values(contributions(snapshot)).reduce((total, count) => total + count, 0)).toBe( + resolution.summary.total_objects, + ); + } + + it('prioritizes the latest object revision and attributes each survivor once', async function () { + const snapshot = await materialize('prioritize_latest_object'); + + expect(snapshot.members).toHaveLength(2); + expect(snapshot.quarantine).toEqual([]); + expect(memberModified(snapshot, exactRevision.stix.id)).toBe(exactRevision.stix.modified); + expect(memberModified(snapshot, conflictRevisionA.stix.id)).toBe( + conflictRevisionB.stix.modified, + ); + expect(contributions(snapshot)).toEqual({ + [componentA.id]: 1, + [componentB.id]: 0, + [componentC.id]: 1, + }); + expectReport(snapshot, 2); + expect(snapshot.composition_resolution.deduplication.conflicts_resolved[0]).toMatchObject({ + strategy: 'prioritize_latest_object', + winner_source: componentC.id, + candidates_count: 2, + }); + }); + + it('prioritizes the latest component snapshot with deterministic source ownership', async function () { + const snapshot = await materialize('prioritize_latest_snapshot'); + + expect(snapshot.members).toHaveLength(2); + expect(snapshot.quarantine).toEqual([]); + expect(memberModified(snapshot, exactRevision.stix.id)).toBe(exactRevision.stix.modified); + expect(memberModified(snapshot, conflictRevisionA.stix.id)).toBe( + conflictRevisionA.stix.modified, + ); + expect(contributions(snapshot)).toEqual({ + [componentA.id]: 0, + [componentB.id]: 2, + [componentC.id]: 0, + }); + expectReport(snapshot, 2); + expect(snapshot.composition_resolution.deduplication.conflicts_resolved[0]).toMatchObject({ + strategy: 'prioritize_latest_snapshot', + winner_source: componentB.id, + candidates_count: 2, + }); + }); + + it('prioritizes the highest-priority component and attributes each survivor once', async function () { + const snapshot = await materialize('prioritize_higher_priority'); + + expect(snapshot.members).toHaveLength(2); + expect(snapshot.quarantine).toEqual([]); + expect(memberModified(snapshot, exactRevision.stix.id)).toBe(exactRevision.stix.modified); + expect(memberModified(snapshot, conflictRevisionA.stix.id)).toBe( + conflictRevisionA.stix.modified, + ); + expect(contributions(snapshot)).toEqual({ + [componentA.id]: 2, + [componentB.id]: 0, + [componentC.id]: 0, + }); + expectReport(snapshot, 2); + expect(snapshot.composition_resolution.deduplication.conflicts_resolved[0]).toMatchObject({ + strategy: 'prioritize_higher_priority', + winner_source: componentA.id, + candidates_count: 2, + }); + }); + + it('quarantines only distinct revisions and retains an exact shared revision', async function () { + const snapshot = await materialize('quarantine'); + + expect(snapshot.members).toEqual([ + { + object_ref: exactRevision.stix.id, + object_modified: exactRevision.stix.modified, + }, + ]); + expect(snapshot.quarantine).toHaveLength(2); + expect(snapshot.quarantine.map((entry) => entry.object_modified).sort()).toEqual( + [conflictRevisionA.stix.modified, conflictRevisionB.stix.modified].sort(), + ); + expect(snapshot.quarantine.map((entry) => entry.source_track_id).sort()).toEqual( + [componentA.id, componentC.id].sort(), + ); + expect(contributions(snapshot)).toEqual({ + [componentA.id]: 1, + [componentB.id]: 0, + [componentC.id]: 0, + }); + expectReport(snapshot, 1); + expect(snapshot.composition_resolution.summary).toEqual({ + total_objects: 1, + quarantined_objects: 2, + }); + expect(snapshot.composition_resolution.deduplication.conflicts_resolved[0]).toEqual({ + object_ref: conflictRevisionA.stix.id, + strategy: 'quarantine', + quarantined_count: 2, + }); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index bffb5a5b..901cfc17 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -548,6 +548,35 @@ Done when: submission. - Tests cover the complete option list and clearing the filter. +## P1 — Distinguish virtual duplicates from revision conflicts + +### [ ] Align resolution metrics and fixtures with deterministic deduplication + +Virtual materialization now reports duplicate contributions and revision +conflicts as related but different concepts: + +- `composition_resolution.deduplication.duplicates_found` counts STIX object + IDs contributed by more than one component, even when every component + supplies the same exact revision. +- `conflicts_resolved` contains only object IDs with genuinely different + `object_modified` revisions. +- An exact revision shared by multiple components produces one member and is + never quarantined. +- Each member is attributed to exactly one component. Consequently, the sum of + `component_snapshots[].objects_contributed` equals + `composition_resolution.summary.total_objects`. + +The current page already displays separate duplicate and conflict counters. +Preserve that distinction instead of assuming the two counts are equal. + +Done when: + +- Resolution fixtures include an identical revision shared across components + and a separate object with conflicting revisions. +- Duplicate and conflict counters render their respective backend fields. +- Component contribution counts add up to the resolved member total. +- Quarantine views never show repeated copies of the same exact revision. + ## P1 — Separate snapshot and preview output-format types ### [ ] Remove the invalid `snapshot` format and model `summary` correctly diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 10bd0f3e..337d9c5a 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -33,13 +33,13 @@ completion backlog. ### P1 — Deduplication correctness -- [ ] Treat the same exact object revision contributed by multiple components +- [x] Treat the same exact object revision contributed by multiple components as one duplicate, not a conflicting revision. -- [ ] Ensure the `quarantine` strategy only quarantines genuinely different +- [x] Ensure the `quarantine` strategy only quarantines genuinely different revisions of the same object. -- [ ] Attribute each surviving revision to one deterministic component so +- [x] Attribute each surviving revision to one deterministic component so `objects_contributed` totals cannot exceed `summary.total_objects`. -- [ ] Add dedicated tests for all four strategies: +- [x] Add dedicated tests for all four strategies: `prioritize_latest_object`, `prioritize_latest_snapshot`, `prioritize_higher_priority`, and `quarantine`. @@ -294,6 +294,63 @@ Verification result (2026-07-29): and exact-revision behavior. ``` +### Current implementation slice — Deterministic virtual deduplication + +- [x] Add materialization regressions for all four deduplication strategies + using both an exact revision shared by multiple components and genuinely + different revisions of the same STIX object. +- [x] Collapse repeated contributions of the same `(object_ref, + object_modified)` revision before applying conflict resolution. +- [x] Count an object contributed by multiple components once in + `duplicates_found`, but include it in `conflicts_resolved` only when multiple + distinct revisions remain after exact-revision collapse. +- [x] Choose one deterministic source component for every surviving revision: + use the active strategy's ordering and use component priority as the stable + tie-breaker. +- [x] Quarantine one entry per distinct conflicting revision and leave an + identical revision shared by multiple components in `members`. +- [x] Derive `objects_contributed` from explicit survivor attribution so its + component total equals `summary.total_objects`. +- [x] Align OpenAPI, user/developer documentation, frontend guidance, Bruno, + and `internalattack` if the clarified response semantics require downstream + changes. +- [x] Run focused regression specs, lint, and the complete `npm test` suite. +- [x] Review the final diff and propose conventional commit messages. + +Verification result (2026-07-29): + +- The dedicated four-strategy deduplication spec passes (4); the combined + deduplication, quarantine, and back-reference release-track group passes + (29). +- OpenAPI validation passes (2), backend lint passes, and the required clean + `npm test` run passes (OpenAPI 2, config 21, API 936, middleware 24). +- An earlier complete run encountered unrelated shared-suite 404, 400, and + connection-reset failures in Recent Activity, References, and Ephemeral + Bundle tests. Those three specs pass together in isolation (30). +- `internalattack` exposes the resolution response as an untyped mapping, so + the clarified metric semantics do not require a Python client change. +- Performance review result: `PERFORMANT`. The implementation replaces the + prior input-to-output nested survivor scan with linear source attribution; + no database, blocking, or resource-management regression was found. +- Proposed REST API commit: + + ```text + fix(release-tracks): deduplicate virtual revisions deterministically + + Collapse exact component revision duplicates before resolving conflicts, + attribute every surviving member to one deterministic source, and quarantine + only genuinely different revisions. + ``` + +- Proposed Bruno commit: + + ```text + docs(release-tracks): clarify virtual deduplication + + Document exact-revision collapse, genuine conflict handling, and deterministic + component contribution accounting. + ``` + ### Tracker consolidation - [x] Consolidate the virtual-track completion backlog into this section. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index d9cd8cf7..7790bca1 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -494,6 +494,11 @@ currently validates and persists all three shapes. Automated execution for omit it to include every object type. Filtering reads the type prefix from each member's immutable `object_ref`, so it preserves the exact revision pinned by the resolved component snapshot +- Exact revisions contributed by multiple components collapse to one member + before conflict resolution. Only genuinely different revisions of one + `object_ref` are resolved or quarantined. Every surviving member is + attributed to exactly one deterministic component, so summed + `objects_contributed` equals `summary.total_objects` - Composition request objects are strict; unknown composition, component, filter, and deduplication keys return `400 Bad Request` - Selector fields form a discriminated request contract: diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 66c4853f..e3a4ed9b 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -105,6 +105,23 @@ means no type filter. Materialization compares each value to the type prefix already encoded in the resolved snapshot member's `object_ref`; it never re-resolves that member to the latest database revision. +Virtual deduplication distinguishes duplicate contributions from revision +conflicts. Entries are grouped first by `object_ref`, then by the exact +`object_modified` timestamp. Multiple components contributing the same exact +revision produce one member and no conflict; multiple distinct revisions of +one object invoke the configured resolution strategy. For exact-revision +source ownership, `prioritize_latest_snapshot` selects the newest resolved +component snapshot, while the other strategies use the required component +priority; priority also breaks equal-snapshot ties. + +Deduplication returns an internal source attribution for every surviving +member. `objects_contributed` is calculated from those attributions rather +than matching each output member back to every input contribution. Therefore +the component contribution total equals +`composition_resolution.summary.total_objects`. Under `quarantine`, repeated +copies of one exact revision remain a single member, and genuine conflicts +produce one quarantine entry per distinct revision. + There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a standard draft. Release planning never resolves composition and rejects a diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 4335e081..8e29fbbc 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1199,26 +1199,38 @@ POST /api/release-tracks/:id/virtual/snapshots/create ```json { - "stix": { - "id": "x-mitre-collection--virtual-uuid", - "modified": "2024-03-01T10:00:00Z", - "x_mitre_version": null, - "type": "virtual" - }, + "id": "release-track--virtual-uuid", + "type": "virtual", + "modified": "2024-03-01T10:00:00Z", + "version": null, + "name": "Enterprise ATT&CK", + "members": [], + "quarantine": [], "composition_resolution": { "resolved_at": "2024-03-01T10:00:00Z", "component_snapshots": [ { - "track_id": "GroupsMonthly--uuid", + "track_id": "release-track--groups-monthly", "track_name": "Groups Monthly", - "resolved_snapshot": "2024-02-15T10:00:00Z", + "track_type": "standard", + "resolved_snapshot_id": "2024-02-15T10:00:00Z", "resolved_version": "5.2", "strategy_used": "latest_tagged", - "object_count": 47 + "total_objects_in_source": 47, + "objects_after_filter": 47, + "objects_contributed": 47 } ], - "total_objects": 870, - "duplicates_resolved": 0 + "deduplication": { + "total_objects_before": 47, + "total_objects_after": 47, + "duplicates_found": 0, + "conflicts_resolved": [] + }, + "summary": { + "total_objects": 47, + "quarantined_objects": 0 + } } } ``` @@ -1230,6 +1242,15 @@ preview is the authoritative comparison and representation of the persisted draft that would be tagged. A non-null `composition_resolution` is the readiness marker for those shared release operations. +`duplicates_found` counts object IDs contributed by more than one component, +including repeated contributions of the same exact revision. +`conflicts_resolved` includes only object IDs for which multiple distinct +`object_modified` revisions remained after exact-revision collapse. The +component `objects_contributed` counts partition the surviving `members`, so +their sum equals `summary.total_objects`. With the `quarantine` strategy, +identical revisions remain one member and only distinct conflicting revisions +enter `quarantine`. + ### Promote a Quarantined Virtual Revision ``` diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 6f965c90..053e50ba 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -240,7 +240,17 @@ as `domain` fail with `400 Bad Request`; use the plural `domains`. ### Deduplication Strategies -When multiple component tracks contain the same object (same `stix.id`), a conflict occurs during the sync operation. The virtual track's deduplication strategy determines how to resolve the conflict. Four strategies are available: +When multiple component tracks contain the same object (same `stix.id`), the +materialization records one duplicate object. Contributions with the same +`modified` timestamp are the same exact revision, so they collapse to one +member and do not constitute a conflict. The configured strategy is applied +only when multiple distinct revisions remain. Four strategies are available: + +Each surviving member is attributed to one component. The active strategy +selects that source where applicable, with the component's required unique +priority providing a stable tie-breaker. As a result, the sum of +`component_snapshots[].objects_contributed` equals +`composition_resolution.summary.total_objects`. #### 1. `prioritize_latest_object` @@ -343,6 +353,11 @@ composition: { Don't automatically choose a version. Instead, store **both** versions in the virtual track's `quarantine` tier for manual review and resolution. +Only distinct revisions are quarantined. If several components contribute the +same exact revision, it remains one ordinary member. If two distinct revisions +are present and either is contributed repeatedly, quarantine contains one +entry for each distinct revision rather than one entry per component. + ```javascript deduplication: { strategy: "quarantine" @@ -497,9 +512,11 @@ POST /api/release-tracks/:id/virtual/snapshots/create - Apply `filters` to get subset of objects - Collect all object references with source metadata 2. Apply deduplication rules across all components: - - If no conflicts: objects go to virtual track's `members` + - Collapse identical `(object_ref, object_modified)` contributions + - If no distinct-revision conflicts remain: objects go to virtual track's `members` - If conflicts + `quarantine` strategy: both versions go to `quarantine` - If conflicts + other strategies: winning version goes to `members` + - Attribute every surviving member to one deterministic source component 3. Create new virtual track snapshot with: - New `snapshot_id` and `modified` timestamp - `version = null` (always starts as draft) From dafdb642220249d2cf9e90e23a50e1e5b861f650 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:20:16 -0400 Subject: [PATCH 27/55] fix(release-tracks): record virtual release provenance Persist immutable component track versions from the materialized virtual draft in release history, validate the provenance map, and align API, documentation, frontend, and regression contracts. --- .../definitions/components/release-tracks.yml | 15 ++ .../release-track-snapshot-schema.js | 21 ++- .../release-tracks/versioning-service.js | 16 ++ .../release-tracks-release.spec.js | 139 ++++++++++++++++++ docs/developer/FRONTEND_TODO.md | 38 +++++ docs/developer/TODO.md | 59 +++++++- docs/developer/release-tracks/entities.md | 10 +- .../release-tracks/implementation-notes.md | 11 ++ docs/user/release-tracks/api-reference.md | 21 +++ docs/user/release-tracks/virtual-tracks.md | 16 +- 10 files changed, 336 insertions(+), 10 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index fb9bc2c4..93e29940 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -456,6 +456,21 @@ components: candidates_count: type: number description: 'Objects in candidates at time of release' + quarantine_count: + type: number + description: 'Objects in quarantine at time of a virtual release' + component_versions: + type: object + description: | + Virtual releases only. Immutable provenance keyed by component + release-track ID; each value is the tagged component version frozen + in the released draft's composition_resolution. Standard release + history entries omit this property. + additionalProperties: + type: string + pattern: '^\d+\.\d+$' + example: + release-track--a1b2c3d4-e5f6-7890-abcd-ef1234567890: '5.2' release-track-registry: type: object diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 4f72d32c..77e369ec 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -154,6 +154,7 @@ const componentSnapshotResolutionDefinition = { resolved_snapshot_id: { type: Date, required: true }, resolved_version: { type: String, + required: true, validate: validateVersion, }, strategy_used: { type: String, required: true }, @@ -288,8 +289,24 @@ const versionHistoryEntryDefinition = { candidates_count: { type: Number }, quarantine_count: { type: Number }, }, - // Virtual tracks only: records which component versions were included - component_versions: { type: mongoose.Schema.Types.Mixed, default: undefined }, + // Virtual tracks only: immutable component track ID → tagged version. + component_versions: { + type: Map, + of: { + type: String, + required: true, + validate: validateVersion, + }, + default: undefined, + validate: { + validator: (value) => { + if (value == null) return true; + const keys = value instanceof Map ? value.keys() : Object.keys(value); + return Array.from(keys).every((key) => validateTrackId.validator(key)); + }, + message: 'Component version keys must be valid release track IDs', + }, + }, }; const versionHistoryEntrySchema = new mongoose.Schema(versionHistoryEntryDefinition, { _id: false, diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index a4f66cb6..6349772a 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -82,6 +82,20 @@ function virtualReleaseChanges(previousSnapshot, draftSnapshot) { }; } +/** + * Capture the tagged component versions frozen into a materialized virtual + * draft. Track IDs are stable provenance keys; component names are descriptive + * metadata and may change or collide. + */ +function virtualComponentVersions(snapshot) { + return Object.fromEntries( + (snapshot.composition_resolution?.component_snapshots || []).map((component) => [ + component.track_id, + component.resolved_version, + ]), + ); +} + /** * Build the complete release plan without reading or writing external state. * @@ -180,6 +194,7 @@ function planRelease( ...after, promoted_count: blockingError ? 0 : staged.length, }, + component_versions: isVirtual ? virtualComponentVersions(snapshot) : undefined, }; const plannedSnapshot = blockingError ? null @@ -274,6 +289,7 @@ exports.planRelease = planRelease; exports._private = { memberRevisions, sameRevisions, + virtualComponentVersions, virtualReleaseChanges, }; diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 228a0195..18390a01 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -159,6 +159,145 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version).toBe(preview.body.version); expect(released.body.version_history.at(-1).summary).toMatchObject(preview.body.after); + expect(released.body.version_history.at(-1)).not.toHaveProperty('component_versions'); + }); + + it('records immutable component versions when previewing and releasing a virtual draft', async function () { + const member = (await post('/api/techniques', buildTechnique('Provenance Member'), 201)).body; + const component = await createTrack('Provenance Component'); + await post(`/api/release-tracks/${component.id}/contents`, { + x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], + }); + const firstComponentRelease = await post( + `/api/release-tracks/${component.id}/snapshots/latest/release`, + {}, + ); + expect(firstComponentRelease.body.version).toBe('1.0'); + + const virtual = ( + await post( + '/api/release-tracks/new', + { + name: 'Virtual Provenance', + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: component.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + }, + }, + 201, + ) + ).body; + const materialized = ( + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 201) + ).body; + expect(materialized.composition_resolution.component_snapshots[0]).toMatchObject({ + track_id: component.id, + resolved_version: '1.0', + }); + + // Advance the component after materialization. Virtual release provenance + // must remain tied to the frozen component resolution, not current state. + await post(`/api/release-tracks/${component.id}/contents`, { + x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], + }); + const secondComponentRelease = await post( + `/api/release-tracks/${component.id}/snapshots/latest/release`, + {}, + ); + expect(secondComponentRelease.body.version).toBe('1.1'); + + const releasePath = + `/api/release-tracks/${virtual.id}/snapshots/` + + `${encodeURIComponent(materialized.modified)}/release`; + const preview = await get(`${releasePath}/preview?format=workbench`); + expect(preview.body.version_history.at(-1).component_versions).toEqual({ + [component.id]: '1.0', + }); + + const unchanged = await get( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(materialized.modified)}`, + ); + expect(unchanged.body.version_history).toEqual([]); + + const released = await post(releasePath, {}); + expect(released.body.version_history.at(-1).component_versions).toEqual({ + [component.id]: '1.0', + }); + }); + + it('validates component release provenance at the persistence boundary', async function () { + const track = await createTrack('Provenance Validation', 'virtual'); + const created = new Date(track.modified); + const historyEntry = { + version: '1.0', + tagged_at: new Date(created.getTime() + 1000), + tagged_by: 'system', + snapshot_id: new Date(created.getTime() + 1000), + summary: { members_count: 0, quarantine_count: 0 }, + }; + + await expect( + dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: historyEntry.snapshot_id, + version: '1.0', + version_history: [ + { + ...historyEntry, + component_versions: { [track.id]: 'latest' }, + }, + ], + }), + ).rejects.toMatchObject({ + name: 'DatabaseError', + details: expect.stringContaining('not a valid version'), + }); + + const invalidKeyModified = new Date(created.getTime() + 2000); + await expect( + dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: invalidKeyModified, + version: '1.1', + version_history: [ + { + ...historyEntry, + version: '1.1', + snapshot_id: invalidKeyModified, + component_versions: { 'Component Display Name': '1.0' }, + }, + ], + }), + ).rejects.toMatchObject({ + name: 'DatabaseError', + details: expect.stringContaining('Component version keys must be valid release track IDs'), + }); + + const missingValueModified = new Date(created.getTime() + 3000); + await expect( + dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: missingValueModified, + version: '1.2', + version_history: [ + { + ...historyEntry, + version: '1.2', + snapshot_id: missingValueModified, + component_versions: { [track.id]: null }, + }, + ], + }), + ).rejects.toMatchObject({ + name: 'DatabaseError', + details: expect.stringContaining('is required'), + }); }); it('resolves latest when the release request is handled', async function () { diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 901cfc17..b2329b48 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -577,6 +577,44 @@ Done when: - Component contribution counts add up to the resolved member total. - Quarantine views never show repeated copies of the same exact revision. +## P1 — Model and display immutable virtual release provenance + +### [ ] Type `component_versions` and resolve component display names + +Virtual release history entries now include: + +```ts +component_versions?: Record; +``` + +Each key is an immutable component release-track ID and each value is the +tagged component version frozen in the virtual draft's +`composition_resolution`. The map is present only for virtual releases; +standard release history entries omit it. Component display names are +deliberately not used as keys because names can change or collide. + +The existing `VersionHistoryEntry` interface currently types this property as +`any`. Replace that with `Record`. If the UI presents +provenance to operators, pair each track ID with the matching +`composition_resolution.component_snapshots[].track_name` from the same +released snapshot while retaining the ID as the authoritative identity. + +Do not fetch each component's latest release to construct this display. A +component may have advanced after virtual materialization; the embedded map is +the release's immutable provenance and must remain unchanged. + +Done when: + +- `component_versions` is strongly typed as an optional track-ID-to-version + map. +- Standard release-history fixtures omit the property. +- Virtual workbench preview and committed-release fixtures include the same + map. +- Any user-facing labels resolve names from the released snapshot's embedded + composition metadata and fall back to the track ID. +- Tests prove that a component's newer current release does not replace the + version shown for an older materialized virtual draft. + ## P1 — Separate snapshot and preview output-format types ### [ ] Remove the invalid `snapshot` format and model `summary` correctly diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 337d9c5a..c343f61d 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -45,9 +45,9 @@ completion backlog. ### P1 — Release provenance -- [ ] Populate virtual release `version_history[].component_versions` from the +- [x] Populate virtual release `version_history[].component_versions` from the materialized snapshot's immutable `composition_resolution`. -- [ ] Define and test the provenance shape in Mongoose, OpenAPI, and user and +- [x] Define and test the provenance shape in Mongoose, OpenAPI, and user and developer documentation. ### P2 — Scheduled materialization @@ -351,6 +351,61 @@ Verification result (2026-07-29): component contribution accounting. ``` +### Current implementation slice — Virtual release provenance + +- [x] Add release preview and commit regressions proving that virtual + `version_history[].component_versions` comes from the selected draft's + immutable `composition_resolution`, even if a component is released again + before the virtual draft is tagged. +- [x] Define `component_versions` as an optional object keyed by immutable + component track ID with tagged `MAJOR.MINOR` version values. +- [x] Populate provenance only for virtual release history entries and leave + standard release history unchanged. +- [x] Enforce the provenance value shape at the Mongoose persistence boundary + and describe it in OpenAPI. +- [x] Align user/developer documentation, frontend guidance, Bruno, and + `internalattack` if the response contract requires downstream changes. +- [x] Run focused regression specs, lint, and the complete `npm test` suite. +- [x] Apply logic and performance review checklists, inspect the final diff, + and propose conventional commit messages. + +Verification result (2026-07-29): + +- The focused release-planning and commit spec passes (16), including + workbench preview, in-place release persistence, standard-track omission, + immutable component advancement, and invalid Mongoose key/value cases. +- OpenAPI validation passes (2), backend lint passes, and the required clean + `npm test` run passes (OpenAPI 2, config 21, API 938, middleware 24). +- An earlier complete run encountered unrelated roaming 404s in Attack Objects + pagination and References after 936 API tests passed. The affected specs pass + together in isolation (30). +- `internalattack` returns release preview and commit responses as raw mappings, + so the additive history field requires no Python client change. +- Logic review result: `ROBUST`. Preview and commit both derive provenance from + the selected persisted draft, malformed map keys/values are rejected, and + standard release history remains unchanged. +- Performance review result: `PERFORMANT`. Provenance construction is a linear + in-memory pass over already-loaded component resolution metadata and adds no + database reads, blocking work, or resource lifecycle. +- Proposed REST API commit: + + ```text + fix(release-tracks): record virtual release provenance + + Persist immutable component track versions from the materialized virtual + draft in release history, validate the provenance map, and align API, + documentation, frontend, and regression contracts. + ``` + +- Proposed Bruno commit: + + ```text + docs(release-tracks): document virtual release provenance + + Describe the track-ID-keyed component version map returned by virtual release + previews and commits. + ``` + ### Tracker consolidation - [x] Consolidate the virtual-track completion backlog into this section. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 7790bca1..b5d20975 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -428,8 +428,8 @@ Virtual release tracks compute their contents by aggregating objects from compon tagged_by: "admin@example.com", snapshot_id: "2024-03-01T10:00:00.000Z", // When snapshot was created component_versions: { - "GroupsMonthly": "5.2", - "TechniquesQuarterly": "2.1" + "release-track--groups-monthly": "5.2", + "release-track--techniques-quarterly": "2.1" } } ] @@ -499,6 +499,12 @@ currently validates and persists all three shapes. Automated execution for `object_ref` are resolved or quarantined. Every surviving member is attributed to exactly one deterministic component, so summed `objects_contributed` equals `summary.total_objects` +- Releasing a materialized virtual draft copies each + `composition_resolution.component_snapshots[].resolved_version` into + `version_history[].component_versions`. This is an object keyed by immutable + component `track_id`, not display name. It records the frozen materialization + inputs even when a component has newer releases by the time the virtual draft + is tagged. Standard release history entries omit the field - Composition request objects are strict; unknown composition, component, filter, and deduplication keys return `400 Bad Request` - Selector fields form a discriminated request contract: diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index e3a4ed9b..83efb79e 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -152,6 +152,17 @@ The first virtual release uses zero-valued `before` counts and `previous_release: null`. Workbench and bundle previews render the same frozen planned snapshot, and the commit path tags that snapshot in place. +Virtual release planning also derives +`version_history[].component_versions` directly from the selected draft's +immutable `composition_resolution.component_snapshots`. The property is a +component track ID to tagged `MAJOR.MINOR` version map. It deliberately does +not query the component tracks at preview or commit time: a component can +advance after virtual materialization without changing the provenance of the +already-frozen draft. Standard release history entries omit the virtual-only +property. Mongoose validates every map value with the shared release-version +validator and requires every persisted component resolution to identify its +tagged `resolved_version`. + ### Snapshot history reads Snapshot history is exposed as a nested collection at diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 8e29fbbc..f733326f 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -510,6 +510,24 @@ For virtual tracks, the selected draft must have a non-null until the virtual snapshot creation endpoint materializes it; preview and release return `409 Conflict` before then. +The virtual release response records the materialized component provenance in +`version_history[].component_versions`: + +```json +{ + "component_versions": { + "release-track--groups-monthly": "5.2", + "release-track--techniques-quarterly": "2.1" + } +} +``` + +Keys are immutable component track IDs and values are the tagged versions +stored in the selected draft's `composition_resolution`. The server does not +look up the components' current releases, so advancing a component after +materialization does not rewrite the virtual release's provenance. Standard +release history entries omit `component_versions`. + ### Clone Release Track From Latest Bootstraps a new `release-track` instance from an existing snapshot. @@ -893,6 +911,9 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview `format=workbench` returns the complete would-be persisted snapshot. `format=bundle` returns its publication-ready STIX bundle. Thus “dry run” is not a separate command: it is a release preview with the desired format. +For a materialized virtual draft, the workbench preview includes the same +track-ID-keyed `version_history[].component_versions` map that a successful +release would persist. For a standard track, `before` is the selected draft before staged members are promoted and `after` is the would-be tagged result. For a virtual track, the diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 053e50ba..5ec397f7 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -634,20 +634,28 @@ POST /api/release-tracks/:id/snapshots/:modified/release "tagged_by": "admin@example.com", "snapshot_id": "2024-03-01T10:00:00.000Z", "component_versions": { - "Groups Monthly": "5.2", - "Techniques Quarterly": "2.1" + "release-track--groups-monthly": "5.2", + "release-track--techniques-quarterly": "2.1" } } ] } ``` +`component_versions` is keyed by immutable component track ID. Its values come +from the selected draft's `composition_resolution`, not from the component +tracks' current releases. If a component advances after this virtual draft was +materialized, the virtual release still records the version that actually +produced its frozen contents. Standard release history entries omit this +virtual-only property. + **Business Logic:** 1. Validate snapshot exists and is a draft (version === null) 2. Calculate/validate version number 3. Set version on snapshot (in-place update) -4. Add entry to version_history -5. Snapshot is now immutable +4. Copy resolved component versions into the virtual release-history entry +5. Add entry to version_history +6. Snapshot is now immutable ### 5. Snapshot Export From 0c99b73e9f3f452e83234cf2e3a4c096cb3c3e0a Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:51:58 -0400 Subject: [PATCH 28/55] feat(release-tracks): schedule virtual snapshot materialization Execute persisted cron and date schedules through the existing virtual snapshot lifecycle. Add durable occurrence claims, restart-safe idempotency, automation-run auditing, retry behavior, scheduled snapshot provenance, and aligned API and operations documentation. --- .../definitions/components/release-tracks.yml | 29 ++- app/config/config.js | 5 + .../release-track-snapshot-schema.js | 29 +++ ...virtual-track-schedule-occurrence-model.js | 46 ++++ .../release-track-dynamic.repository.js | 14 + .../release-track-registry.repository.js | 16 ++ ...al-track-schedule-occurrence.repository.js | 118 +++++++++ app/scheduler/virtual-track-snapshots-task.js | 244 ++++++++++++++++++ .../release-tracks/snapshot-service.js | 2 + .../release-tracks/virtual-track-service.js | 19 +- .../virtual-track-snapshots-task.spec.js | 220 ++++++++++++++++ docs/README.md | 1 + docs/admin/configuration.md | 11 +- docs/admin/virtual-track-schedules.md | 50 ++++ docs/developer/FRONTEND_TODO.md | 25 +- docs/developer/TODO.md | 64 ++++- docs/developer/release-tracks/entities.md | 25 +- .../release-tracks/implementation-notes.md | 12 +- docs/developer/task-scheduler.md | 22 +- docs/user/release-tracks/api-reference.md | 11 +- docs/user/release-tracks/virtual-tracks.md | 37 ++- template.env | 8 +- 22 files changed, 950 insertions(+), 58 deletions(-) create mode 100644 app/models/release-tracks/virtual-track-schedule-occurrence-model.js create mode 100644 app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js create mode 100644 app/scheduler/virtual-track-snapshots-task.js create mode 100644 app/tests/scheduler/virtual-track-snapshots-task.spec.js create mode 100644 docs/admin/virtual-track-schedules.md diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 93e29940..eee05059 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -81,6 +81,12 @@ components: one component, while conflicts include only IDs with genuinely different revisions. Each surviving member is attributed to exactly one component in objects_contributed. + scheduled_materialization: + nullable: true + description: | + Server-controlled provenance for a virtual draft created by a + snapshot schedule. Manually created snapshots omit this property. + $ref: '#/components/schemas/scheduled-materialization' config: $ref: '#/components/schemas/track-config' version_history: @@ -535,7 +541,7 @@ components: description: 'When the track metadata was last updated' snapshot_schedule: nullable: true - description: 'Stored snapshot schedule metadata for virtual tracks. Automated execution is not yet implemented.' + description: 'Snapshot creation schedule for virtual tracks' $ref: '#/components/schemas/snapshot-schedule' tagged-release-reference: @@ -585,7 +591,9 @@ components: format: date-time snapshot-schedule: - description: 'Stored schedule metadata for virtual track snapshot creation; automated execution is not yet implemented' + description: | + Virtual snapshot-creation schedule. Cron and explicit dates are + interpreted in UTC and execute only when the global scheduler is enabled. oneOf: - type: object additionalProperties: false @@ -628,3 +636,20 @@ components: type: string format: date-time description: 'Explicit UTC dates for snapshot creation' + + scheduled-materialization: + type: object + description: 'Immutable scheduler occurrence that created a virtual draft' + required: + - schedule_mode + - scheduled_for + properties: + schedule_mode: + type: string + enum: + - cron + - dates + scheduled_for: + type: string + format: date-time + description: 'UTC occurrence timestamp; also serves as the idempotency key' diff --git a/app/config/config.js b/app/config/config.js index ded334f6..4ff7b493 100644 --- a/app/config/config.js +++ b/app/config/config.js @@ -266,6 +266,11 @@ function loadConfig() { default: '0 3 * * *', // daily at 3 AM env: 'VALIDATE_OBJECTS_CRON', }, + virtualTrackSchedulesCron: { + doc: 'Cron pattern for reconciling persisted virtual release-track snapshot schedules.', + default: '* * * * *', // every minute + env: 'VIRTUAL_TRACK_SCHEDULES_CRON', + }, enableScheduler: { format: Boolean, default: true, diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 77e369ec..e88496a6 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -188,6 +188,18 @@ const compositionResolutionSchema = new mongoose.Schema(compositionResolutionDef _id: false, }); +const scheduledMaterializationDefinition = { + schedule_mode: { + type: String, + enum: ['cron', 'dates'], + required: true, + }, + scheduled_for: { type: Date, required: true }, +}; +const scheduledMaterializationSchema = new mongoose.Schema(scheduledMaterializationDefinition, { + _id: false, +}); + // --- Config sub-schemas --- const promotionConflictsDefinition = { @@ -366,6 +378,10 @@ const releaseTrackSnapshotDefinition = { // --- Virtual track composition --- composition: { type: compositionSchema, default: undefined }, composition_resolution: { type: compositionResolutionSchema, default: undefined }, + scheduled_materialization: { + type: scheduledMaterializationSchema, + default: undefined, + }, // --- Shared --- config: { type: configSchema, default: () => ({}) }, @@ -384,6 +400,18 @@ releaseTrackSnapshotSchema.index({ id: 1, modified: -1 }, { unique: true }); // Find the latest tagged version releaseTrackSnapshotSchema.index({ id: 1, version: 1 }); +// A scheduled occurrence may materialize at most one snapshot, including +// after restart recovery or duplicate delivery by multiple scheduler nodes. +releaseTrackSnapshotSchema.index( + { 'scheduled_materialization.scheduled_for': 1 }, + { + unique: true, + partialFilterExpression: { + 'scheduled_materialization.scheduled_for': { $type: 'date' }, + }, + }, +); + // Historical releases-by-object lookup. Draft snapshots are deliberately // excluded because they are numerous, mutable through cloning, and never // eligible for the endpoint. @@ -408,6 +436,7 @@ module.exports = { quarantineEntrySchema, compositionSchema, compositionResolutionSchema, + scheduledMaterializationSchema, configSchema, versionHistoryEntrySchema, }; diff --git a/app/models/release-tracks/virtual-track-schedule-occurrence-model.js b/app/models/release-tracks/virtual-track-schedule-occurrence-model.js new file mode 100644 index 00000000..ce1f9f35 --- /dev/null +++ b/app/models/release-tracks/virtual-track-schedule-occurrence-model.js @@ -0,0 +1,46 @@ +'use strict'; + +const mongoose = require('mongoose'); +const { validateTrackId } = require('../../lib/release-tracks/release-track-validators'); + +const virtualTrackScheduleOccurrenceSchema = new mongoose.Schema( + { + track_id: { + type: String, + required: true, + validate: validateTrackId, + }, + schedule_mode: { + type: String, + enum: ['cron', 'dates'], + required: true, + }, + scheduled_for: { type: Date, required: true }, + status: { + type: String, + enum: ['pending', 'running', 'completed', 'failed', 'skipped'], + required: true, + default: 'pending', + }, + attempt_count: { type: Number, required: true, default: 0 }, + claimed_at: { type: Date, default: null }, + claim_expires_at: { type: Date, default: null }, + next_retry_at: { type: Date, default: null }, + finished_at: { type: Date, default: null }, + snapshot_modified: { type: Date, default: null }, + last_error: { type: mongoose.Schema.Types.Mixed, default: null }, + }, + { + collection: 'virtualTrackScheduleOccurrences', + bufferCommands: false, + }, +); + +virtualTrackScheduleOccurrenceSchema.index({ track_id: 1, scheduled_for: 1 }, { unique: true }); +virtualTrackScheduleOccurrenceSchema.index({ status: 1, next_retry_at: 1 }); +virtualTrackScheduleOccurrenceSchema.index({ status: 1, claim_expires_at: 1 }); + +module.exports = mongoose.model( + 'VirtualTrackScheduleOccurrence', + virtualTrackScheduleOccurrenceSchema, +); diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 6ddb1a98..b761ee15 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -109,6 +109,20 @@ class ReleaseTrackDynamicRepository { } } + async getSnapshotByScheduledMaterialization(trackId, scheduledFor) { + try { + const Model = this._getModel(trackId); + return await Model.findOne({ + id: trackId, + 'scheduled_materialization.scheduled_for': scheduledFor, + }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async getTaggedSnapshotMetadata(trackId) { try { const Model = this._getModel(trackId); diff --git a/app/repository/release-tracks/release-track-registry.repository.js b/app/repository/release-tracks/release-track-registry.repository.js index d3520cd4..e7ef8c02 100644 --- a/app/repository/release-tracks/release-track-registry.repository.js +++ b/app/repository/release-tracks/release-track-registry.repository.js @@ -104,6 +104,22 @@ class ReleaseTrackRegistryRepository { } } + async findScheduledVirtualTracks() { + try { + return await this.model + .find({ + type: 'virtual', + 'snapshot_schedule.mode': { $in: ['cron', 'dates'] }, + }) + .select('track_id name snapshot_schedule') + .sort({ track_id: 1 }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async replaceTaggedReleases(trackId, taggedReleases, latestTaggedVersion) { try { return await this.model diff --git a/app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js b/app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js new file mode 100644 index 00000000..b470f00b --- /dev/null +++ b/app/repository/release-tracks/virtual-track-schedule-occurrence.repository.js @@ -0,0 +1,118 @@ +'use strict'; + +const VirtualTrackScheduleOccurrence = require('../../models/release-tracks/virtual-track-schedule-occurrence-model'); +const { DatabaseError } = require('../../exceptions'); + +class VirtualTrackScheduleOccurrenceRepository { + async register(trackId, scheduleMode, scheduledFor) { + try { + return await VirtualTrackScheduleOccurrence.findOneAndUpdate( + { track_id: trackId, scheduled_for: scheduledFor }, + { + $setOnInsert: { + track_id: trackId, + schedule_mode: scheduleMode, + scheduled_for: scheduledFor, + status: 'pending', + attempt_count: 0, + }, + }, + { upsert: true, new: true, lean: true }, + ).exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async findDue(now) { + try { + return await VirtualTrackScheduleOccurrence.find({ + $or: [ + { status: 'pending' }, + { status: 'failed', next_retry_at: { $lte: now } }, + { status: 'running', claim_expires_at: { $lte: now } }, + ], + }) + .sort({ scheduled_for: 1, track_id: 1 }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async claim(trackId, scheduledFor, now, claimExpiresAt) { + try { + return await VirtualTrackScheduleOccurrence.findOneAndUpdate( + { + track_id: trackId, + scheduled_for: scheduledFor, + $or: [ + { status: 'pending' }, + { status: 'failed', next_retry_at: { $lte: now } }, + { status: 'running', claim_expires_at: { $lte: now } }, + ], + }, + { + $set: { + status: 'running', + claimed_at: now, + claim_expires_at: claimExpiresAt, + next_retry_at: null, + finished_at: null, + last_error: null, + }, + $inc: { attempt_count: 1 }, + }, + { new: true, lean: true }, + ).exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async complete(trackId, scheduledFor, snapshotModified) { + return this._finish(trackId, scheduledFor, { + status: 'completed', + snapshot_modified: snapshotModified, + finished_at: new Date(), + claim_expires_at: null, + next_retry_at: null, + last_error: null, + }); + } + + async fail(trackId, scheduledFor, error, nextRetryAt) { + return this._finish(trackId, scheduledFor, { + status: 'failed', + finished_at: new Date(), + claim_expires_at: null, + next_retry_at: nextRetryAt, + last_error: error, + }); + } + + async skip(trackId, scheduledFor, reason) { + return this._finish(trackId, scheduledFor, { + status: 'skipped', + finished_at: new Date(), + claim_expires_at: null, + next_retry_at: null, + last_error: { message: reason }, + }); + } + + async _finish(trackId, scheduledFor, updates) { + try { + return await VirtualTrackScheduleOccurrence.findOneAndUpdate( + { track_id: trackId, scheduled_for: scheduledFor }, + { $set: updates }, + { new: true, lean: true }, + ).exec(); + } catch (err) { + throw new DatabaseError(err); + } + } +} + +module.exports = new VirtualTrackScheduleOccurrenceRepository(); diff --git a/app/scheduler/virtual-track-snapshots-task.js b/app/scheduler/virtual-track-snapshots-task.js new file mode 100644 index 00000000..ef4eee98 --- /dev/null +++ b/app/scheduler/virtual-track-snapshots-task.js @@ -0,0 +1,244 @@ +'use strict'; + +const mongoose = require('mongoose'); +const schedule = require('node-schedule'); +const config = require('../config/config'); +const logger = require('../lib/logger'); +const { createAutomationRunRecorder, serializeError } = require('../lib/automation-run-recorder'); +const registryRepo = require('../repository/release-tracks/release-track-registry.repository'); +const occurrenceRepo = require('../repository/release-tracks/virtual-track-schedule-occurrence.repository'); +const virtualTrackService = require('../services/release-tracks/virtual-track-service'); + +const TASK_NAME = 'virtual-track-snapshot-materialization'; +const JOB_PREFIX = `${TASK_NAME}:`; +const CLAIM_TTL_MS = 5 * 60 * 1000; +const RETRY_DELAY_MS = 60 * 1000; +const cronJobs = new Map(); + +function scheduledJobName(trackId) { + return `${JOB_PREFIX}${trackId}`; +} + +function isConfiguredOccurrence(track, occurrence) { + const configured = track?.snapshot_schedule; + if (!configured || configured.mode !== occurrence.schedule_mode) return false; + if (configured.mode === 'cron') return true; + + const scheduledTime = new Date(occurrence.scheduled_for).getTime(); + return configured.dates.some((date) => new Date(date).getTime() === scheduledTime); +} + +async function auditAttempt(occurrence, execute) { + const scheduledFor = new Date(occurrence.scheduled_for); + const db = mongoose.connection.getClient().db(); + const recorder = await createAutomationRunRecorder(db, { + automationType: 'scheduler', + name: TASK_NAME, + trigger: { + source: 'snapshot_schedule', + scheduled_for: scheduledFor, + }, + scope: { + track_id: occurrence.track_id, + schedule_mode: occurrence.schedule_mode, + }, + metadata: { + attempt: occurrence.attempt_count, + }, + }); + + try { + const snapshot = await execute(); + await recorder.recordItem({ + status: 'changed', + action: 'materialize_virtual_snapshot', + target: { + kind: 'release-track', + document_id: occurrence.track_id, + }, + details: { + scheduled_for: scheduledFor, + snapshot_modified: snapshot.modified, + members_count: snapshot.members?.length || 0, + quarantine_count: snapshot.quarantine?.length || 0, + }, + }); + await recorder.finish({ + status: 'completed', + counts: { materialized: 1, failed: 0 }, + summary: { + message: `Materialized scheduled virtual snapshot for ${occurrence.track_id}`, + }, + }); + return snapshot; + } catch (err) { + const serialized = serializeError(err); + await recorder.recordItem({ + status: 'failed', + action: 'materialize_virtual_snapshot', + target: { + kind: 'release-track', + document_id: occurrence.track_id, + }, + error: serialized, + details: { scheduled_for: scheduledFor }, + }); + await recorder.finish({ + status: 'failed', + counts: { materialized: 0, failed: 1 }, + errorSummary: serialized, + summary: { + message: `Scheduled virtual snapshot failed for ${occurrence.track_id}`, + }, + }); + throw err; + } +} + +async function executeOccurrence(occurrence, now = new Date()) { + const scheduledFor = new Date(occurrence.scheduled_for); + const claimed = await occurrenceRepo.claim( + occurrence.track_id, + scheduledFor, + now, + new Date(now.getTime() + CLAIM_TTL_MS), + ); + if (!claimed) return null; + + const track = await registryRepo.findByTrackId(claimed.track_id); + if (!isConfiguredOccurrence(track, claimed)) { + await occurrenceRepo.skip( + claimed.track_id, + scheduledFor, + 'Track was deleted or no longer has the schedule that produced this occurrence', + ); + return null; + } + + try { + const snapshot = await auditAttempt(claimed, () => + virtualTrackService.createVirtualSnapshot(claimed.track_id, { + scheduledMaterialization: { + schedule_mode: claimed.schedule_mode, + scheduled_for: scheduledFor, + }, + }), + ); + await occurrenceRepo.complete(claimed.track_id, scheduledFor, snapshot.modified); + return snapshot; + } catch (err) { + await occurrenceRepo.fail( + claimed.track_id, + scheduledFor, + serializeError(err), + new Date(now.getTime() + RETRY_DELAY_MS), + ); + logger.error( + `[${TASK_NAME}] ${claimed.track_id} occurrence ${scheduledFor.toISOString()} failed: ${err.message}`, + ); + return null; + } +} + +async function executeCronOccurrence(trackId, fireDate) { + const occurrence = await occurrenceRepo.register(trackId, 'cron', fireDate); + return executeOccurrence(occurrence); +} + +async function registerCronTrack(track) { + const trackId = track.track_id; + const cronPattern = track.snapshot_schedule.cron; + const existing = cronJobs.get(trackId); + if (existing?.cronPattern === cronPattern) return; + + if (existing) { + schedule.cancelJob(existing.job); + cronJobs.delete(trackId); + } + + const job = schedule.scheduleJob( + scheduledJobName(trackId), + { rule: cronPattern, tz: 'Etc/UTC' }, + async (fireDate) => { + try { + await executeCronOccurrence(trackId, fireDate); + } catch (err) { + logger.error( + `[${TASK_NAME}] Unable to register ${trackId} occurrence ${fireDate.toISOString()}: ${err.message}`, + ); + logger.error(err.stack); + } + }, + ); + + if (!job) { + throw new Error(`Unable to schedule cron expression "${cronPattern}" for ${trackId}`); + } + cronJobs.set(trackId, { cronPattern, job }); +} + +async function reconcileSchedules(now = new Date()) { + const tracks = await registryRepo.findScheduledVirtualTracks(); + const cronTrackIds = new Set(); + + for (const track of tracks) { + if (track.snapshot_schedule.mode === 'cron') { + cronTrackIds.add(track.track_id); + await registerCronTrack(track); + continue; + } + + for (const scheduledFor of track.snapshot_schedule.dates) { + if (new Date(scheduledFor) <= now) { + await occurrenceRepo.register(track.track_id, 'dates', scheduledFor); + } + } + } + + for (const [trackId, registered] of cronJobs) { + if (!cronTrackIds.has(trackId)) { + schedule.cancelJob(registered.job); + cronJobs.delete(trackId); + } + } + + const due = await occurrenceRepo.findDue(now); + for (const occurrence of due) { + await executeOccurrence(occurrence, now); + } + + return { + scheduled_tracks: tracks.length, + due_occurrences: due.length, + }; +} + +function initializeTask() { + const cronPattern = config.scheduler.virtualTrackSchedulesCron; + logger.info(`[${TASK_NAME}] Scheduling reconciliation with cron pattern: ${cronPattern}`); + + schedule.scheduleJob(`${TASK_NAME}:reconcile`, { rule: cronPattern, tz: 'Etc/UTC' }, async () => { + try { + await reconcileSchedules(); + } catch (err) { + logger.error(`[${TASK_NAME}] Reconciliation failed: ${err.message}`); + logger.error(err.stack); + } + }); + + reconcileSchedules().catch((err) => { + logger.error(`[${TASK_NAME}] Startup reconciliation failed: ${err.message}`); + logger.error(err.stack); + }); +} + +if (config.scheduler.enableScheduler) { + initializeTask(); +} + +module.exports = { + executeCronOccurrence, + executeOccurrence, + initializeTask, + reconcileSchedules, +}; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 4ad5f62c..dc58ac95 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -300,6 +300,7 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov const clone = deepClone(sourceSnapshot); clone.modified = new Date(); clone.version = null; // clones are always drafts + delete clone.scheduled_materialization; // Apply overrides if (overrides) { @@ -371,6 +372,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { clone.created = now; clone.created_by_ref = options.userAccountId || sourceSnapshot.created_by_ref; clone.version_history = []; + delete clone.scheduled_materialization; const normalized = tierRevisionInvariant.normalizeSnapshot(clone); diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 4ccb869c..cd9b8c4a 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -25,6 +25,7 @@ const Events = require('../../lib/event-constants'); const logger = require('../../lib/logger'); const { BadRequestError, + DuplicateIdError, TrackNotFoundError, NoTaggedSnapshotsError, InvalidComponentTypeError, @@ -412,6 +413,12 @@ exports.updateComposition = async function updateComposition(trackId, compositio * @returns {Promise} The new snapshot with composition_resolution metadata */ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, options = {}) { + const scheduledFor = options.scheduledMaterialization?.scheduled_for; + if (scheduledFor) { + const existing = await dynamicRepo.getSnapshotByScheduledMaterialization(trackId, scheduledFor); + if (existing) return existing; + } + const source = await snapshotService.getLatestSnapshot(trackId); assertVirtualTrack(source); @@ -437,13 +444,23 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op members, quarantine: quarantined, composition_resolution: compositionResolution, + scheduled_materialization: options.scheduledMaterialization, }; if (options.description !== undefined) { overrides.description = options.description; } - const snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides); + let snapshot; + try { + snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides); + } catch (err) { + if (!scheduledFor || !(err instanceof DuplicateIdError)) throw err; + + const existing = await dynamicRepo.getSnapshotByScheduledMaterialization(trackId, scheduledFor); + if (!existing) throw err; + snapshot = existing; + } logger.verbose( `VirtualTrackService: Created virtual snapshot for track "${trackId}" ` + diff --git a/app/tests/scheduler/virtual-track-snapshots-task.spec.js b/app/tests/scheduler/virtual-track-snapshots-task.spec.js new file mode 100644 index 00000000..2e33c389 --- /dev/null +++ b/app/tests/scheduler/virtual-track-snapshots-task.spec.js @@ -0,0 +1,220 @@ +'use strict'; + +const { expect } = require('expect'); +const mongoose = require('mongoose'); +const schedule = require('node-schedule'); + +const config = require('../../config/config'); +const database = require('../../lib/database-in-memory'); +const databaseConfiguration = require('../../lib/database-configuration'); +const ReleaseTrackRegistry = require('../../models/release-tracks/release-track-registry-model'); +const VirtualTrackScheduleOccurrence = require('../../models/release-tracks/virtual-track-schedule-occurrence-model'); +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); +const releaseTracksService = require('../../services/release-tracks/release-tracks-service'); + +describe('Scheduled virtual release-track materialization', function () { + let task; + let sequence = 0; + + before(async function () { + config.scheduler.enableScheduler = false; + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + task = require('../../scheduler/virtual-track-snapshots-task'); + }); + + after(async function () { + await schedule.gracefulShutdown(); + await database.closeConnection(); + }); + + async function createComponent({ released = true } = {}) { + sequence += 1; + const component = await releaseTracksService.createTrack({ + name: `Scheduled Component ${sequence}`, + type: 'standard', + }); + if (released) { + await releaseTracksService.releaseLatest(component.id, { + version: '1.0', + userAccountId: 'scheduler-test', + }); + } + return component; + } + + async function createVirtual(componentId, snapshotSchedule) { + sequence += 1; + return releaseTracksService.createTrack({ + name: `Scheduled Virtual ${sequence}`, + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: componentId, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + }, + snapshot_schedule: snapshotSchedule, + }); + } + + async function snapshotCount(trackId) { + return (await dynamicRepo.getAllSnapshots(trackId)).pagination.total; + } + + it('recovers missed dates exactly once and records the automation run', async function () { + const component = await createComponent(); + const scheduledFor = new Date('2026-01-15T12:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [scheduledFor.toISOString()], + }); + const now = new Date('2026-01-15T12:05:00.000Z'); + + await task.reconcileSchedules(now); + + expect(await snapshotCount(virtual.id)).toBe(2); + const materialized = await dynamicRepo.getSnapshotByScheduledMaterialization( + virtual.id, + scheduledFor, + ); + expect(materialized).toMatchObject({ + type: 'virtual', + scheduled_materialization: { + schedule_mode: 'dates', + scheduled_for: scheduledFor, + }, + }); + + const occurrence = await VirtualTrackScheduleOccurrence.findOne({ + track_id: virtual.id, + scheduled_for: scheduledFor, + }) + .lean() + .exec(); + expect(occurrence).toMatchObject({ + status: 'completed', + attempt_count: 1, + snapshot_modified: materialized.modified, + }); + + const automationRun = await mongoose.connection + .getClient() + .db() + .collection('automationRuns') + .findOne({ 'scope.track_id': virtual.id }); + expect(automationRun).toMatchObject({ + automation_type: 'scheduler', + name: 'virtual-track-snapshot-materialization', + status: 'completed', + counts: { materialized: 1, failed: 0 }, + }); + + await task.reconcileSchedules(new Date('2026-01-15T12:10:00.000Z')); + expect(await snapshotCount(virtual.id)).toBe(2); + expect( + await mongoose.connection + .getClient() + .db() + .collection('automationRuns') + .countDocuments({ 'scope.track_id': virtual.id }), + ).toBe(1); + + const manual = await releaseTracksService.createVirtualSnapshot(virtual.id); + expect(manual).not.toHaveProperty('scheduled_materialization'); + expect(await snapshotCount(virtual.id)).toBe(3); + }); + + it('materializes duplicate cron delivery once', async function () { + const component = await createComponent(); + const virtual = await createVirtual(component.id, { + mode: 'cron', + cron: '0 0 1 1,7 *', + }); + const scheduledFor = new Date('2026-07-01T00:00:00.000Z'); + + await Promise.all([ + task.executeCronOccurrence(virtual.id, scheduledFor), + task.executeCronOccurrence(virtual.id, scheduledFor), + ]); + + expect(await snapshotCount(virtual.id)).toBe(2); + expect( + await VirtualTrackScheduleOccurrence.countDocuments({ + track_id: virtual.id, + scheduled_for: scheduledFor, + status: 'completed', + }), + ).toBe(1); + }); + + it('audits component failures and retries them during reconciliation', async function () { + const component = await createComponent({ released: false }); + const scheduledFor = new Date('2026-02-01T00:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [scheduledFor.toISOString()], + }); + const firstAttempt = new Date('2026-02-01T00:01:00.000Z'); + + await task.reconcileSchedules(firstAttempt); + + let occurrence = await VirtualTrackScheduleOccurrence.findOne({ + track_id: virtual.id, + scheduled_for: scheduledFor, + }) + .lean() + .exec(); + expect(occurrence).toMatchObject({ + status: 'failed', + attempt_count: 1, + }); + expect(occurrence.last_error.message).toContain('has no tagged snapshots'); + expect(await snapshotCount(virtual.id)).toBe(1); + + await releaseTracksService.releaseLatest(component.id, { + version: '1.0', + userAccountId: 'scheduler-test', + }); + await task.reconcileSchedules(new Date(firstAttempt.getTime() + 60 * 1000)); + + occurrence = await VirtualTrackScheduleOccurrence.findOne({ + track_id: virtual.id, + scheduled_for: scheduledFor, + }) + .lean() + .exec(); + expect(occurrence).toMatchObject({ + status: 'completed', + attempt_count: 2, + }); + expect(await snapshotCount(virtual.id)).toBe(2); + + const runs = await mongoose.connection + .getClient() + .db() + .collection('automationRuns') + .find({ 'scope.track_id': virtual.id }) + .sort({ started_at: 1 }) + .toArray(); + expect(runs.map((run) => run.status)).toEqual(['failed', 'completed']); + }); + + it('does not schedule or materialize manual tracks', async function () { + const component = await createComponent(); + const virtual = await createVirtual(component.id, { mode: 'manual' }); + + await task.reconcileSchedules(new Date('2027-01-01T00:00:00.000Z')); + + expect(await snapshotCount(virtual.id)).toBe(1); + expect(await VirtualTrackScheduleOccurrence.countDocuments({ track_id: virtual.id })).toBe(0); + expect( + await ReleaseTrackRegistry.findOne({ track_id: virtual.id }).lean().exec(), + ).toMatchObject({ + snapshot_schedule: { mode: 'manual' }, + }); + }); +}); diff --git a/docs/README.md b/docs/README.md index 1ceae506..bf0ca6bf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,6 +57,7 @@ Configuration, deployment, and identity provider setup. - [Configuration](admin/configuration.md): Complete configuration guide (environment variables, JSON files) - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records +- [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability ### Authentication diff --git a/docs/admin/configuration.md b/docs/admin/configuration.md index ba47748b..eee484d4 100644 --- a/docs/admin/configuration.md +++ b/docs/admin/configuration.md @@ -516,22 +516,23 @@ See sample: [multiple-apikey-services.json](../resources/sample-configurations/m Background job scheduler configuration. -| Option | Environment Variable | JSON Path | Type | Default | Description | -|----------------|----------------------------|----------------------------------|---------|---------|--------------------------------------| -| Enable | `ENABLE_SCHEDULER` | `scheduler.enableScheduler` | boolean | `true` | Enable background job scheduler | -| Check Interval | `CHECK_WORKBENCH_INTERVAL` | `scheduler.checkWorkbenchInterval` | integer | `10` | Scheduler check interval in seconds | +| Option | Environment Variable | JSON Path | Type | Default | Description | +|---|---|---|---|---|---| +| Enable | `ENABLE_SCHEDULER` | `scheduler.enableScheduler` | boolean | `true` | Enable background job scheduler | +| Virtual-track reconciliation | `VIRTUAL_TRACK_SCHEDULES_CRON` | `scheduler.virtualTrackSchedulesCron` | string | `* * * * *` | Discover and retry persisted virtual snapshot schedules | **Scheduler Functions:** - Checks for collection index updates - Downloads collection bundles from remote URLs - Processes subscription update policies +- Materializes virtual release-track snapshots from cron and date schedules **Example:** ```bash ENABLE_SCHEDULER=true -CHECK_WORKBENCH_INTERVAL=30 +VIRTUAL_TRACK_SCHEDULES_CRON="* * * * *" ``` ### Validation diff --git a/docs/admin/virtual-track-schedules.md b/docs/admin/virtual-track-schedules.md new file mode 100644 index 00000000..627366ea --- /dev/null +++ b/docs/admin/virtual-track-schedules.md @@ -0,0 +1,50 @@ +# Virtual Release-Track Schedules + +Virtual release tracks can materialize draft snapshots explicitly or through +their persisted `snapshot_schedule`. Scheduled execution uses the same +composition-resolution and snapshot-persistence services as the explicit +virtual snapshot creation endpoint. + +## Activation and timing + +`ENABLE_SCHEDULER=true` activates all Workbench scheduler tasks, including +virtual-track materialization. `VIRTUAL_TRACK_SCHEDULES_CRON` controls how +often the server reconciles persisted schedules; it defaults to once per +minute. + +All five-field cron expressions and explicit dates are interpreted in UTC. +Cron jobs fire only while a scheduler instance is running. They do not +backfill occurrences missed during downtime. Date schedules are durable: +every configured timestamp at or before reconciliation is registered and +processed after startup. + +`manual` schedules register no executable work. Operators must call +`POST /api/release-tracks/:id/virtual/snapshots/create`. + +## Idempotency and multiple instances + +The `virtualTrackScheduleOccurrences` collection stores one durable occurrence +per track and UTC timestamp. Workers atomically claim pending or retryable +occurrences. The resulting snapshot also records +`scheduled_materialization.scheduled_for` under a unique track-local index. +Together, these controls prevent duplicate drafts across restarts, retry +delivery, and multiple scheduler-enabled API instances. + +## Failures and retries + +An occurrence commonly fails when a component resolution has no matching +tagged snapshot. The occurrence remains `failed` and becomes retryable after +one minute. The reconciliation task retries it automatically; no schedule +resubmission is required. Permanent configuration errors continue to retry +until an operator corrects the component release state or removes the track. + +Every attempt creates an `automationRuns` record with: + +- `automation_type: "scheduler"` +- `name: "virtual-track-snapshot-materialization"` +- `scope.track_id` and `scope.schedule_mode` +- `trigger.scheduled_for` +- terminal counts and an item-level error or created snapshot timestamp + +See [Automation Run Audit Trail](automation-runs.md) for queries and +operational inspection patterns. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index b2329b48..f57ad0ac 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -510,9 +510,23 @@ therefore submits only `{ mode: 'dates' }`, which the server rejects. The cron control is also not conditionally required, allowing `{ mode: 'cron' }` to be submitted. -Schedule configuration is metadata only for now. The UI must not imply that -automatic creation is active until the P2 backend scheduler integration is -implemented. +Automatic creation is now active when the backend scheduler is enabled. +Explain that cron and dates use UTC, cron occurrences are not backfilled after +downtime, and due dates are recovered after restart. A component-resolution +failure is retried by the backend; the UI does not need to resubmit the +schedule. + +Scheduled virtual drafts include read-only provenance: + +```ts +scheduled_materialization?: { + schedule_mode: 'cron' | 'dates'; + scheduled_for: string; +}; +``` + +Use it to identify scheduled drafts where useful, but never include it in +create or update payloads. Done when: @@ -522,7 +536,10 @@ Done when: - Selecting manual clears both selector fields. - Standard-track creation never sends schedule metadata. - Tests cover all three modes and mode switching. -- User-facing copy says scheduled execution is not yet active. +- User-facing copy explains UTC execution and the difference between cron and + restart-recoverable dates. +- Scheduled drafts tolerate and preserve the read-only + `scheduled_materialization` response property. ## P1 — Align virtual component object-type filters diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index c343f61d..5a5b132c 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -52,25 +52,71 @@ completion backlog. ### P2 — Scheduled materialization -- [ ] Connect virtual `snapshot_schedule` metadata to the existing task +- [x] Connect virtual `snapshot_schedule` metadata to the existing task scheduler. This is required for virtual-track completion, not an optional future enhancement. -- [ ] Implement `cron` execution so each matching schedule occurrence +- [x] Implement `cron` execution so each matching schedule occurrence materializes a new virtual draft through the same lifecycle and validation used by `POST /api/release-tracks/:id/virtual/snapshots/create`. -- [ ] Implement `dates` execution so every configured timestamp materializes +- [x] Implement `dates` execution so every configured timestamp materializes exactly one virtual draft, including deterministic handling for restart recovery, missed timestamps, and duplicate-delivery prevention. -- [ ] Preserve `manual` semantics: store no executable schedule and create +- [x] Preserve `manual` semantics: store no executable schedule and create drafts only through the explicit virtual snapshot-creation endpoint. -- [ ] Define failure behavior when a component has no matching tagged +- [x] Define failure behavior when a component has no matching tagged snapshot, including automation-run audit records and retry policy. -- [ ] Add scheduler integration tests for both `cron` and `dates`, including +- [x] Add scheduler integration tests for both `cron` and `dates`, including successful execution, restart recovery, idempotency, component-resolution failure, and retry behavior. -- [ ] Add operational documentation covering scheduler activation, UTC +- [x] Add operational documentation covering scheduler activation, UTC interpretation, observability, failures, and retries. +### Current implementation slice — Scheduled virtual materialization + +- [x] Add a scheduler reconciliation task for persisted virtual-track + `cron` and `dates` schedules while preserving explicit-only `manual` mode. +- [x] Persist schedule occurrences and claim them atomically so multiple + scheduler instances cannot concurrently process the same occurrence. +- [x] Make snapshot persistence idempotent by recording the scheduled + occurrence on the resulting virtual draft. +- [x] Recover missed `dates` occurrences and failed `cron` or `dates` + occurrences during reconciliation. +- [x] Record every materialization attempt in the automation-run audit trail. +- [x] Add scheduler integration coverage for success, restart recovery, + duplicate delivery, component failure, and retry. +- [x] Update OpenAPI, user/developer/operations documentation, frontend + guidance, and Bruno. +- [x] Run focused scheduler tests, lint, and the complete `npm test` suite. +- [x] Review the final diff and propose conventional commit messages. + +Verification result (2026-07-29): + +- The focused scheduler integration spec passes (4), OpenAPI validation + passes (2), and backend lint passes. +- The first complete run encountered five unrelated roaming failures after + 933 API tests passed. Each affected spec passed in isolation. +- The required clean `npm test` rerun passes (OpenAPI 2, config 21, API 938, + middleware 24). +- Proposed REST API commit: + + ```text + feat(release-tracks): schedule virtual snapshot materialization + + Execute persisted cron and date schedules through the existing virtual + snapshot lifecycle. Add durable occurrence claims, restart-safe + idempotency, automation-run auditing, retry behavior, scheduled snapshot + provenance, and aligned API and operations documentation. + ``` + +- Proposed companion Bruno commit: + + ```text + docs(release-tracks): document scheduled materialization + + Describe UTC cron and date execution, restart recovery, idempotency, + and retry behavior for virtual snapshot schedules. + ``` + ### P2 — Contract decisions - [ ] Decide whether virtual tracks can compose virtual tracks. The @@ -97,8 +143,8 @@ completion backlog. or implement the documented `by_type`, `by_tier`, and native statistics. - [ ] Align documented error envelopes with centralized error-handler output. - [x] Include required `priority` values in every composition example. -- [ ] Clearly distinguish configured composition from a materialized draft and - describe scheduled behavior as unavailable until scheduler execution exists. +- [x] Clearly distinguish configured composition from a materialized draft and + document scheduler activation, timing, recovery, and retry behavior. ### Verified complete diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index b5d20975..2cdd7c3b 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -408,8 +408,7 @@ Virtual release tracks compute their contents by aggregating objects from compon } }, - // Optional schedule metadata. Choose exactly one mode-specific shape. - // This example uses cron; automated execution is a required P2 capability. + // Optional schedule. Choose exactly one mode-specific shape. snapshot_schedule: { mode: "cron", cron: "0 0 1 1,7 *" // Jan 1 and July 1 at midnight UTC @@ -455,9 +454,20 @@ The three valid `snapshot_schedule` shapes are: } ``` -These are alternatives, not fields to combine in one schedule. The API -currently validates and persists all three shapes. Automated execution for -`cron` and `dates` is not implemented yet and is a required P2 deliverable. +These are alternatives, not fields to combine in one schedule. `manual` +persists no executable work. A scheduler reconciliation task registers UTC +cron jobs and durable due-date occurrences. Each scheduled draft records: + +```javascript +scheduled_materialization: { + schedule_mode: "cron", // "cron" | "dates" + scheduled_for: "2027-01-01T00:00:00.000Z" +} +``` + +The track-local unique index on `scheduled_for`, together with the durable +`virtualTrackScheduleOccurrences` claim record, makes duplicate delivery and +restart recovery idempotent. Failed occurrences remain retryable. **Key Differences from Standard Tracks:** @@ -486,9 +496,8 @@ currently validates and persists all three shapes. Automated execution for - Snapshot schedules are strict and mode-discriminated: `manual` accepts only `mode`, `cron` requires only a five-field `cron` expression, and `dates` requires only a nonempty `dates` array -- Standard tracks reject `snapshot_schedule`; schedules are stored as virtual - registry metadata. Automated `cron` and `dates` execution is required but - remains pending until scheduler integration exists +- Standard tracks reject `snapshot_schedule`; virtual `cron` and `dates` + schedules execute through the global scheduler - `filters.object_types` uses the canonical Workbench STIX type names from `app/lib/types.js`. When present, it must be nonempty and duplicate-free; omit it to include every object type. Filtering reads the type prefix from diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 83efb79e..ce52f092 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -94,8 +94,16 @@ controller and service boundaries. `manual` has no selector field, `cron` requires a five-field cron expression, and `dates` requires a nonempty array of ISO timestamps. Standard-track creation rejects `snapshot_schedule` instead of silently dropping it. Mongoose repeats the mode and track-type invariants for -direct persistence callers. Schedule configuration remains registry metadata; -P2 scheduler execution is not implemented. +direct persistence callers. + +The virtual snapshot scheduler reconciles persisted schedules at startup and +on `VIRTUAL_TRACK_SCHEDULES_CRON`. Cron jobs use `Etc/UTC`; explicit dates at +or before the reconciliation time become durable occurrences. Atomic +occurrence claims prevent concurrent workers from processing the same run, +and a unique scheduled-materialization index on each track collection prevents +duplicate snapshots after restarts or duplicate delivery. Failures are +recorded in the automation-run audit trail and retried at the next eligible +reconciliation. Component `filters.object_types` values are constrained to the canonical Workbench STIX vocabulary exported by `app/lib/types.js`. The request schema diff --git a/docs/developer/task-scheduler.md b/docs/developer/task-scheduler.md index 7a86ecf0..5a421054 100644 --- a/docs/developer/task-scheduler.md +++ b/docs/developer/task-scheduler.md @@ -38,6 +38,26 @@ if (config.scheduler.enableScheduler) { // <-- make sure to condition the task t - Future tasks must follow a similar pattern: - Add the task file +## Persisted virtual release-track schedules + +`virtual-track-snapshots-task.js` is different from the static maintenance +tasks because each virtual track supplies its own schedule. A global +reconciliation job runs on `VIRTUAL_TRACK_SCHEDULES_CRON` and: + +1. registers or refreshes one UTC cron job per cron-configured virtual track; +2. turns every due explicit date into a durable schedule occurrence; +3. atomically claims pending, failed, or stale occurrences; and +4. retries failures after their retry timestamp. + +`virtualTrackScheduleOccurrences` is the durable delivery and retry ledger. +The materialized snapshot also stores the occurrence timestamp under a unique +track-local index. The ledger prevents concurrent workers from doing the same +work, while the snapshot index is the final idempotency guard after crashes or +duplicate delivery. + +Do not put release-track composition logic in the scheduler task. It delegates +to `virtual-track-service`, which is also used by the explicit HTTP operation. + ## TODO - [ ] Add robust documentation to `USAGE.md` explaining how task scheduling works and how to create new tasks @@ -45,4 +65,4 @@ if (config.scheduler.enableScheduler) { // <-- make sure to condition the task t - [ ] There is another task called `check-wip-attack-ids-task.js` that should probably be deleted - It was created with the goal of restricting ATT&CK IDs to only exist on non-WIP objects - That conversation is sort of out of scope - - I think we're going to move away from this approach and that the task will probably be moot \ No newline at end of file + - I think we're going to move away from this approach and that the task will probably be moot diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index f733326f..e3a336e7 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1145,8 +1145,8 @@ When present, the array must contain at least one unique value. Omit it to include all object types. Type filtering preserves each member revision pinned by the resolved component snapshot. -`snapshot_schedule` is stored as metadata only; automated execution is not -yet implemented. Its shape depends on `mode`: +`snapshot_schedule` controls virtual draft creation when the server scheduler +is enabled. Its shape depends on `mode`: - `manual` accepts only `{ "mode": "manual" }`; - `cron` requires a five-field `cron` expression and rejects `dates`; @@ -1155,6 +1155,13 @@ yet implemented. Its shape depends on `mode`: Unknown schedule properties return `400 Bad Request`. Standard tracks also reject `snapshot_schedule` rather than silently ignoring it. +Cron expressions and explicit dates are interpreted in UTC. Cron occurrences +run while the scheduler is active; they are not backfilled after downtime. +Every due date is recovered after restart and creates exactly one draft. +Failed cron and date occurrences are retried by the scheduler. Scheduled +drafts include a server-controlled `scheduled_materialization` object with +`schedule_mode` and `scheduled_for`; manual drafts omit it. + Composition, component, filter, and deduplication objects are strict. Unknown keys, including the incorrect singular `filters.domain`, return `400 Bad Request`. Component selectors are also strategy-specific: diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 5ec397f7..f3e2793c 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -536,10 +536,6 @@ snapshot_schedule: { } ``` -The configuration is currently persisted as registry metadata only. No -release-track scheduler consumes it yet, so `cron` and `dates` schedules do -not create snapshots automatically. - Schedule payloads are strict and mode-specific: - `manual` accepts only `{ mode: "manual" }`. @@ -549,25 +545,24 @@ Schedule payloads are strict and mode-specific: Unknown schedule fields return `400 Bad Request`. Standard tracks do not support `snapshot_schedule`. -**Planned scheduler integration:** -```javascript -scheduler.register({ - type: "virtual-track-snapshot", - trackId: "release-track--uuid-virtual", - schedule: "0 0 1 1,7 *", - handler: async (trackId) => { - await virtualTrackService.createSnapshot(trackId, { - description: `Scheduled snapshot ${new Date().toISOString()}` - }); +The global scheduler must be enabled. Five-field cron expressions and dates +are interpreted in UTC. Each cron occurrence creates a draft while the server +is running; missed cron occurrences are not backfilled. Due dates are durable: +the scheduler recovers them after a restart and persists exactly one draft per +configured timestamp. A failed occurrence is audited and retried once per +scheduler reconciliation interval. - // Optionally notify team - await notificationService.send({ - to: "enterprise-team@example.com", - subject: "Enterprise ATT&CK snapshot created", - body: "A new draft snapshot is ready for review and tagging" - }); +Scheduled drafts follow the same composition resolution, deduplication, +validation, and persistence path as +`POST /api/release-tracks/:id/virtual/snapshots/create`. They also include: + +```json +{ + "scheduled_materialization": { + "schedule_mode": "cron", + "scheduled_for": "2027-01-01T00:00:00.000Z" } -}); +} ``` ### 2. Snapshot Review diff --git a/template.env b/template.env index bc5c8f75..8dfcc8c7 100644 --- a/template.env +++ b/template.env @@ -67,9 +67,11 @@ DATABASE_URL= # Default: true #ENABLE_SCHEDULER=true -# CHECK_WORKBENCH_INTERVAL (int, seconds) - Scheduler start interval -# Default: 10 -#CHECK_WORKBENCH_INTERVAL=10 +# VIRTUAL_TRACK_SCHEDULES_CRON (string) - Reconcile virtual-track schedules +# Discovers new cron schedules, recovers due dates, and retries failed runs. +# Standard 5-field cron syntax, interpreted in UTC. +# Default: * * * * * (every minute) +#VIRTUAL_TRACK_SCHEDULES_CRON=* * * * * # Validation # VALIDATE_WITH_ADM_SCHEMAS (bool) - Validate POST/PUT bodies against the ATT&CK Data Model From 39d8e944701b58e5b0e9aeee4898276d859a74f9 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:00:46 -0400 Subject: [PATCH 29/55] fix(release-tracks): freeze dynamic staged revisions Preserve latest selectors through candidate and staged workflow state, resolve them during standard release planning, and persist exact immutable members. Align virtual materialization, enrichment, exports, backrefs, schemas, tests, clients, and documentation with the deterministic membership boundary. --- .../definitions/components/release-tracks.yml | 39 +++- .../paths/release-tracks-paths.yml | 73 +++++-- app/lib/release-tracks/backref-reconciler.js | 15 +- app/lib/release-tracks/conflict-resolution.js | 7 +- .../release-tracks/release-track-schemas.js | 9 +- app/lib/release-tracks/revision-reference.js | 85 ++++++++ .../release-tracks/tier-revision-invariant.js | 5 +- .../release-track-snapshot-schema.js | 20 +- app/services/release-tracks/export-service.js | 14 +- .../release-tracks/member-sync-service.js | 54 ++++- .../release-tracks/release-tracks-service.js | 26 ++- .../release-tracks/snapshot-service.js | 41 +++- .../release-tracks/standard-track-service.js | 24 ++- .../release-tracks/versioning-service.js | 22 +- .../release-tracks/virtual-track-service.js | 54 ++++- .../release-tracks-backrefs.spec.js | 63 +++++- .../release-tracks-change-capture.spec.js | 8 +- .../release-tracks-release.spec.js | 150 ++++++++++++- .../virtual-composition-validation.spec.js | 32 ++- .../virtual-determinism.spec.js | 201 +++++++++++++++++ docs/developer/FRONTEND_TODO.md | 93 +++++++- docs/developer/TODO.md | 176 +++++++++++++-- .../release-tracks/backref-reconciliation.md | 18 +- .../developer/release-tracks/bundle-export.md | 56 +++-- docs/developer/release-tracks/entities.md | 40 ++-- .../release-tracks/implementation-notes.md | 60 +++++- .../release-tracks/member-sync-strategies.md | 109 +++++++--- docs/user/release-tracks/api-reference.md | 109 +++++++--- docs/user/release-tracks/object-backrefs.md | 32 +-- docs/user/release-tracks/output-formats.md | 7 + docs/user/release-tracks/release-workflow.md | 39 ++-- docs/user/release-tracks/summary.md | 32 ++- docs/user/release-tracks/terminology.md | 29 ++- docs/user/release-tracks/versioning.md | 15 +- docs/user/release-tracks/virtual-tracks.md | 202 +++++------------- 35 files changed, 1554 insertions(+), 405 deletions(-) create mode 100644 app/lib/release-tracks/revision-reference.js create mode 100644 app/tests/api/release-tracks/virtual-determinism.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index eee05059..243ef13d 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -165,18 +165,14 @@ components: type: integer minimum: 0 - tier-entry: + tier-entry-base: type: object - description: 'A reference to a specific version of a STIX object' + description: 'Shared fields for a release-track object reference' properties: object_ref: type: string description: 'STIX ID of the object' example: 'attack-pattern--12345678-1234-1234-1234-123456789012' - object_modified: - type: string - format: date-time - description: 'Version pin: the modified timestamp of this object version' attack_id: type: string description: 'ATT&CK ID, if found' @@ -204,11 +200,30 @@ components: type: string description: 'Display name, or username if display name is missing' + tier-entry: + allOf: + - $ref: '#/components/schemas/tier-entry-base' + - type: object + description: 'An immutable reference to a specific STIX object revision' + properties: + object_modified: + type: string + format: date-time + description: 'Exact modified timestamp of this object revision' + candidate-entry: allOf: - - $ref: '#/components/schemas/tier-entry' + - $ref: '#/components/schemas/tier-entry-base' - type: object properties: + object_modified: + description: 'An exact revision timestamp or the dynamic selector "latest"' + oneOf: + - type: string + format: date-time + - type: string + enum: + - latest object_status: type: string enum: @@ -227,9 +242,17 @@ components: staged-entry: allOf: - - $ref: '#/components/schemas/tier-entry' + - $ref: '#/components/schemas/tier-entry-base' - type: object properties: + object_modified: + description: 'An exact revision timestamp or the dynamic selector "latest"' + oneOf: + - type: string + format: date-time + - type: string + enum: + - latest object_status: type: string enum: diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 9571c82c..9b1c4c64 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -227,9 +227,10 @@ paths: objects are strict, and component selectors must match their resolution_strategy. Component IDs and priorities must be unique, every priority is required, and referenced components must already - exist as standard tracks. Virtual snapshot schedules are strict by - mode: manual accepts no selector, cron requires cron, and dates - requires at least one date. Standard tracks reject snapshot_schedule. + exist as standard tracks; virtual-track nesting and native members are + unsupported. Virtual snapshot schedules are strict by mode: manual + accepts no selector, cron requires cron, and dates requires at least + one date. Standard tracks reject snapshot_schedule. tags: - 'Release Tracks' # Request body validation moved to Zod in controller @@ -333,6 +334,10 @@ paths: Replace the members tier of a standard track with new contents (x_mitre_contents format). Virtual membership can only be produced by POST /api/release-tracks/{id}/virtual/snapshots/create. + obj_modified may be an exact ISO timestamp or the request-time + shorthand "latest"; the server resolves "latest" to the actual latest + stix.modified timestamp before persistence. Snapshot members always + store exact revision pins. Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. Creates a new snapshot clone. @@ -376,10 +381,12 @@ paths: operationId: 'release-tracks-release-latest' description: | Immutably tag the latest snapshot with a version. Standard tracks - promote staged entries to members. The `latest` selector is resolved - when the request is handled. Supply either `increment` (`major` or - `minor`) or an explicit `version` in `MAJOR.MINOR` form, but never - both. Omitting both defaults to a minor increment. + promote staged entries to members. Any staged `object_modified: + "latest"` selector is resolved to the object's actual latest + `stix.modified` timestamp during release planning; tagged members + always contain exact revision timestamps. Supply either `increment` + (`major` or `minor`) or an explicit `version` in `MAJOR.MINOR` form, + but never both. Omitting both defaults to a minor increment. tags: - 'Release Tracks' parameters: @@ -415,7 +422,8 @@ paths: Plan without persisting. Summary is the default; workbench and bundle render the complete would-be release. `increment` and `version` are mutually exclusive; omitting both defaults to a minor increment. - Standard summaries show staged-to-members promotion. Virtual summaries + Standard preview plans resolve dynamic staged references exactly as a + commit would and show staged-to-members promotion. Virtual summaries compare the persisted draft with its chronologically preceding tagged release; composition is never recomputed. A virtual draft whose composition has not been materialized returns 409. @@ -530,8 +538,12 @@ paths: operationId: 'release-tracks-candidates-add' description: | Add one or more objects to the candidates tier. - If modified is omitted or 'latest', resolves to the latest version of the object. - If that exact revision is already pinned in any tier, the add is idempotently skipped. + If modified is omitted or 'latest', persist a dynamic selector that + follows the latest object revision until release. An explicit ISO + timestamp remains an exact pin. Dynamic selectors are preserved when + promoted to staged and resolved only when staged content is released + into immutable members. + If that same selector is already pinned in any tier, the add is idempotently skipped. Different revisions of the same object may occupy different tiers. If auto_promote is enabled and candidates meet the threshold, they are auto-promoted to staged. Request body validated via Zod: { object_refs: Array } @@ -554,7 +566,7 @@ paths: description: | Transition candidates from one workflow status to another (forward-only). If auto_promote is enabled and candidates meet the threshold after transition, they are auto-promoted to staged. - Tier changes retain an exact revision in only one tier and repair legacy cross-tier duplicates. + Tier changes retain a selector in only one tier and repair legacy cross-tier duplicates. `from` also accepts the server-assigned `modified-in-place` status; `to` accepts only the user-settable statuses (work-in-progress, awaiting-review, reviewed). Request body validated via Zod: { from, to, object_refs? } @@ -623,7 +635,8 @@ paths: operationId: 'release-tracks-candidates-update-version' description: | Change which version of an object is being tracked in the candidates tier. - If the new pin exactly matches another tier, the authoritative existing tier is retained. + old_modified and new_modified may be exact ISO timestamps or "latest". + If the new selector exactly matches another tier, the authoritative existing tier is retained. Request body validated via Zod: { old_modified, new_modified } tags: - 'Release Tracks' @@ -678,7 +691,9 @@ paths: operationId: 'release-tracks-staged-demote' description: | Move objects from staged tier back to candidates tier. - Applies into_candidates conflicts only to different revisions; exact revisions remain in one tier. + The modified selector may be an exact ISO timestamp or "latest". + Applies into_candidates conflicts only to different selectors; exact + duplicates remain in one tier. Request body validated via Zod: { object_refs: Array<{id, modified}> } tags: - 'Release Tracks' @@ -781,8 +796,15 @@ paths: object_ref: type: string object_modified: - type: string - format: date-time + description: | + Exact timestamp for members, or an exact timestamp + or "latest" selector for candidates and staged. + oneOf: + - type: string + format: date-time + - type: string + enum: + - latest object_status: type: string nullable: true @@ -833,6 +855,10 @@ paths: draft snapshot. The draft must subsequently be reviewed and explicitly released through the shared snapshot release endpoints. This operation is available only for tracks whose type is `virtual`. + Every member and quarantine entry is persisted with an exact + object_ref and object_modified revision. The resulting snapshot never + follows later component track_latest activity, and snapshot retrieval + does not re-resolve composition. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -964,7 +990,11 @@ paths: summary: 'Get the latest snapshot of a release track' operationId: 'release-tracks-snapshot-get-latest' description: | - Return the most recent full snapshot for a release track. + Return the most recent full snapshot for a release track. Virtual + snapshot membership is the exact revision set persisted during + materialization; retrieval never re-resolves component tracks. Bundle + formatting may append secondary relationships and supporting objects + resolved at request time. tags: - 'Release Tracks' parameters: @@ -1180,6 +1210,10 @@ paths: Update member contents on a historical standard-track snapshot. Virtual membership can only be produced by POST /api/release-tracks/{id}/virtual/snapshots/create. + obj_modified may be an exact ISO timestamp or the request-time + shorthand "latest"; the server resolves "latest" to the actual latest + stix.modified timestamp before persistence. Snapshot members always + store exact revision pins. Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. Request body validated via Zod in controller. @@ -1235,7 +1269,9 @@ paths: the same version-selection contract as the latest release operation: supply `increment` or `version`, never both; omit both for a minor increment. Virtual drafts must have composition_resolution from a - successful materialization. + successful materialization. For standard tracks, dynamic staged + references are resolved to exact object revisions when this release + request is handled, including when the selected snapshot is historical. tags: - 'Release Tracks' parameters: @@ -1277,7 +1313,8 @@ paths: exclusive; omitting both defaults to a minor increment. For a virtual draft, compare against the latest tagged snapshot whose modified timestamp precedes this selected snapshot; never recompute composition. - An unmaterialized virtual draft returns 409. + An unmaterialized virtual draft returns 409. Standard previews resolve + dynamic staged references exactly as the corresponding release would. tags: - 'Release Tracks' parameters: diff --git a/app/lib/release-tracks/backref-reconciler.js b/app/lib/release-tracks/backref-reconciler.js index fd0cd48a..0a539cc2 100644 --- a/app/lib/release-tracks/backref-reconciler.js +++ b/app/lib/release-tracks/backref-reconciler.js @@ -31,6 +31,7 @@ // ============================================================================= const logger = require('../logger'); +const revisionReference = require('./revision-reference'); // Snapshot tier array names, also used verbatim as the backref `tier` value. // Order matters: if a revision somehow appears in multiple tiers, the first @@ -102,7 +103,19 @@ function computeDesiredEntries(snapshot, includeRef) { * @returns {Promise<{added: number, updated: number, removed: number}>} */ async function reconcile(repository, trackId, snapshot, includeRef) { - const desired = computeDesiredEntries(snapshot, includeRef); + let resolvedSnapshot = snapshot; + if (snapshot) { + const latestByObjectRef = new Map(); + resolvedSnapshot = { ...snapshot }; + for (const tierName of TIERS) { + resolvedSnapshot[tierName] = await revisionReference.resolveEntries( + (snapshot[tierName] || []).filter((entry) => includeRef(entry.object_ref)), + latestByObjectRef, + ); + } + } + + const desired = computeDesiredEntries(resolvedSnapshot, includeRef); const current = await repository.retrieveReleaseTrackRefsLean(trackId); const operations = []; diff --git a/app/lib/release-tracks/conflict-resolution.js b/app/lib/release-tracks/conflict-resolution.js index abb426e8..56c43382 100644 --- a/app/lib/release-tracks/conflict-resolution.js +++ b/app/lib/release-tracks/conflict-resolution.js @@ -16,6 +16,7 @@ const { ReleaseConflictError } = require('../../exceptions'); const { sameRevision } = require('./tier-revision-invariant'); +const revisionReference = require('./revision-reference'); /** * Merge incoming entries into an existing tier, applying a conflict policy. @@ -60,9 +61,9 @@ exports.applyConflictPolicy = function applyConflictPolicy(existingTier, incomin break; case 'prefer_latest': { - const incomingTime = new Date(incoming.object_modified).getTime(); - const incumbentTime = new Date(incumbent.object_modified).getTime(); - if (incomingTime > incumbentTime) { + if ( + revisionReference.compareModified(incoming.object_modified, incumbent.object_modified) > 0 + ) { merged[conflictIdx] = incoming; } else { rejected.push(incoming); diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 13e30b3c..8f0c6d85 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -373,6 +373,7 @@ const createTrackBodySchema = z composition: compositionSchema.optional(), snapshot_schedule: snapshotScheduleSchema.optional(), }) + .strict() .superRefine((track, context) => { if (track.type !== 'virtual' && track.snapshot_schedule !== undefined) { context.addIssue({ @@ -452,7 +453,7 @@ const reviewCandidatesBodySchema = z.object({ stixIdentifierSchema, z.object({ id: stixIdentifierSchema, - modified: z.iso.datetime().optional(), + modified: z.iso.datetime().or(z.literal('latest')).optional(), }), ]), ) @@ -470,7 +471,7 @@ const demoteStagedBodySchema = z.object({ .array( z.object({ id: stixIdentifierSchema, - modified: z.iso.datetime(), + modified: z.iso.datetime().or(z.literal('latest')), }), ) .min(1), @@ -478,8 +479,8 @@ const demoteStagedBodySchema = z.object({ /** POST /release-tracks/:id/candidates/:objectRef/update-version */ const updateCandidateVersionBodySchema = z.object({ - old_modified: z.iso.datetime(), - new_modified: z.iso.datetime(), + old_modified: z.iso.datetime().or(z.literal('latest')), + new_modified: z.iso.datetime().or(z.literal('latest')), }); /** PUT /release-tracks/:id/config */ diff --git a/app/lib/release-tracks/revision-reference.js b/app/lib/release-tracks/revision-reference.js new file mode 100644 index 00000000..1c066756 --- /dev/null +++ b/app/lib/release-tracks/revision-reference.js @@ -0,0 +1,85 @@ +'use strict'; + +const objectResolver = require('./object-resolver'); +const { BadRequestError } = require('../../exceptions'); + +const LATEST = 'latest'; + +function isLatest(value) { + return value === LATEST; +} + +function normalize(value) { + return isLatest(value) ? LATEST : new Date(value); +} + +function modifiedKey(value) { + if (isLatest(value)) return LATEST; + const timestamp = new Date(value).getTime(); + return Number.isNaN(timestamp) ? String(value) : String(timestamp); +} + +function sameModified(left, right) { + return modifiedKey(left) === modifiedKey(right); +} + +/** + * Compare two exact or dynamic revision selectors. + * + * A dynamic `latest` selector is at least as recent as every exact revision + * that currently exists, so it wins `prefer_latest` comparisons against an + * exact selector. Two dynamic selectors compare equally. + */ +function compareModified(left, right) { + if (isLatest(left)) return isLatest(right) ? 0 : 1; + if (isLatest(right)) return -1; + return new Date(left).getTime() - new Date(right).getTime(); +} + +/** + * Resolve dynamic entries without mutating the stored snapshot representation. + * + * @param {Array} entries + * @param {Map>} [latestByObjectRef] + * @returns {Promise>} + */ +async function resolveEntries(entries, latestByObjectRef = new Map()) { + const resolveLatest = (objectRef) => { + if (!latestByObjectRef.has(objectRef)) { + latestByObjectRef.set(objectRef, objectResolver.resolveLatestModified(objectRef)); + } + return latestByObjectRef.get(objectRef); + }; + + return Promise.all( + (entries || []).map(async (entry) => { + const objectModified = isLatest(entry.object_modified) + ? await resolveLatest(entry.object_ref) + : new Date(entry.object_modified); + + if (!(objectModified instanceof Date) || Number.isNaN(objectModified.getTime())) { + throw new BadRequestError({ + message: 'Invalid release-track revision selector', + details: + `Object "${entry.object_ref}" must reference "latest" or a valid ` + + 'object_modified timestamp', + }); + } + + return { + ...entry, + object_modified: objectModified, + }; + }), + ); +} + +module.exports = { + LATEST, + isLatest, + normalize, + modifiedKey, + sameModified, + compareModified, + resolveEntries, +}; diff --git a/app/lib/release-tracks/tier-revision-invariant.js b/app/lib/release-tracks/tier-revision-invariant.js index c98225ad..c9bdf0ad 100644 --- a/app/lib/release-tracks/tier-revision-invariant.js +++ b/app/lib/release-tracks/tier-revision-invariant.js @@ -1,13 +1,14 @@ 'use strict'; +const revisionReference = require('./revision-reference'); + // A released/member pin is authoritative over workflow and quarantine pins. // This order also matches backref reconciliation's long-standing defensive // "first tier wins" behavior. const TIER_PRECEDENCE = ['members', 'staged', 'candidates', 'quarantine']; function modifiedKey(value) { - const timestamp = new Date(value).getTime(); - return Number.isNaN(timestamp) ? String(value) : String(timestamp); + return revisionReference.modifiedKey(value); } /** diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index e88496a6..fee1a485 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -1,6 +1,7 @@ 'use strict'; const mongoose = require('mongoose'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); const { validateTrackId, validateTrackName, @@ -27,13 +28,28 @@ const memberEntryDefinition = { }; const memberEntrySchema = new mongoose.Schema(memberEntryDefinition, { _id: false }); +const workflowRevisionDefinition = { + type: mongoose.Schema.Types.Mixed, + required: true, + validate: { + validator(value) { + return ( + revisionReference.isLatest(value) || + (value instanceof Date && !Number.isNaN(value.getTime())) || + (typeof value === 'string' && !Number.isNaN(new Date(value).getTime())) + ); + }, + message: 'object_modified must be an exact Date or "latest"', + }, +}; + const stagedEntryDefinition = { object_ref: { type: String, required: true, validate: validateStixId, }, - object_modified: { type: Date, required: true }, + object_modified: workflowRevisionDefinition, object_status: { type: String, enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'], @@ -50,7 +66,7 @@ const candidateEntryDefinition = { required: true, validate: validateStixId, }, - object_modified: { type: Date, required: true }, + object_modified: workflowRevisionDefinition, object_status: { type: String, enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'], diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index 7d9cdec0..b02e8447 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -24,6 +24,7 @@ const linkById = require('../../lib/linkById'); const EventBus = require('../../lib/event-bus'); const Events = require('../../lib/event-constants'); const { selectRelationshipsForBundle } = require('../../lib/stix-bundle-relationships'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); const { bundleTransformSchema, workbenchTransformSchema, @@ -83,10 +84,19 @@ function getRepositoryMap() { */ exports.hydrateMembers = async function hydrateMembers(entries) { if (!entries || entries.length === 0) return []; + const resolvedEntries = await revisionReference.resolveEntries(entries); + const uniqueResolvedEntries = []; + const seenResolvedEntries = new Set(); + for (const entry of resolvedEntries) { + const key = `${entry.object_ref}::` + revisionReference.modifiedKey(entry.object_modified); + if (seenResolvedEntries.has(key)) continue; + seenResolvedEntries.add(key); + uniqueResolvedEntries.push(entry); + } // Group entries by STIX type prefix const byType = {}; - for (const entry of entries) { + for (const entry of uniqueResolvedEntries) { const type = entry.object_ref.split('--')[0]; if (!byType[type]) byType[type] = []; byType[type].push(entry); @@ -157,7 +167,7 @@ function collectBundleEntries(snapshot, options) { const seen = new Set(); const deduped = []; for (const entry of entries) { - const key = `${entry.object_ref}::${new Date(entry.object_modified).getTime()}`; + const key = `${entry.object_ref}::` + revisionReference.modifiedKey(entry.object_modified); if (seen.has(key)) continue; seen.add(key); deduped.push(entry); diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index a5c9800b..282e7a8b 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -35,6 +35,7 @@ const registryRepo = require('../../repository/release-tracks/release-track-regi const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const snapshotService = require('./snapshot-service'); const workflowGate = require('../../lib/release-tracks/workflow-gate'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); @@ -161,9 +162,15 @@ async function processMemberSync(trackId, snapshot, event) { // Get member sync config with defaults const config = getMemberSyncConfig(snapshot); + const dynamicWorkflowEntry = [...(snapshot.candidates || []), ...(snapshot.staged || [])].find( + (entry) => entry.object_ref === objectRef && revisionReference.isLatest(entry.object_modified), + ); // Check strategy if (config.strategy === 'manual') { + if (dynamicWorkflowEntry && trigger !== 'in-place-update') { + await snapshotService.emitContentsChanged(trackId, snapshot); + } logger.debug(`[member-sync] Track ${trackId} uses manual strategy, skipping auto-enrollment`); return null; } @@ -203,6 +210,16 @@ async function processMemberSync(trackId, snapshot, event) { break; case 'ignore': default: + if ( + existingEntry && + revisionReference.isLatest(existingEntry.object_modified) && + trigger !== 'in-place-update' + ) { + // The persisted selector already follows this revision even though + // the supplant policy requests no workflow mutation. Reconcile + // backrefs so the newly-latest object document reflects that fact. + await snapshotService.emitContentsChanged(trackId, snapshot); + } logger.debug( `[member-sync] Track ${trackId}: ignoring ${objectRef} (existing entry in ${existingTier})`, ); @@ -210,6 +227,18 @@ async function processMemberSync(trackId, snapshot, event) { } } + // A dynamic selector already follows the newly-created revision. Queueing a + // second `latest` entry would create an indistinguishable cross-tier + // duplicate, so retain the existing workflow entry and move its backref. + if ( + mode === 'queue' && + existingEntry && + revisionReference.isLatest(existingEntry.object_modified) + ) { + await snapshotService.emitContentsChanged(trackId, snapshot); + return null; + } + // Workflow gate: the single decision point for the entry's tier and // status given all priors — including the candidacy threshold, so // auto-promotion is decided here in one step instead of bouncing the @@ -225,15 +254,18 @@ async function processMemberSync(trackId, snapshot, event) { autoPromote: snapshot.config?.auto_promote === true, }); - const incomingTime = new Date(newModified).getTime(); + const targetModified = revisionReference.LATEST; if (mode === 'enroll' || mode === 'queue') { // Skip if this exact revision is already pinned in any tier — enrolling - // it again would create a duplicate cross-tier reference (e.g. a - // re-import announcing an already-released revision). + // a dynamic selector for the same revision would create a redundant + // cross-tier reference (e.g. a re-import announcing an already-released + // revision). const alreadyPinned = ['members', 'staged', 'candidates'].some((tier) => (snapshot[tier] || []).some( - (e) => e.object_ref === objectRef && new Date(e.object_modified).getTime() === incomingTime, + (e) => + e.object_ref === objectRef && + revisionReference.sameModified(e.object_modified, newModified), ), ); if (alreadyPinned) { @@ -248,13 +280,15 @@ async function processMemberSync(trackId, snapshot, event) { if (mode === 'move-pin') { // Skip no-op moves: same pin key, same tier, same status (e.g. a second // in-place edit of an entry already marked modified-in-place). - const existingTime = new Date(existingEntry.object_modified).getTime(); const currentStatus = existingEntry.object_status || 'work-in-progress'; if ( - existingTime === incomingTime && + revisionReference.sameModified(existingEntry.object_modified, targetModified) && placement.tier === existingTier && placement.status === currentStatus ) { + if (trigger !== 'in-place-update') { + await snapshotService.emitContentsChanged(trackId, snapshot); + } logger.debug( `[member-sync] Track ${trackId}: change to ${objectRef} leaves the pinned entry ` + `unchanged, skipping`, @@ -267,7 +301,7 @@ async function processMemberSync(trackId, snapshot, event) { const now = new Date(); const newEntry = { object_ref: objectRef, - object_modified: new Date(newModified), + object_modified: targetModified, object_status: placement.status, }; if (placement.tier === 'staged') { @@ -284,9 +318,11 @@ async function processMemberSync(trackId, snapshot, event) { // Remove the previous entry when moving the pin if (mode === 'move-pin') { - const removeTime = new Date(existingEntry.object_modified).getTime(); const keep = (e) => - !(e.object_ref === objectRef && new Date(e.object_modified).getTime() === removeTime); + !( + e.object_ref === objectRef && + revisionReference.sameModified(e.object_modified, existingEntry.object_modified) + ); if (existingTier === 'candidates') { newCandidates = newCandidates.filter(keep); } else { diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index cda77646..6aa14bb6 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -30,6 +30,7 @@ const memberSyncService = require('./member-sync-service'); const releaseHistoryService = require('./release-history-service'); const attackObjectsService = require('../stix/attack-objects-service'); const userAccountsService = require('../system/user-accounts-service'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); const MODULE = 'release-tracks-service'; const TIER_NAMES = ['members', 'staged', 'candidates', 'quarantine']; @@ -85,8 +86,15 @@ async function getUsersById(userIds) { return usersById; } -function addObjectInfo(entry, objectsByVersion, usersById) { - const object = objectsByVersion.get(versionKey(entry.object_ref, entry.object_modified)); +function selectorKey(entry) { + return `${entry.object_ref}:${revisionReference.modifiedKey(entry.object_modified)}`; +} + +function addObjectInfo(entry, resolvedModifiedBySelector, objectsByVersion, usersById) { + const resolvedModified = resolvedModifiedBySelector.get(selectorKey(entry)); + const object = resolvedModified + ? objectsByVersion.get(versionKey(entry.object_ref, resolvedModified)) + : undefined; const entryWithObjectInfo = { ...entry, }; @@ -115,9 +123,17 @@ async function addObjectInfoToSnapshot(snapshot) { return snapshot; } + const resolvedEntries = await revisionReference.resolveEntries(tierEntries); + const resolvedModifiedBySelector = new Map(); const uniqueEntriesByVersion = new Map(); - for (const entry of tierEntries) { - uniqueEntriesByVersion.set(versionKey(entry.object_ref, entry.object_modified), entry); + for (let index = 0; index < tierEntries.length; index++) { + const entry = tierEntries[index]; + const resolvedEntry = resolvedEntries[index]; + resolvedModifiedBySelector.set(selectorKey(entry), resolvedEntry.object_modified); + uniqueEntriesByVersion.set( + versionKey(resolvedEntry.object_ref, resolvedEntry.object_modified), + resolvedEntry, + ); } const objects = await attackObjectsService.getBulkByIdAndModified([ @@ -133,7 +149,7 @@ async function addObjectInfoToSnapshot(snapshot) { for (const tierName of TIER_NAMES) { if (snapshot[tierName]) { snapshotWithObjectInfo[tierName] = snapshot[tierName].map((entry) => - addObjectInfo(entry, objectsByVersion, usersById), + addObjectInfo(entry, resolvedModifiedBySelector, objectsByVersion, usersById), ); } } diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index dc58ac95..b148a061 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -20,6 +20,7 @@ const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); const versionUtils = require('../../lib/release-tracks/version-utils'); +const objectResolver = require('../../lib/release-tracks/object-resolver'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const { TrackNotFoundError, @@ -63,6 +64,36 @@ function assertStandardTrack(snapshot) { } } +/** + * Convert contents request entries into exact revision pins. + * + * `latest` is request-time shorthand only. It must never be persisted because + * snapshot membership is defined by an immutable `(object_ref, + * object_modified)` pair. + * + * @param {Array<{obj_ref: string, obj_modified: string}>} contents + * @returns {Promise>} + */ +async function resolveContentsMembers(contents) { + const latestByObjectRef = new Map(); + const resolveLatest = (objectRef) => { + if (!latestByObjectRef.has(objectRef)) { + latestByObjectRef.set(objectRef, objectResolver.resolveLatestModified(objectRef)); + } + return latestByObjectRef.get(objectRef); + }; + + return Promise.all( + contents.map(async (entry) => ({ + object_ref: entry.obj_ref, + object_modified: + entry.obj_modified === 'latest' + ? await resolveLatest(entry.obj_ref) + : new Date(entry.obj_modified), + })), + ); +} + /** * Recompute and persist denormalized registry counters from actual snapshot data. * @@ -487,10 +518,7 @@ exports.updateMetadataByModified = async function updateMetadataByModified( exports.updateContents = async function updateContents(trackId, contents, _userId) { const source = await exports.getLatestSnapshot(trackId); assertStandardTrack(source); - const members = contents.x_mitre_contents.map((c) => ({ - object_ref: c.obj_ref, - object_modified: c.obj_modified === 'latest' ? new Date() : new Date(c.obj_modified), - })); + const members = await resolveContentsMembers(contents.x_mitre_contents); return exports.cloneSnapshot(trackId, source, { members }); }; @@ -512,10 +540,7 @@ exports.updateContentsByModified = async function updateContentsByModified( ) { const source = await exports.getSnapshotByModified(trackId, modified); assertStandardTrack(source); - const members = contents.x_mitre_contents.map((c) => ({ - object_ref: c.obj_ref, - object_modified: c.obj_modified === 'latest' ? new Date() : new Date(c.obj_modified), - })); + const members = await resolveContentsMembers(contents.x_mitre_contents); return exports.cloneSnapshot(trackId, source, { members }); }; diff --git a/app/services/release-tracks/standard-track-service.js b/app/services/release-tracks/standard-track-service.js index d59ed3e1..47655146 100644 --- a/app/services/release-tracks/standard-track-service.js +++ b/app/services/release-tracks/standard-track-service.js @@ -14,6 +14,7 @@ const snapshotService = require('./snapshot-service'); const objectResolver = require('../../lib/release-tracks/object-resolver'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const logger = require('../../lib/logger'); @@ -74,7 +75,8 @@ function normalizeObjectRef(entry) { * Add one or more objects as candidates on the latest snapshot. * * For each entry: - * - If `modified` is "latest" or omitted, resolve via the STIX service layer. + * - If `modified` is "latest" or omitted, validate the object exists and + * preserve a dynamic selector through the candidate/staged workflow. * - Skip duplicates (same object_ref + object_modified already in any tier). * - New candidates start as "work-in-progress". * @@ -101,10 +103,13 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId for (const raw of objectRefs) { const entry = normalizeObjectRef(raw); - // Resolve modified timestamp + // `latest` is a dynamic workflow-tier selector. Resolve it once here to + // validate that the object exists, but preserve the selector until the + // staged entry is frozen by a release operation. let modified; if (!entry.modified || entry.modified === 'latest') { - modified = await objectResolver.resolveLatestModified(entry.id); + await objectResolver.resolveLatestModified(entry.id); + modified = revisionReference.LATEST; } else { modified = new Date(entry.modified); } @@ -115,7 +120,7 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId if (isDuplicate) { logger.verbose( `StandardTrackService: Skipping already-pinned candidate ${entry.id} @ ` + - modified.toISOString(), + `${revisionReference.isLatest(modified) ? modified : modified.toISOString()}`, ); continue; } @@ -374,19 +379,18 @@ exports.updateCandidateVersion = async function updateCandidateVersion(trackId, const source = await snapshotService.getLatestSnapshot(trackId); assertStandardTrack(source); - const oldTime = new Date(data.old_modified).getTime(); const existingCandidates = source.candidates || []; let found = false; const updatedCandidates = existingCandidates.map((candidate) => { if ( candidate.object_ref === objectRef && - new Date(candidate.object_modified).getTime() === oldTime + revisionReference.sameModified(candidate.object_modified, data.old_modified) ) { found = true; return { ...candidate, - object_modified: new Date(data.new_modified), + object_modified: revisionReference.normalize(data.new_modified), }; } return candidate; @@ -447,13 +451,15 @@ exports.demoteStaged = async function demoteStaged(trackId, objectRefs, userId) const existingCandidates = source.candidates || []; // Build a lookup key for the refs to demote - const demoteKeys = new Set(objectRefs.map((r) => `${r.id}::${new Date(r.modified).getTime()}`)); + const demoteKeys = new Set( + objectRefs.map((r) => `${r.id}::${revisionReference.modifiedKey(r.modified)}`), + ); const remainingStaged = []; const demotedEntries = []; for (const staged of existingStaged) { - const key = `${staged.object_ref}::${new Date(staged.object_modified).getTime()}`; + const key = `${staged.object_ref}::` + revisionReference.modifiedKey(staged.object_modified); if (demoteKeys.has(key)) { // Convert back to a candidate entry, preserving workflow status demotedEntries.push({ diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 6349772a..857cb238 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -9,6 +9,7 @@ const dynamicRepo = require('../../repository/release-tracks/release-track-dynam const versionUtils = require('../../lib/release-tracks/version-utils'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); const releaseHistoryService = require('./release-history-service'); const logger = require('../../lib/logger'); const { @@ -124,6 +125,12 @@ function planRelease( 'Create a persisted draft with POST /api/release-tracks/:id/virtual/snapshots/create before previewing or releasing it', }); } + if ( + sourceSnapshot.type === 'standard' && + (sourceSnapshot.staged || []).some((entry) => revisionReference.isLatest(entry.object_modified)) + ) { + throw new TypeError('Standard release planning requires resolved staged revisions'); + } const normalized = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot); const snapshot = normalized.snapshot; @@ -237,15 +244,26 @@ function planRelease( } async function planLoadedSnapshot(trackId, snapshot, options) { - const [versionHistory, previousTaggedSnapshot] = await Promise.all([ + const [versionHistory, previousTaggedSnapshot, resolvedStaged] = await Promise.all([ releaseHistoryService.getTrackWideVersionHistory(trackId), snapshot.type === 'virtual' ? dynamicRepo.getLatestTaggedSnapshotBefore(trackId, snapshot.modified) : Promise.resolve(null), + snapshot.type === 'standard' + ? revisionReference.resolveEntries(snapshot.staged || []) + : Promise.resolve(snapshot.staged || []), ]); + const releaseInput = + snapshot.type === 'standard' + ? { + ...snapshot, + staged: resolvedStaged, + } + : snapshot; + return planRelease( trackId, - snapshot, + releaseInput, versionHistory, options, new Date(), diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index cd9b8c4a..2697a63b 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -20,6 +20,7 @@ const snapshotService = require('./snapshot-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); const deduplicationStrategies = require('../../lib/release-tracks/deduplication-strategies'); +const objectResolver = require('../../lib/release-tracks/object-resolver'); const EventBus = require('../../lib/event-bus'); const Events = require('../../lib/event-constants'); const logger = require('../../lib/logger'); @@ -101,7 +102,7 @@ async function validateComponentTracks(componentTracks) { if (!registry) { throw new TrackNotFoundError(component.track_id); } - if (registry.type === 'virtual') { + if (registry.type !== 'standard') { throw new InvalidComponentTypeError(component.track_id); } registryMap.set(component.track_id, registry); @@ -267,6 +268,54 @@ async function hydrateDomains(componentTracks, resolutions) { ); } +/** + * Lock every component member to an exact revision before filtering and + * deduplication. Current snapshots already store Date-valued pins; resolving + * missing or `latest` values is a defensive compatibility boundary for legacy + * component data. The virtual snapshot itself never persists a moving ref. + * + * @param {Array} resolutions + * @returns {Promise>} + */ +async function lockComponentMemberRevisions(resolutions) { + const latestByObjectRef = new Map(); + + const resolveLatest = (objectRef) => { + if (!latestByObjectRef.has(objectRef)) { + latestByObjectRef.set(objectRef, objectResolver.resolveLatestModified(objectRef)); + } + return latestByObjectRef.get(objectRef); + }; + + return Promise.all( + resolutions.map(async (snapshot) => ({ + ...snapshot, + members: await Promise.all( + (snapshot.members || []).map(async (member) => { + const unresolved = member.object_modified == null || member.object_modified === 'latest'; + const objectModified = unresolved + ? await resolveLatest(member.object_ref) + : new Date(member.object_modified); + + if (Number.isNaN(objectModified.getTime())) { + throw new BadRequestError({ + message: 'Component snapshot contains an invalid member revision', + details: + `Component ${snapshot.id} member ${member.object_ref} must identify ` + + 'an exact object_modified revision', + }); + } + + return { + ...member, + object_modified: objectModified, + }; + }), + ), + })), + ); +} + /** * Resolve the current virtual composition into concrete member revisions. * @@ -285,9 +334,10 @@ async function resolveComposition(snapshot, registryMap) { const allAnnotatedMembers = []; // Resolve each component track in parallel - const resolutions = await Promise.all( + const resolvedComponentSnapshots = await Promise.all( componentTracks.map((component) => resolveComponentSnapshot(component)), ); + const resolutions = await lockComponentMemberRevisions(resolvedComponentSnapshots); const domainsByVersion = await hydrateDomains(componentTracks, resolutions); for (let i = 0; i < componentTracks.length; i++) { diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 9bb9dfe2..e862a3a3 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -462,6 +462,60 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { }); }); + it('moves a dynamic candidate backref even when supplant ignores workflow changes', async function () { + const revisionA = await postObject( + '/api/techniques', + buildTechnique('Backref Dynamic Ignore'), + ); + const trackId = await createTrack('Backref Dynamic Ignore Track'); + await postObject( + `/api/release-tracks/${trackId}/contents`, + { + x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], + }, + 200, + ); + await request(app) + .put(`/api/release-tracks/${trackId}/config`) + .send({ + member_sync: { + strategy: 'track_latest', + supplant: { behavior: 'ignore', status_policy: 'reset' }, + }, + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + const revisionBData = buildTechnique('Backref Dynamic Ignore v2'); + revisionBData.stix.id = revisionA.stix.id; + revisionBData.stix.created = revisionA.stix.created; + revisionBData.stix.modified = new Date( + new Date(revisionA.stix.modified).getTime() + 1000, + ).toISOString(); + const revisionB = await postObject('/api/techniques', revisionBData); + + const revisionCData = buildTechnique('Backref Dynamic Ignore v3'); + revisionCData.stix.id = revisionA.stix.id; + revisionCData.stix.created = revisionA.stix.created; + revisionCData.stix.modified = new Date( + new Date(revisionB.stix.modified).getTime() + 1000, + ).toISOString(); + const revisionC = await postObject('/api/techniques', revisionCData); + + expect(entryForTrack(await getTechniqueVersion(revisionB), trackId)).toBeUndefined(); + expect(entryForTrack(await getTechniqueVersion(revisionC), trackId)).toEqual({ + id: trackId, + type: 'standard', + tier: 'candidates', + status: 'work-in-progress', + }); + + const snapshot = await getObjectVersion(`/api/release-tracks/${trackId}/snapshots/latest`); + expect(snapshot.candidates).toHaveLength(1); + expect(snapshot.candidates[0].object_modified).toBe('latest'); + }); + it('manual strategy leaves candidate pins on the original revision', async function () { const revisionA = await postObject('/api/techniques', buildTechnique('Backref Manual Sync')); const trackId = await createTrack('Backref Manual Sync Track'); @@ -562,7 +616,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { return data; } - it('re-adding an object after a new revision replaces the stale pin (prefer_latest default)', async function () { + it('keeps an omitted candidate selector dynamic as newer revisions are created', async function () { const revisionA = await postObject('/api/techniques', buildTechnique('Backref Readd')); const trackId = await createTrack('Backref Readd Track'); // manual strategy isolates the add-candidates path from revision sync @@ -578,8 +632,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { buildNextRevision(revisionA, 'Backref Readd v2'), ); - // Re-add without modified — resolves to the latest revision and - // replaces the stale pin instead of duplicating it + // Re-adding the same dynamic selector is idempotent. await postObject( `/api/release-tracks/${trackId}/candidates`, { object_refs: [{ id: revisionA.stix.id }] }, @@ -588,7 +641,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const candidates = await listCandidates(trackId); expect(candidates).toHaveLength(1); - expect(new Date(candidates[0].object_modified).toISOString()).toBe(revisionB.stix.modified); + expect(candidates[0].object_modified).toBe('latest'); expect(entryForTrack(await getTechniqueVersion(revisionA), trackId)).toBeUndefined(); expect(entryForTrack(await getTechniqueVersion(revisionB), trackId)).toEqual({ @@ -598,7 +651,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { status: 'work-in-progress', }); - // An exact re-add of the same revision is idempotent + // Another dynamic re-add remains idempotent. await postObject( `/api/release-tracks/${trackId}/candidates`, { object_refs: [{ id: revisionA.stix.id }] }, diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index 23d79c6a..f1c42d55 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -211,7 +211,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { const { candidates } = await getJson(`/api/release-tracks/${trackId}/candidates`); expect(candidates).toHaveLength(1); expect(candidates[0].object_status).toBe('modified-in-place'); - expect(new Date(candidates[0].object_modified).toISOString()).toBe(technique.stix.modified); + expect(candidates[0].object_modified).toBe('latest'); // The marker is reviewable: modified-in-place → awaiting-review await postObject( @@ -440,14 +440,12 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { status: 'work-in-progress', }); - // The pin moved to the converted revision + // The dynamic pin now resolves to the converted revision. const oldRevision = await getTechniqueVersion(technique.stix.id, technique.stix.modified); expect(entryForTrack(oldRevision, trackId)).toBeUndefined(); const { candidates } = await getJson(`/api/release-tracks/${trackId}/candidates`); expect(candidates).toHaveLength(1); - expect(new Date(candidates[0].object_modified).toISOString()).toBe( - result.primary.stix.modified, - ); + expect(candidates[0].object_modified).toBe('latest'); }); it('enrolls the converted revision as a candidate in member tracks (convert-to-technique)', async function () { diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 18390a01..67b5df71 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -100,6 +100,15 @@ describe('Release-track release planning and commit API', function () { .expect(status); } + async function put(path, body, status = 200) { + return request(app) + .put(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + } + async function createTrack(name, type = 'standard') { return (await post('/api/release-tracks/new', { name, type }, 201)).body; } @@ -162,6 +171,145 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version_history.at(-1)).not.toHaveProperty('component_versions'); }); + it('freezes a dynamic staged reference to the latest revision during release', async function () { + const revisionA = (await post('/api/techniques', buildTechnique('Dynamic Release A'), 201)) + .body; + const track = await createTrack('Dynamic Standard Release'); + await put(`/api/release-tracks/${track.id}/config`, { + member_sync: { strategy: 'manual' }, + }); + + const candidate = await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [revisionA.stix.id], + }); + expect(candidate.body.candidates).toEqual([ + expect.objectContaining({ + object_ref: revisionA.stix.id, + object_modified: 'latest', + }), + ]); + + const staged = await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [revisionA.stix.id], + }); + expect(staged.body.staged).toEqual([ + expect.objectContaining({ + object_ref: revisionA.stix.id, + object_modified: 'latest', + }), + ]); + + const revisionB = ( + await post('/api/techniques', buildTechnique('Dynamic Release B', revisionA), 201) + ).body; + const draft = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(draft.body.staged[0]).toMatchObject({ + object_ref: revisionB.stix.id, + object_modified: 'latest', + name: revisionB.stix.name, + }); + + const draftBundle = await get( + `/api/release-tracks/${track.id}/snapshots/latest` + + '?format=bundle&include=staged&includeToc=false', + ); + expect(draftBundle.body.objects).toEqual([ + expect.objectContaining({ + id: revisionB.stix.id, + modified: revisionB.stix.modified, + }), + ]); + + const preview = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=workbench`, + ); + expect(preview.body.staged).toEqual([]); + expect(preview.body.members).toEqual([ + expect.objectContaining({ + object_ref: revisionB.stix.id, + object_modified: revisionB.stix.modified, + }), + ]); + + const unchangedDraft = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(unchangedDraft.body.staged[0].object_modified).toBe('latest'); + + const revisionC = ( + await post('/api/techniques', buildTechnique('Dynamic Release C', revisionB), 201) + ).body; + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}); + expect(released.body.staged).toEqual([]); + expect(released.body.members).toEqual([ + { + object_ref: revisionC.stix.id, + object_modified: revisionC.stix.modified, + }, + ]); + + await post('/api/techniques', buildTechnique('Dynamic Release D', revisionC), 201); + const immutable = await get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.body.modified)}`, + ); + expect(immutable.body.members[0].object_modified).toBe(revisionC.stix.modified); + }); + + it('resolves a historical draft dynamic selector when that draft is released', async function () { + const revisionA = (await post('/api/techniques', buildTechnique('Historical Dynamic A'), 201)) + .body; + const track = await createTrack('Historical Dynamic Release'); + await put(`/api/release-tracks/${track.id}/config`, { + member_sync: { strategy: 'manual' }, + }); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: revisionA.stix.id, modified: 'latest' }], + }); + const staged = await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [revisionA.stix.id], + }); + await post(`/api/release-tracks/${track.id}/meta`, { + description: 'newer unrelated draft', + }); + + const revisionB = ( + await post('/api/techniques', buildTechnique('Historical Dynamic B', revisionA), 201) + ).body; + const releasePath = + `/api/release-tracks/${track.id}/snapshots/` + + `${encodeURIComponent(staged.body.modified)}/release`; + const released = await post(releasePath, { version: '4.0' }); + + expect(released.body.members).toEqual([ + { + object_ref: revisionB.stix.id, + object_modified: revisionB.stix.modified, + }, + ]); + }); + + it('preserves an explicitly pinned staged revision during release', async function () { + const revisionA = (await post('/api/techniques', buildTechnique('Pinned Release A'), 201)).body; + const track = await createTrack('Pinned Standard Release'); + await put(`/api/release-tracks/${track.id}/config`, { + member_sync: { strategy: 'manual' }, + }); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: revisionA.stix.id, modified: revisionA.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [revisionA.stix.id], + }); + + await post('/api/techniques', buildTechnique('Pinned Release B', revisionA), 201); + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}); + + expect(released.body.members).toEqual([ + { + object_ref: revisionA.stix.id, + object_modified: revisionA.stix.modified, + }, + ]); + }); + it('records immutable component versions when previewing and releasing a virtual draft', async function () { const member = (await post('/api/techniques', buildTechnique('Provenance Member'), 201)).body; const component = await createTrack('Provenance Component'); @@ -572,7 +720,7 @@ describe('Release-track release planning and commit API', function () { x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], }); await post(`/api/release-tracks/${track.id}/candidates`, { - object_refs: [{ id: revisionB.stix.id, modified: revisionB.stix.modified }], + object_refs: [{ id: revisionB.stix.id, modified: 'latest' }], }); await post(`/api/release-tracks/${track.id}/candidates/promote`, { object_refs: [revisionB.stix.id], diff --git a/app/tests/api/release-tracks/virtual-composition-validation.spec.js b/app/tests/api/release-tracks/virtual-composition-validation.spec.js index 8613152a..9b28bf28 100644 --- a/app/tests/api/release-tracks/virtual-composition-validation.spec.js +++ b/app/tests/api/release-tracks/virtual-composition-validation.spec.js @@ -183,7 +183,7 @@ describe('Virtual release-track composition validation API', function () { } }); - it('validates initial component existence and standard-track type before persistence', async function () { + it('requires standard component tracks during creation and composition update', async function () { const missingComponentName = 'Missing Component Create'; await createVirtual( composition({ @@ -207,5 +207,35 @@ describe('Virtual release-track composition validation API', function () { virtualComponentName, ); expect(await listTracks(virtualComponentName)).toEqual([]); + + await putComposition( + composition({ + track_id: virtualTrack.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }), + 400, + ); + }); + + it('rejects native members instead of silently creating a hybrid virtual track', async function () { + const name = 'Native Members Rejected'; + await post( + '/api/release-tracks/new', + { + name, + type: 'virtual', + composition: composition(component('latest_tagged')), + native_members: [ + { + object_ref: 'attack-pattern--11111111-1111-4111-8111-111111111111', + object_modified: '2024-02-01T10:00:00.000Z', + }, + ], + }, + 400, + ); + + expect(await listTracks(name)).toEqual([]); }); }); diff --git a/app/tests/api/release-tracks/virtual-determinism.spec.js b/app/tests/api/release-tracks/virtual-determinism.spec.js new file mode 100644 index 00000000..f4f70380 --- /dev/null +++ b/app/tests/api/release-tracks/virtual-determinism.spec.js @@ -0,0 +1,201 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const modelFactory = require('../../../models/release-tracks/model-factory'); +const login = require('../../shared/login'); +const { cloneForCreate } = require('../../shared/clone-for-create'); + +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Virtual release-track deterministic membership API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function get(path) { + const response = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return response.body; + } + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + function buildMitigation(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'course-of-action', + labels: ['test'], + x_mitre_version: '1.0', + x_mitre_domains: ['enterprise-attack'], + object_marking_refs: [staticMarkingDefinitionId], + }, + }; + } + + async function createRevision(name, previous) { + const body = previous ? cloneForCreate(previous) : buildMitigation(name); + if (previous) { + body.stix.name = name; + body.stix.modified = new Date( + new Date(previous.stix.modified).getTime() + 1000, + ).toISOString(); + } + return post('/api/mitigations', body); + } + + async function createReleasedComponent(name, member, modified = member.stix.modified) { + const component = await post('/api/release-tracks/new', { + name, + type: 'standard', + }); + const contents = await post( + `/api/release-tracks/${component.id}/contents`, + { + x_mitre_contents: [ + { + obj_ref: member.stix.id, + obj_modified: modified, + }, + ], + }, + 200, + ); + await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}, 200); + return { component, contents }; + } + + async function createVirtual(name, componentTrackId) { + return post('/api/release-tracks/new', { + name, + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: componentTrackId, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + deduplication: { strategy: 'prioritize_latest_object' }, + }, + }); + } + + function revisionKeys(snapshot) { + return (snapshot.members || []).map( + (member) => `${member.object_ref}::${new Date(member.object_modified).toISOString()}`, + ); + } + + it('resolves latest shorthand before persistence and freezes the tagged component revision', async function () { + const revisionA = await createRevision('Deterministic Member A'); + const { component, contents } = await createReleasedComponent( + 'Deterministic Exact Component', + revisionA, + 'latest', + ); + + expect(contents.members).toEqual([ + { + object_ref: revisionA.stix.id, + object_modified: revisionA.stix.modified, + }, + ]); + + // The standard track's default track_latest policy enrolls this new + // revision into a draft candidate. It must not alter the already-tagged + // component snapshot selected by virtual composition. + const revisionB = await createRevision('Deterministic Member B', revisionA); + const virtual = await createVirtual('Deterministic Exact Virtual', component.id); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + + expect(materialized.members).toEqual([ + { + object_ref: revisionA.stix.id, + object_modified: revisionA.stix.modified, + }, + ]); + expect(materialized.members[0].object_modified).not.toBe(revisionB.stix.modified); + + const firstLatest = await get(`/api/release-tracks/${virtual.id}/snapshots/latest`); + const explicit = await get( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(materialized.modified)}`, + ); + + await createRevision('Deterministic Member C', revisionB); + const secondLatest = await get(`/api/release-tracks/${virtual.id}/snapshots/latest`); + + expect(revisionKeys(firstLatest)).toEqual(revisionKeys(materialized)); + expect(revisionKeys(explicit)).toEqual(revisionKeys(materialized)); + expect(revisionKeys(secondLatest)).toEqual(revisionKeys(materialized)); + }); + + it('locks a legacy moving component member to an exact revision during materialization', async function () { + const revisionA = await createRevision('Legacy Moving Member A'); + const { component } = await createReleasedComponent('Legacy Moving Component', revisionA); + const revisionB = await createRevision('Legacy Moving Member B', revisionA); + + // Bypass Mongoose to simulate data created before exact Date-valued member + // pins were enforced. The virtual materialization boundary must consume + // the shorthand but never copy it into the virtual snapshot. + const ComponentModel = modelFactory.getModel(component.id); + await ComponentModel.collection.updateOne( + { id: component.id, version: '1.0' }, + { $set: { 'members.0.object_modified': 'latest' } }, + ); + + const virtual = await createVirtual('Legacy Moving Virtual', component.id); + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + ); + + expect(materialized.members).toEqual([ + { + object_ref: revisionB.stix.id, + object_modified: revisionB.stix.modified, + }, + ]); + expect(materialized.members[0].object_modified).not.toBe('latest'); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index f57ad0ac..1a1fe4b8 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -25,6 +25,52 @@ Keep these rules in mind while updating the connector: the namespace convention and do not currently include `/standard/`. - A release preview is a read-only `GET`. A release commit is a `POST`. +## P0 — Model draft revision selectors separately from released member pins + +### [ ] Preserve `"latest"` in candidate and staged frontend state + +Candidate and staged entries no longer always contain an ISO timestamp. +Their `object_modified` field is a revision selector: + +```ts +type WorkflowRevisionSelector = string | 'latest'; + +interface CandidateOrStagedEntry { + object_ref: string; + object_modified: WorkflowRevisionSelector; +} +``` + +Here, `string` should be validated as an ISO timestamp when it is not the +literal `"latest"`. Member and quarantine models should remain stricter: +their `object_modified` value is always an exact ISO timestamp. + +When `POST /api/release-tracks/:id/candidates` omits `modified` or sends +`"latest"`, the response preserves `"latest"` instead of replacing it with +the current timestamp. Promotion to staged preserves that selector. The UI +should render it as a moving/latest reference and must not parse it as a date. +An explicitly supplied timestamp remains an exact pin. + +Release preview is the freezing boundary. Before a standard release preview is +rendered, the backend resolves every staged `"latest"` selector. Therefore, +`format=workbench` shows exact timestamps in the would-be `members`, and a +committed release always stores exact member revisions. Preview and commit are +separate resolutions; if an object changes between them, the committed member +may legitimately be newer than the previewed one. + +Done when: + +- Candidate and staged DTOs accept either an ISO timestamp or `"latest"`. +- Member and quarantine DTOs accept exact timestamps only. +- Candidate/staged views display a useful “latest” label without date parsing + errors. +- Add-candidate flows omit `modified` or send `"latest"` when the operator + chooses a moving reference, and send an ISO timestamp for an exact pin. +- Candidate-version updates and staged demotions can send `"latest"` as their + selector. +- Release-preview fixtures show dynamic staged input becoming exact + would-be members, and committed-release fixtures contain no dynamic members. + ## P0 — Align the Angular connector with the current routes ### [ ] Use only the explicit snapshot-retrieval endpoints @@ -474,7 +520,13 @@ strategy: The server validates component identity during both creation and update. Referenced tracks must already exist and must be standard tracks, and duplicate -component track IDs are rejected. +component track IDs are rejected. Do not offer virtual tracks in a component +selector. + +Virtual tracks are purely compositional. Do not expose candidate, staged, +direct-member, or `native_members` controls for them. If operators need +aggregate-specific content, direct them to create or select a standard +component track that owns that content. Done when: @@ -484,8 +536,47 @@ Done when: - Changing resolution strategy clears the selector from the previous strategy. - Every component row requires a priority, and duplicate priorities or track selections are blocked before submission. +- Component selectors list standard tracks only. +- Virtual-track forms never submit `native_members` or direct membership + fields. - Submitted composition payloads contain only server-supported properties. +## P1 — Treat virtual snapshot members as exact revision pins + +### [ ] Remove any lazy-resolution assumptions from virtual snapshot views + +Virtual composition is completed when +`POST /api/release-tracks/:id/virtual/snapshots/create` succeeds. The returned +draft directly contains `members`, `quarantine`, and +`composition_resolution`; every tier entry has an exact `object_ref` and +`object_modified` timestamp. + +Do not send a `resolve` query parameter and do not expect a +`resolved_content` response wrapper. Shared workbench retrieval returns the +persisted tier arrays directly. A component's `track_latest` policy may move +pins in newer standard-track drafts, but it cannot change a previously +materialized virtual snapshot. + +The `latest` path segment selects the newest release-track snapshot; it does +not mean “resolve every member to its latest object revision.” If the track has +not acquired another snapshot, repeated `/snapshots/latest` calls identify the +same primary revision set. + +Bundle downloads remain a documented exception: the backend appends secondary +relationships and supporting objects at request time, so the complete +`format=bundle` graph is not guaranteed to reproduce an earlier download. + +Done when: + +- Virtual views read `members` and `quarantine` directly from the snapshot. +- No connector or model exposes `resolve` or `resolved_content`. +- Member links and comparison keys use both `object_ref` and + `object_modified`. +- Tests prove that advancing a component after materialization does not change + the displayed virtual member revision. +- User-facing export guidance does not promise byte-identical bundle + regeneration. + ## P1 — Submit mode-correct virtual snapshot schedules ### [ ] Add conditional validation and complete the dates-mode UI diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 5a5b132c..f413ec1f 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,40 @@ # Release Track TODOs +## Current implementation slice — Deterministic standard releases + +- [x] Preserve `modified: "latest"` and omitted candidate selectors as dynamic + references through the candidate and staged tiers; preserve explicit + timestamps as exact revision pins. +- [x] Resolve every dynamic staged reference to the actual latest + `stix.modified` timestamp during standard release planning, before conflict + detection, preview rendering, or commit. +- [x] Ensure tagged members contain exact revisions only and that preview and + commit use the same release-planning rules. +- [x] Make dynamic candidate/staged references safe in tier comparison, + Workbench enrichment, bundle rendering, back-reference reconciliation, and + member-sync paths. +- [x] Add regression coverage for dynamic and explicit candidate promotion, + release-time resolution after a newer revision is created, historical + release targeting, conflict handling, and member immutability. +- [x] Update OpenAPI, user/developer documentation, frontend guidance, + `internalattack`, and Bruno as required by the corrected contract. +- [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` + suite. +- [x] Apply logic review, inspect the final diff, and propose conventional + commit messages. + +Verification result (2026-07-30): + +- The combined release, back-reference, change-capture, bundle, + tier-invariant, and virtual-determinism regression group passes (81); the + strengthened release-planning spec passes (19). +- OpenAPI validation passes (2), backend lint passes, and the required clean + full suite passes with routine logs suppressed (OpenAPI 2, config 21, API + 945, middleware 24). +- The focused `internalattack` release-track suite passes (30), its complete + suite passes (247), and changed-file Ruff checks pass. +- Relevant REST API, `internalattack`, and Bruno diffs pass whitespace checks. + ## Virtual release tracks This section records the 2026-07-29 documentation-to-implementation audit of @@ -119,18 +154,135 @@ Verification result (2026-07-29): ### P2 — Contract decisions -- [ ] Decide whether virtual tracks can compose virtual tracks. The - implementation currently rejects nesting while portions of the - documentation say standard or virtual components are supported. -- [ ] Decide whether to implement the documented native-members/hybrid model. - Prefer a dedicated standard component track unless a demonstrated use case - requires a second membership authority inside virtual tracks. -- [ ] Decide whether to implement `resolve=true` and `resolved_content`. - Remove these claims from documentation if eager materialization remains the - only supported model. -- [ ] Implement caching and component-release notifications only if measured - scale or an approved product workflow requires them; otherwise describe them - as future considerations rather than current capabilities. +- [x] Virtual tracks cannot compose virtual tracks. Components must be + standard tracks; revisit nesting only if a concrete future use case requires + it. +- [x] Do not implement the documented native-members/hybrid model. Virtual + tracks are purely compositional; content that is not already represented + belongs in a dedicated standard component track. +- [x] Do not implement `resolve=true` or `resolved_content`. Virtual + composition is resolved eagerly into exact object revisions when a draft is + materialized; retrieval must never re-resolve a persisted snapshot. +- [x] Do not implement caching or component-release notifications without + measured scale or an approved operator workflow. Persisted snapshots already + avoid composition recomputation, and no notification recipient, channel, or + expected action has been defined. + +### Current implementation slice — Deterministic virtual membership + +- [x] Resolve the `latest` request shorthand to the actual latest + `stix.modified` value before standard-track contents are persisted. +- [x] Defensively lock any unresolved component member to an exact revision + during virtual materialization, while preserving exact revisions already + frozen into tagged component snapshots. +- [x] Add regression coverage proving that component `track_latest` behavior + cannot move a materialized virtual member and repeated snapshot retrieval + returns the same exact revision set. +- [x] Remove `resolve=true` and `resolved_content` from the documented + retrieval contract. +- [x] Clearly document that persisted primary member revisions are + deterministic while bundle-time secondary-object and relationship + expansion is not. +- [x] Update OpenAPI, frontend guidance, and Bruno where the clarified + contract affects consumers. +- [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` + suite. +- [x] Apply logic review, inspect the final diff, and propose conventional + commit messages. + +Verification result (2026-07-29): + +- The dedicated virtual-determinism spec passes (2), and the combined + determinism, release-track lifecycle, and virtual-domain regression group + passes (4). +- OpenAPI validation passes (2), backend lint passes, and the complete + `npm test` suite passes (OpenAPI 2, config 21, API 941, middleware 24). +- Logic review result: `ROBUST`. Request-time `latest` resolution, immutable + tagged component pins, legacy unresolved-member locking, invalid-date + rejection, and repeated-reference resolution were covered without finding a + remaining correctness defect. +- Proposed commits: + + ```text + fix(release-tracks): enforce pure virtual composition + + Require virtual components to be standard tracks and reject unsupported + native-member input across the API contract and documentation. + ``` + + ```text + fix(release-tracks): freeze virtual member revisions + + Resolve latest member shorthand before persistence, lock virtual composition + to exact revisions, and document the bundle graph consistency boundary. + ``` + + ```text + docs(release-tracks): clarify snapshot determinism + + Document exact virtual member pins and the bundle-time secondary-content + consistency boundary in the Bruno collection. + ``` + +### Future architecture — Deterministic bundle graphs + +- [ ] Design version-controlled STIX Relationship Objects whose source and + target references identify exact `(object_id, object_modified)` revisions + rather than an entire STIX object provenance chain. +- [ ] Evaluate cloning every affected SRO when a new SDO revision is created, + including atomicity, fan-out, concurrency, migration, and rollback behavior. +- [ ] Measure the resulting database-storage amplification and query/index + costs before approving implementation. +- [ ] Define and persist an export manifest that pins every secondary object, + supporting object, and relationship revision required to reproduce a bundle. +- [ ] Until that architecture is approved and implemented, preserve and + prominently document the accepted constraint that `format=bundle` output is + not graph- or byte-level deterministic. + +### Current implementation slice — Pure standard-track composition + +- [x] Make standard component tracks a positive service-layer requirement, + preserving rejection during both virtual-track creation and composition + replacement. +- [x] Reject unsupported top-level creation properties such as + `native_members` instead of silently stripping them. +- [x] Add regression coverage for virtual-track nesting on both creation and + composition update, and for attempted native-member creation. +- [x] Remove nesting and hybrid/native-member claims from OpenAPI, user and + developer documentation, frontend guidance, and Bruno. +- [x] Run the focused virtual-composition spec, lint, and complete `npm test` + suite. +- [x] Review the final diff and propose conventional commit messages. + +Verification result (2026-07-29): + +- The focused virtual-composition validation spec passes (6), OpenAPI + validation passes (2), and backend lint passes. +- The first complete run encountered six unrelated roaming failures after + 910 API tests passed. All affected specs passed in isolation. +- The required clean `npm test` rerun passes in full, including OpenAPI, + configuration, API, and middleware suites. +- Architecture review result: the positive standard-track allowlist and strict + creation schema keep the contract explicit without adding a parallel + composition path or new abstraction. +- Proposed REST API commit: + + ```text + fix(release-tracks): enforce pure virtual composition + + Require every virtual component to be a standard track during creation and + composition updates. Reject unsupported native-member input and align + OpenAPI, documentation, frontend guidance, and regression coverage. + ``` + +- Proposed companion Bruno commit: + + ```text + docs(release-tracks): clarify pure virtual composition + + Document standard-only components, rejected virtual nesting, and the absence + of native virtual members. + ``` ### Documentation corrections diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index 05d1ba5d..e3b427e3 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -70,11 +70,12 @@ are consistent by the time the triggering API call returns. For one `(repository, trackId, snapshot, includeRef)`: -1. **Desired set** — walk the snapshot tiers in order `members`, `staged`, - `candidates`, `quarantine`, keyed by `(object_ref, object_modified)`. - Snapshot persistence enforces this exact-revision uniqueness invariant; - first-tier-wins remains a defensive fallback for legacy/directly written - invalid documents. Status mapping: +1. **Desired set** — resolve candidate/staged `"latest"` selectors for the + current reconciliation pass, then walk the snapshot tiers in order + `members`, `staged`, `candidates`, `quarantine`, keyed by the resulting + exact `(object_ref, object_modified)` pair. The persisted workflow selector + remains unchanged. First-tier-wins remains a defensive fallback for + legacy/directly written invalid documents. Status mapping: members → `reviewed`; staged/candidates → the entry's `object_status`; quarantine → none. 2. **Current set** — `find({ 'workspace.release_tracks.id': trackId })`, @@ -119,9 +120,10 @@ track's pin (which would strand the pin and orphan the backref). New revisions created through `create()` are covered by the strip; if any track references the object (members, candidates, or staged), member sync -enrolls or re-pins the new revision and the resulting snapshot clone triggers -reconciliation, which stamps the backref on the new revision (see -`member-sync-strategies.md`). +enrolls a dynamic workflow selector or refreshes an existing one. A snapshot +clone or an explicit contents-changed reconciliation then moves that dynamic +backref to the newly latest revision without rewriting the stored selector +(see `member-sync-strategies.md`). ## Known limitations diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 28cadaaa..074eebe9 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -100,9 +100,11 @@ Implemented in filter, mirroring the fact that members are inherently reviewed. `state` never affects members. `reviewed` is intentionally not a valid `state` value for this reason. -2. **Hydration** — the selected `{object_ref, object_modified}` pins are - batch-fetched per STIX type via each repository's - `findManyByIdAndModified`. +2. **Hydration** — any selected candidate/staged `"latest"` selectors are + resolved for this export request, then the concrete + `{object_ref, object_modified}` pairs are batch-fetched per STIX type via + each repository's `findManyByIdAndModified`. The stored draft selectors are + not mutated. 3. **Relationships** — the relationship service fetches the latest active relationship revisions whose `source_ref` and `target_ref` are both among the selected objects. Deprecated data-component `detects` relationships @@ -134,13 +136,33 @@ Because snapshot contents are explicitly curated, the export intentionally does **not** apply the legacy attack-id / deprecated / revoked filters — if a revision is in the snapshot, it is exported. -#### Relationship consistency boundary - -Snapshot SDOs are reproducible because each member records an exact -`object_modified` revision. Relationships are intentionally different: the -bundle resolves their latest active revisions when it is requested. This -keeps relationships secondary and automatically reflects new links between -released objects, but it creates several tradeoffs: +#### Relationship and secondary-object consistency boundary + +Release-track snapshots distinguish **primary** and **secondary** content: + +- Primary objects are explicit snapshot tier entries. Members and quarantine + record exact `(object_ref, object_modified)` revisions. Standard candidates + and staged entries may instead store `"latest"` and are resolved just in + time when a draft export includes those tiers. +- Secondary objects are not snapshot members. They are discovered because a + primary object references them through an embedded STIX ID, an SRO connects + two selected primary objects, or the bundle needs a supporting identity or + marking definition. + +Tagged standard membership is deterministic because release planning resolves +staged selectors before promoting them to members. Virtual materialization +likewise copies exact member revisions from tagged component snapshots and +never follows a component's later `track_latest` candidate movement. Draft +exports that explicitly include dynamic candidate/staged tiers are snapshots +of the latest revisions at export time. Secondary content is also resolved +just in time during bundle generation. + +Relationships are the largest consistency boundary. Current SRO +`source_ref`/`target_ref` fields identify STIX object IDs, not exact +`(object_id, object_modified)` revisions. An SRO can consequently describe the +whole revision chain of each endpoint rather than one precise pair of SDO +entities. The exporter resolves the latest active relationship revisions when +the bundle is requested. This creates several tradeoffs: - exporting the same tagged snapshot at different times can produce different relationship objects or TOC contents; @@ -152,9 +174,17 @@ released objects, but it creates several tradeoffs: constrained to relationships whose two endpoints are already selected. Consumers that require byte-for-byte or graph-level reproducibility must -archive the emitted bundle. A future model that pins relationship revisions -in a separate, generated manifest could preserve the indirect ownership model -while making repeat exports deterministic. +archive the emitted bundle. + +Making bundle graphs deterministic requires a separate, high-risk data-model +change rather than virtual composition re-resolution. A future design must +version-control relationships, pin each SRO endpoint to an exact SDO revision, +and likely clone every affected SRO whenever a new endpoint revision is +created. It must also persist an export manifest containing the selected +relationship and other secondary-object revisions. That one-to-one SDO/SRO +model has significant migration, write-amplification, concurrency, and +database-storage costs and is deliberately deferred pending design and +measurement. ### Where validation happens diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 2cdd7c3b..abd4272d 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -135,7 +135,7 @@ Each release track snapshot will be tracked as an individual MongoDB Document in // Automatically promoted from candidates when track-scoped status → "reviewed" { object_ref: "attack-pattern--ddd", - object_modified: "2024-01-14T10:00:00Z", // VERSION PIN: specific object version + object_modified: "latest", // DYNAMIC SELECTOR: resolved at release object_status: "reviewed", // Track-scoped status object_staged_at: "2024-01-14T11:00:00Z", object_staged_by: "reviewer@example.com" @@ -147,14 +147,14 @@ Each release track snapshot will be tracked as an individual MongoDB Document in // Objects being worked on (in THIS release track), not yet ready for release { object_ref: "attack-pattern--eee", - object_modified: "2024-01-12T09:00:00Z", // VERSION PIN: specific object version + object_modified: "2024-01-12T09:00:00Z", // EXACT SELECTOR: fixed object version object_status: "work-in-progress", // Track-scoped status object_added_at: "2024-01-10T10:00:00Z", object_added_by: "alice@example.com" }, { object_ref: "attack-pattern--fff", - object_modified: "2024-01-13T14:00:00Z", // VERSION PIN: specific object version + object_modified: "latest", // DYNAMIC SELECTOR: follows latest object_status: "awaiting-review", // Track-scoped status object_added_at: "2024-01-12T14:30:00Z", object_added_by: "bob@example.com" @@ -389,11 +389,6 @@ Virtual release tracks compute their contents by aggregating objects from compon conflicts_resolved: [] }, - // Native objects (if virtual track has its own objects in addition to composed) - native_objects: { - members_count: 0 // Virtual tracks can optionally have native members - }, - // Final statistics summary: { total_objects: 870, @@ -414,10 +409,9 @@ Virtual release tracks compute their contents by aggregating objects from compon cron: "0 0 1 1,7 *" // Jan 1 and July 1 at midnight UTC }, - // Configuration - config: { - notification_email: "enterprise-team@example.com" - }, + // Shared release-track configuration. Virtual tracks do not use + // candidate/staged/member-sync workflow controls. + config: {}, // Version history (same as standard tracks) version_history: [ @@ -435,6 +429,25 @@ Virtual release tracks compute their contents by aggregating objects from compon } ``` +Standard `candidates` and `staged` entries may use either an exact +`object_modified` timestamp or the dynamic selector `"latest"`. Promotion +between those workflow tiers preserves the selector. During release planning, +every dynamic staged selector is resolved to the latest stored object revision +before conflict handling and rendering. Only exact revision timestamps may be +persisted in `members`, so tagged standard snapshots have deterministic primary +membership. + +Virtual `members` and `quarantine` entries always store exact +`(object_ref, object_modified)` revision pairs. They never store `"latest"` or +inherit the component track's `track_latest` behavior. Composition resolution +copies the exact member revisions from the selected tagged component +snapshots, and later component activity cannot change the persisted virtual +snapshot. + +This deterministic guarantee covers primary snapshot membership. Secondary +objects and relationships discovered while rendering `format=bundle` remain +an export-time concern and can change between bundle requests. + The three valid `snapshot_schedule` shapes are: ```javascript @@ -485,6 +498,9 @@ restart recovery idempotent. Failed occurrences remain retryable. - Can only reference **tagged snapshots** from component tracks (not drafts) - Can only sync from component tracks' **`members` tier** (released objects only) - Can only compose from **standard release tracks** (not other virtual tracks - no nesting allowed) +- Is purely compositional and has no `native_members` or second membership + authority; aggregate-specific content belongs in another standard component + track - Snapshots are created **manually or on schedule** (never event-driven) - All snapshots start as **drafts** and must be explicitly tagged - Component tracks must exist and have at least one tagged release diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index ce52f092..95d0da3c 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -15,13 +15,14 @@ db.objects.createIndex({ 'workspace.workflow.status': 1 }); ## Validation Rules -- **Same object version** can only be in one tier per release-track snapshot +- **Same revision selector** can only be in one tier per release-track snapshot (`members`, `staged`, `candidates`, or `quarantine`) -- **Different versions** of same object CAN exist in multiple tiers simultaneously +- **Different selectors** for the same object CAN exist in multiple tiers simultaneously - Status transitions must be valid: WIP → Awaiting → Reviewed (no backwards transitions) - Candidacy threshold must be valid enum value - Object version must exist before adding as candidate (validate `stix.id` and `stix.modified` exist) -- Version pin (`object_modified`) is immutable once set for a tier entry +- Candidate/staged `object_modified` may be an exact timestamp or `"latest"`; + member/quarantine entries must be exact - Release version selection accepts either an `increment` or an explicit `version`, never both. Controller validation returns 400 at the HTTP boundary, and `version-utils.calculateNextVersion` repeats the invariant so internal @@ -29,8 +30,8 @@ db.objects.createIndex({ 'workspace.workflow.status': 1 }); ### Cross-tier revision enforcement -`app/lib/release-tracks/tier-revision-invariant.js` owns exact-revision -identity (`object_ref` + normalized `object_modified`) and normalization. +`app/lib/release-tracks/tier-revision-invariant.js` owns selector identity +(`object_ref` + normalized `object_modified`) and normalization. Every clone-based mutation passes through `snapshot-service.cloneSnapshot`; track cloning uses the same normalizer. Tagging is the one in-place mutation, so `versioning-service` normalizes before the atomic tag update. This covers @@ -39,17 +40,33 @@ candidate pin changes, member sync, direct content replacement, bundle import, standard/virtual snapshot creation, and release commits without route-specific guards. -Normalization keeps the first occurrence in the authoritative order +Normalization keeps the first identical selector in the authoritative order `members` → `staged` → `candidates` → `quarantine`. The order matches backref reconciliation's defensive precedence: published membership wins over in-flight workflow state, and resolved virtual membership wins over quarantine. Exact duplicates within one tier are not collapsed because quarantine entries may retain source-specific provenance. -`conflict-resolution.applyConflictPolicy` separately treats an exact -destination duplicate as an idempotent successful move. It does not reject -the incoming entry, so callers remove its source-tier occurrence. Conflict -policies remain responsible only for different revisions of one object. +`conflict-resolution.applyConflictPolicy` separately treats an identical +destination selector as an idempotent successful move. It does not reject the +incoming entry, so callers remove its source-tier occurrence. Conflict +policies remain responsible only for different selectors of one object. + +### Standard release resolution boundary + +Candidate requests that omit `modified` or specify `"latest"` persist that +literal selector. Candidate-to-staged promotion does not freeze it. +`versioning-service.planLoadedSnapshot` is the single resolution boundary for +both latest and historical standard release targets: it resolves staged +selectors before normalization, conflict detection, summary calculation, or +workbench/bundle rendering. The pure `planRelease` function rejects any +standard input whose staged tier still contains `"latest"`, preventing +internal callers from accidentally persisting a dynamic member. + +Preview and commit intentionally resolve independently. A new object revision +between those requests may change the plan; the successful commit freezes the +revision it resolved. Candidate entries remain workflow state and are not +resolved or promoted by release. ## Performance Considerations @@ -70,6 +87,16 @@ Virtual-only operations are deliberately scoped beneath - `POST /virtual/snapshots/create` resolves tagged component snapshots and persists the concrete members, quarantine, and immutable `composition_resolution`. +- Every persisted member and quarantine entry uses an exact + `(object_ref, object_modified)` revision. Standard candidate/staged entries + may persist `"latest"`, but standard release planning resolves staged + selectors before they enter members. Virtual materialization also normalizes + unresolved legacy component entries at its boundary; it never persists a + moving reference. +- `member_sync.strategy = track_latest` applies only to standard tracks. New + object revisions may update a component's newer candidate/staged draft, but + they cannot rewrite the members of the tagged component snapshot selected + during virtual materialization or an already-persisted virtual snapshot. - `POST /virtual/quarantine/promote` clones the latest virtual snapshot, selects one exact quarantined revision for members, and removes all quarantined alternatives for that object. @@ -87,7 +114,18 @@ strategy does not inspect it. Zod rejects duplicate component IDs and priorities before service delegation. The facade also asks the virtual-track service to verify that every component exists and is a standard track before persisting an initial virtual track; update and materialization retain the same -service-layer validation. +service-layer validation. Standard type is a positive requirement, so virtual +tracks cannot compose other virtual tracks. Virtual tracks are also purely +compositional: the strict creation contract rejects unsupported properties +such as `native_members`, and content unique to an aggregate must be modeled in +a standard component track. + +Snapshot retrieval never re-runs composition, so there is no `resolve` query +parameter or `resolved_content` response wrapper. Workbench retrieval returns +the persisted primary membership. Bundle export is a separate consistency +boundary: secondary relationships and supporting objects are discovered at +request time and are not deterministic until relationships become +version-controlled against exact endpoint revisions. Snapshot schedules use the same strict, mode-discriminated Zod schema at the controller and service boundaries. `manual` has no selector field, `cron` diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index d6e78427..c0288a9d 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -70,18 +70,32 @@ A **Member Sync Strategy** is a configuration setting on a release track that de ### When Does Member Sync Apply? -Member sync logic is triggered by **object modification events**. Specifically, when a STIX object is created or updated (resulting in a new `modified` timestamp), the system checks whether that object is referenced by any release track's latest snapshot — in `members`, `candidates`, or `staged`. For each referencing track, the configured member sync strategy determines what action (if any) to take: - -- **Object in `members`:** the new revision is auto-enrolled as a candidate (the original behavior). If a candidate/staged entry for the object already exists, the supplant config governs the overlap. -- **Object pinned only in `candidates`/`staged`:** the pin follows the new revision per the supplant config (`replace` moves the pin — to the same tier under `status_policy: preserve`, back to `candidates` under `reset`; `queue` adds a second candidate entry; `ignore` does nothing). +Member sync logic is triggered by **object modification events**. Specifically, +when a STIX object is created or updated, the system checks whether that object +is referenced by any release track's latest snapshot — in `members`, +`candidates`, or `staged`. For each referencing track, the configured member +sync strategy determines what workflow action (if any) to take: + +- **Object in `members`:** the object is auto-enrolled as a candidate with + `object_modified: "latest"`. The exact released member remains unchanged. +- **Object referenced only in `candidates`/`staged`:** a dynamic selector + already follows the new revision. `replace` may reset or preserve its + workflow standing; `queue` cannot add a second indistinguishable `"latest"` + entry; and `ignore` leaves workflow standing unchanged. If the existing + workflow entry is explicitly timestamp-pinned, the supplant policy can + replace it with `"latest"`, queue a dynamic candidate beside it, or retain + the exact pin. > **Behavior evolution (2026-07-10):** member sync originally applied *only* to > objects in `members`, on the rationale that candidates/staged entries were > still in-flight. In practice that meant a candidate pin silently went stale > the moment the author kept editing — the release would ship the old pinned > revision, and the object's latest view lost its `workspace.release_tracks` -> backref (the membership appeared to vanish). Under `track_latest`, pins now -> follow new revisions for all three tiers; `manual` tracks are unaffected. +> backref (the membership appeared to vanish). Under `track_latest`, workflow +> entries now use an explicit `"latest"` selector. Exact `members` pins never +> move. A `manual` track does not auto-enroll or replace entries, although a +> manually created `"latest"` candidate/staged selector still follows the +> object by definition. > Relationships are deliberately excluded from sync — bundle export pulls > active relationships dynamically. > @@ -168,7 +182,8 @@ The `strategy` field determines the primary behavior of member sync. ##### `"track_latest"` (Default for New Release Tracks) -When a new revision of a member object is created, **automatically add it to `candidates`**. +When a new revision of a member object is created, **automatically add a +dynamic `"latest"` reference to `candidates`**. This is the recommended setting for most release tracks. It provides the intuitive "once enrolled, always tracked" behavior that users expect. With this strategy enabled, users can focus on editing objects without worrying about manually re-enrolling them after each release. @@ -197,7 +212,7 @@ members: candidates: - object_ref: attack-pattern--abc - object_modified: 2025-06-15 # Automatically enrolled! + object_modified: latest # Resolves to 2025-06-15 now and keeps following object_status: "work-in-progress" object_added_at: "2025-06-15T10:30:00Z" object_added_by: "system" # Indicates auto-enrollment @@ -233,7 +248,11 @@ members: #### `member_sync.supplant` -The `supplant` configuration controls what happens when a new revision is created **and** an older revision of the same object already exists in `candidates` or `staged`. This scenario is common when users make multiple edits to an object before a release occurs. +The `supplant` configuration controls workflow placement and status when a new +revision is created and the same object already exists in `candidates` or +`staged`. For an exact existing selector, it also controls whether that fixed +revision is retained or replaced by a dynamic one. It never changes the +meaning of an already-persisted `"latest"` selector. ##### `supplant.behavior` @@ -270,7 +289,7 @@ staged: candidates: - object_ref: attack-pattern--abc - object_modified: 2027-01-01 # v27 + object_modified: latest # Currently resolves to v27 object_status: "work-in-progress" # Status reset staged: [] # v26 removed @@ -278,7 +297,8 @@ staged: [] # v26 removed ###### `"queue"` -Keep the older revision where it is and add the newer revision to `candidates` alongside it. +Keep an exact older revision where it is and add a dynamic `"latest"` +candidate alongside it. This setting allows both revisions to coexist and progress through the workflow independently. It is useful when a previous revision needs to ship in an imminent release while a newer revision is still being developed for a subsequent release. @@ -306,7 +326,7 @@ staged: candidates: - object_ref: attack-pattern--abc - object_modified: 2027-01-01 # v27 + object_modified: latest # Currently resolves to v27 object_status: "work-in-progress" staged: @@ -315,7 +335,12 @@ staged: object_status: "reviewed" ``` -**Note:** When using `queue`, multiple versions of the same object can exist across `candidates` and `staged`. The existing conflict resolution policies (configured via `config.promotion_conflicts`) will handle conflicts when these versions are eventually promoted. For example, if the release track is configured with `staged_to_members: "abort"`, the system will prevent releasing if both v26 and v27 somehow end up competing for promotion to `members`. +**Note:** `queue` can preserve parallel work only when the incumbent entry is +an exact timestamp. If it is already `"latest"`, a second dynamic entry would +be indistinguishable, so the existing selector simply continues following the +object. Multiple exact/dynamic selectors can otherwise coexist across +`candidates` and `staged`; release-time resolution occurs before the configured +promotion conflict policy is applied. ###### `"ignore"` @@ -390,7 +415,7 @@ staged: # With status_policy: "preserve", the new revision: staged: - object_ref: attack-pattern--abc - object_modified: 2027-01-01 + object_modified: latest object_status: "reviewed" # Preserved from old revision ``` @@ -423,11 +448,12 @@ staged: [] members: - { object_ref: attack-pattern--T1, object_modified: v25 } candidates: - - { object_ref: attack-pattern--T1, object_modified: v26, object_status: "work-in-progress" } + - { object_ref: attack-pattern--T1, object_modified: latest, object_status: "work-in-progress" } staged: [] ``` -**Explanation:** The new revision v26 is automatically enrolled as a candidate. The released version v25 remains in `members`. This is the most common scenario and demonstrates the core value of member sync. +**Explanation:** A dynamic candidate is automatically enrolled and currently +resolves to v26. The released version v25 remains exactly pinned in `members`. ### Scenario 2: Replacement with Status Reset @@ -451,11 +477,13 @@ staged: members: - { object_ref: attack-pattern--T1, object_modified: v25 } candidates: - - { object_ref: attack-pattern--T1, object_modified: v27, object_status: "work-in-progress" } + - { object_ref: attack-pattern--T1, object_modified: latest, object_status: "work-in-progress" } staged: [] ``` -**Explanation:** v26 is removed from `staged` and v27 is added to `candidates` with reset status. The user will need to re-review v27 before it can be staged again. This ensures that the new changes receive proper scrutiny. +**Explanation:** The exact v26 selector is removed from `staged` and a +dynamic selector, currently resolving to v27, is added to `candidates` with +reset status. The user must re-review it before staging. ### Scenario 3: Replacement with Status Preserved @@ -478,10 +506,11 @@ staged: members: - { object_ref: attack-pattern--T1, object_modified: v25 } staged: - - { object_ref: attack-pattern--T1, object_modified: v27, object_status: "reviewed" } + - { object_ref: attack-pattern--T1, object_modified: latest, object_status: "reviewed" } ``` -**Explanation:** v26 is replaced by v27, but v27 inherits the `reviewed` status and remains in `staged`. This is faster but assumes the new changes don't require re-review. +**Explanation:** The exact v26 selector is replaced by `"latest"`, which +currently resolves to v27, but it inherits `reviewed` and remains staged. ### Scenario 4: Queueing Alongside Existing Revision @@ -507,10 +536,12 @@ members: staged: - { object_ref: attack-pattern--T1, object_modified: v26, object_status: "reviewed" } candidates: - - { object_ref: attack-pattern--T1, object_modified: v27, object_status: "work-in-progress" } + - { object_ref: attack-pattern--T1, object_modified: latest, object_status: "work-in-progress" } ``` -**Explanation:** Both v26 and v27 coexist. v26 will ship in the next release while v27 progresses through the workflow for a subsequent release. This is useful for parallel development across release cycles. +**Explanation:** Exact v26 and dynamic `"latest"` coexist. v26 can ship in +the imminent release while the moving candidate, currently v27, progresses +for a later release. ### Scenario 5: Ignoring When Revision Already Exists @@ -568,9 +599,9 @@ members: - { object_ref: T3, object_modified: v25 } staged: [] # T1-v26 removed candidates: - - { object_ref: T1, object_modified: v27, object_status: "work-in-progress" } # Replaced T1-v26 - - { object_ref: T2, object_modified: v26, object_status: "work-in-progress" } # New enrollment - - { object_ref: T3, object_modified: v27, object_status: "work-in-progress" } # Replaced T3-v26 + - { object_ref: T1, object_modified: latest, object_status: "work-in-progress" } # Currently T1-v27 + - { object_ref: T2, object_modified: latest, object_status: "work-in-progress" } # Currently T2-v26 + - { object_ref: T3, object_modified: latest, object_status: "work-in-progress" } # Currently T3-v27 ``` **Explanation:** Each object is handled according to the strategy: @@ -602,7 +633,7 @@ members: - { object_ref: T1, object_modified: v25 } candidates: [] # Immediately promoted! staged: - - { object_ref: T1, object_modified: v26, object_status: "work-in-progress" } + - { object_ref: T1, object_modified: latest, object_status: "work-in-progress" } ``` **Explanation:** v26 is auto-enrolled to `candidates`, but because the candidacy threshold is `work-in-progress` and auto-promote is enabled, v26 is immediately promoted to `staged`. This demonstrates how member sync integrates with existing promotion logic. @@ -621,17 +652,25 @@ This can lead to interesting scenarios: ### Interaction with Conflict Resolution Policies -When `supplant.behavior` is `queue`, multiple revisions of the same object can coexist across `candidates` and `staged`. This creates potential for conflicts during promotion: +When `supplant.behavior` is `queue`, an exact selector and a dynamic selector +for the same object can coexist across `candidates` and `staged`. This creates +potential for conflicts during promotion: 1. **Candidates to Staged:** If v26 is in `candidates` and v27 is also in `candidates`, promoting one may conflict with the other. The `candidates_to_staged` conflict policy determines resolution. -2. **Staged to Members:** If v26 and v27 are both in `staged` (which can happen with `queue` + subsequent manual promotions), the `staged_to_members` policy applies during release. +2. **Staged to Members:** Release planning resolves `"latest"` first. If the + resulting exact revision conflicts with an existing member, the + `staged_to_members` policy applies. The existing conflict resolution policies (`always_overwrite`, `always_reject`, `prefer_latest`, `abort`) handle these situations. No changes to conflict resolution are required for member sync to function correctly. ### Snapshot Creation -Any change to a release track's `candidates`, `staged`, or `members` arrays results in a new draft snapshot. Member sync follows this convention. When a new revision is auto-enrolled or an existing revision is supplanted, the system creates a new draft snapshot with the updated arrays. +Any change to a release track's `candidates`, `staged`, or `members` arrays +results in a new draft snapshot. If an existing dynamic selector needs no +workflow change, member sync skips the redundant snapshot and emits a +contents-changed reconciliation so its backref moves to the newly latest +object revision. This means: - Auto-enrollment generates a new snapshot @@ -642,7 +681,8 @@ This means: Member sync requires listening for object modification events. When a STIX object is created or modified: -1. The system identifies all release tracks where this object appears in `members` +1. The system identifies all release tracks where this object appears in + `members`, `candidates`, or `staged` 2. For each relevant release track, the configured member sync strategy is evaluated 3. If the strategy dictates action (e.g., auto-enrollment), the appropriate snapshot modifications are made @@ -743,10 +783,11 @@ The following matrix summarizes the behavior for each combination of settings: | Scenario | `track_latest` + `replace` + `reset` | `track_latest` + `replace` + `preserve` | `track_latest` + `queue` | `track_latest` + `ignore` | `manual` | |----------|--------------------------------------|----------------------------------------|--------------------------|--------------------------|----------| -| New revision created (nothing in candidates/staged) | Add to candidates as WIP | Add to candidates as WIP | Add to candidates as WIP | Add to candidates as WIP | No action | -| New revision created (older in candidates as WIP) | Replace in candidates as WIP | Replace in candidates as WIP | Add alongside as WIP | No action | No action | -| New revision created (older in candidates as awaiting-review) | Replace in candidates as WIP | Replace in candidates as awaiting-review | Add alongside as WIP | No action | No action | -| New revision created (older in staged as reviewed) | Remove from staged, add to candidates as WIP | Replace in staged as reviewed | Keep in staged, add to candidates as WIP | No action | No action | +| New revision created (nothing in candidates/staged) | Add `"latest"` to candidates as WIP | Add `"latest"` to candidates as WIP | Add `"latest"` to candidates as WIP | Add `"latest"` to candidates as WIP | No auto-enrollment | +| New revision created (exact older candidate as WIP) | Replace with `"latest"` as WIP | Replace with `"latest"` as WIP | Add `"latest"` alongside | Keep exact pin | Keep exact pin | +| New revision created (exact older candidate as awaiting-review) | Replace with `"latest"` as WIP | Replace with `"latest"` as awaiting-review | Add `"latest"` alongside | Keep exact pin | Keep exact pin | +| New revision created (exact older staged as reviewed) | Replace with candidate `"latest"` as WIP | Replace with staged `"latest"` as reviewed | Keep exact staged and add candidate `"latest"` | Keep exact pin | Keep exact pin | +| New revision created (existing workflow selector is `"latest"`) | Apply reset/preserve workflow policy; selector stays dynamic | Apply reset/preserve workflow policy; selector stays dynamic | No duplicate; selector keeps following | No workflow change; selector keeps following | No workflow change; selector keeps following | --- diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index e3a336e7..6a3d0584 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -473,10 +473,25 @@ Using either contents endpoint with a virtual track returns `400 Bad Request`. ```json { - "x_mitre_contents": ["attack-pattern--uuid1", "malware--uuid2"] + "x_mitre_contents": [ + { + "obj_ref": "attack-pattern--uuid1", + "obj_modified": "2024-02-01T10:00:00.000Z" + }, + { + "obj_ref": "malware--uuid2", + "obj_modified": "latest" + } + ] } ``` +Every entry must include an object ID and either an ISO `obj_modified` +timestamp or the request-time shorthand `"latest"`. The server resolves +`"latest"` to the object's actual latest `stix.modified` value before +persisting the new standard-track snapshot. Snapshot members never store a +moving reference. + ### Release Latest Snapshot Converts the latest draft snapshot to a tagged release. Tags the snapshot in-place (does not create new snapshot). Dynamically sets `x_mitre_version` based on the request body options. @@ -650,10 +665,11 @@ DELETE /api/release-tracks/:id/snapshots/:modified Adds STIX objects as candidates to the latest draft snapshot. Each object is identified by its `stix.id` field, as well as (optionally) its `stix.modified` field. If `stix.modified` is omitted, the latest permutation of the relevant STIX object will be added. The candidacy reference will follow the latest version of the object until the moment the draft is converted to a release, at which point the reference will become locked to the specific permutation of the object that was considered "latest" at the time the release occurred. -If the resolved revision (the same `stix.id` and `stix.modified`) is already -present in any tier of the snapshot, the add is idempotently skipped. A newer -or older revision of an object already in `members` can still be added as a -candidate. +If the same selector is already present in any tier of the snapshot, the add +is idempotently skipped. Thus, a second omitted/`"latest"` request does not +create another dynamic entry. An exact revision and a dynamic selector are +different workflow references, and an older or newer exact revision of an +object already in `members` can still be added as a candidate. ``` POST /api/release-tracks/:id/candidates @@ -706,7 +722,7 @@ GET /api/release-tracks/:id/candidates }, { "object_ref": "malware--fff", - "object_modified": "2024-01-13T14:00:00Z", + "object_modified": "latest", "object_name": "New Malware ABC", "object_type": "malware", "status": "awaiting-review", @@ -737,9 +753,10 @@ Bidirectional status transition is supported here. For example, objects can be t Notably, changes to an object's status (e.g., "work-in-progress" → "awaiting-review") will automatically update its release track membership standing (e.g., candidate, staged, member). In the most restrictive (typical) scenario, a candidate object transitioning to the "reviewed" state will trigger a new draft snapshot creation wherein the object is now staged. -Tier transitions preserve the exact-revision uniqueness invariant. If legacy -state already contains the same revision in `members` and `candidates`, the -transition repairs the duplicate and retains the `members` occurrence. +Tier transitions preserve selector uniqueness. If legacy state already +contains the same exact revision in `members` and `candidates`, the transition +repairs the duplicate and retains the `members` occurrence. A dynamic +candidate remains `"latest"` if it is promoted to staged. ``` POST /api/release-tracks/:id/candidates/review @@ -774,7 +791,7 @@ GET /api/release-tracks/:id/staged "staged": [ { "object_ref": "attack-pattern--ddd", - "object_modified": "2024-01-14T10:00:00Z", + "object_modified": "latest", "object_name": "Reviewed Technique", "object_type": "attack-pattern", "status": "reviewed", @@ -789,9 +806,9 @@ GET /api/release-tracks/:id/staged ### Promote Candidate Objects To Staged Promotion conflict policies apply when `staged` contains a different revision -of the same object. An exact revision already present in another tier is not a -conflict; the operation retains a single occurrence, with `members` taking -precedence over workflow tiers. +selector for the same object. An identical selector already present in another +tier is not a conflict; the operation retains a single occurrence, with +`members` taking precedence over workflow tiers. ``` POST /api/release-tracks/:id/candidates/promote @@ -821,9 +838,10 @@ POST /api/release-tracks/:id/candidates/promote ### Demote Staged Objects To Candidates -Demotion follows the same rule: different revisions are handled by -`promotion_conflicts.into_candidates`, while an exact revision is retained in -only one tier. +Demotion follows the same rule: different selectors are handled by +`promotion_conflicts.into_candidates`, while an identical selector is retained +in only one tier. The request's `modified` value may be an exact timestamp or +`"latest"`. ``` POST /api/release-tracks/:id/staged/demote @@ -916,12 +934,18 @@ track-ID-keyed `version_history[].component_versions` map that a successful release would persist. For a standard track, `before` is the selected draft before staged members are -promoted and `after` is the would-be tagged result. For a virtual track, the -contents were already resolved and frozen when the draft was explicitly -created. A virtual draft without `composition_resolution` returns -`409 Conflict` instead of previewing stale or empty members. A materialized -draft's release summary compares that persisted draft with the most recent -tagged snapshot that precedes it: +promoted and `after` is the would-be tagged result. Before either summary or +rendered preview output is produced, every staged `"latest"` selector is +resolved to the object revision that is latest for that request. The would-be +members in `format=workbench` and `format=bundle` therefore contain exact +timestamps. A later commit performs its own resolution and may select a newer +revision if the object changed after the preview. + +For a virtual track, the contents were already resolved and frozen when the +draft was explicitly created. A virtual draft without +`composition_resolution` returns `409 Conflict` instead of previewing stale or +empty members. A materialized draft's release summary compares that persisted +draft with the most recent tagged snapshot that precedes it: ```json { @@ -957,7 +981,9 @@ preview and release never re-resolve virtual composition. ### Update Candidate Version Pin -Updates which version of an object a candidate reference is pinned to. This allows upgrading a candidate to track a newer version of an object, or downgrading to a previous version. +Updates the revision selector of a candidate reference. Either value may be an +exact ISO timestamp or `"latest"`, allowing a candidate to switch between a +specific revision and a moving reference. ``` POST /api/release-tracks/:id/candidates/:objectRef/update-version @@ -967,7 +993,7 @@ POST /api/release-tracks/:id/candidates/:objectRef/update-version ```json { - "old_modified": "2024-01-15T10:00:00Z", + "old_modified": "latest", "new_modified": "2024-01-20T14:00:00Z" } ``` @@ -982,7 +1008,9 @@ POST /api/release-tracks/:id/candidates/:objectRef/update-version ### List Object Versions in Release Track -Lists all versions of a specific object referenced across all tiers (candidates, staged, members) in the release track. +Lists all occurrences of a specific object across candidates, staged, and +members. Candidate and staged occurrences may report `"latest"`; members +always report an exact timestamp. ``` GET /api/release-tracks/:id/objects/:objectRef/versions @@ -1075,11 +1103,16 @@ Release track not found. ## Virtual Release Tracks -Virtual release tracks are computed aggregations of other release tracks. Unlike standard tracks, virtual tracks don't directly manage objects through the candidate → staged → released workflow. Instead, they compose content from multiple "component tracks" based on configurable rules. +Virtual release tracks are computed aggregations of standard release tracks. +Unlike standard tracks, virtual tracks don't directly manage objects through +the candidate → staged → released workflow. Instead, they compose content from +multiple standard component tracks based on configurable rules. **Key Characteristics:** -- Compute contents from component standard or virtual tracks +- Compute contents only from standard component tracks; virtual-track nesting + is rejected +- Are purely compositional and cannot own native members - Only reference **tagged snapshots** from component tracks (never drafts) - Create snapshots **manually or on schedule** (never event-driven) - All snapshots start as **drafts** and must be explicitly tagged @@ -1169,7 +1202,9 @@ keys, including the incorrect singular `filters.domain`, return only `version`; and `specific_snapshot` requires only `snapshot`. Every component requires a unique, non-negative integer `priority`; lower numbers have higher priority. When composition is supplied during creation, -each referenced track must already exist and must be a standard track. +each referenced track must already exist and must be a standard track. Virtual +tracks cannot reference other virtual tracks, and unsupported top-level +properties such as `native_members` return `400 Bad Request`. ### Update Virtual Track Composition @@ -1270,6 +1305,24 @@ preview is the authoritative comparison and representation of the persisted draft that would be tagged. A non-null `composition_resolution` is the readiness marker for those shared release operations. +Each resulting `members` and `quarantine` entry contains an exact +`(object_ref, object_modified)` pair. Virtual materialization preserves exact +revisions already frozen in the selected tagged component snapshots. It also +resolves any unresolved legacy component entry before persistence. The virtual +snapshot never stores `"latest"` and does not inherit a standard component's +`track_latest` member-sync behavior. + +Shared snapshot retrieval returns these persisted fields directly. There is no +`resolve` query parameter and no `resolved_content` response property; +retrieval never recomputes virtual composition. As long as the track does not +acquire a newer snapshot, `/snapshots/latest` selects the same primary revision +set, and `/snapshots/:modified` addresses that set explicitly. + +This determinism does not extend to the complete `format=bundle` graph. +Secondary relationships, identities, marking definitions, and other supporting +objects are resolved during bundle generation and may change independently of +the primary snapshot members. + `duplicates_found` counts object IDs contributed by more than one component, including repeated contributions of the same exact revision. `conflicts_resolved` includes only object IDs for which multiple distinct diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 7a2193b9..ad7e0049 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -35,23 +35,25 @@ An object referenced by multiple tracks carries one entry per track. ## Semantics -- **Revision-pinned.** Release-track tiers pin specific object revisions - (`object_ref` + `object_modified`). The backref lives on exactly the pinned - revision document. If a track's candidate pin is moved to a newer revision - (`POST /:id/candidates/:objectRef/update-version`), the backref moves with - it. Different revisions of the same object can carry entries for the same - track — e.g. after member sync auto-enrolls a new revision as a candidate, - the released revision keeps its `members` entry and the new revision gets a - `candidates` entry. +- **Resolved to a revision.** Member and quarantine tiers pin an exact + (`object_ref`, `object_modified`) revision. Candidate/staged tiers may + instead store `"latest"`; their backref is attached to the exact revision + that currently satisfies that selector. If a candidate selector is changed + (`POST /:id/candidates/:objectRef/update-version`), reconciliation moves the + backref accordingly. Different revisions of the same object can carry + entries for the same track — e.g. after member sync auto-enrolls a dynamic + candidate, the released revision keeps its `members` entry and the latest + revision gets a `candidates` entry. - **Follows new revisions under `track_latest`.** Creating a new revision of a tracked object keeps the backref on the object's latest revision: for - `members`, the new revision is auto-enrolled as a candidate; for - `candidates`/`staged` pins, the pin (and its backref) moves to the new - revision per the track's member-sync supplant config. Under the `manual` - strategy, pins stay where they are — the old pinned revision keeps the - backref, and the new revision (which the track genuinely does not - reference) has none; use `?versions=all` to see membership across - revisions. + `members`, the new revision is auto-enrolled with a dynamic candidate + selector; an existing dynamic `candidates`/`staged` selector keeps its + literal `"latest"` value while reconciliation moves its backref. An + explicitly timestamp-pinned workflow entry remains fixed unless member-sync + policy replaces it. Under the `manual` strategy, an exact pin stays where it + is, while an explicitly chosen `"latest"` selector still follows the newest + revision because that behavior is inherent in the selector; use + `?versions=all` to see membership across revisions. - **Reflects the latest snapshot.** Backrefs mirror the track's *current* (most recent) snapshot. Deleting the latest snapshot reverts backrefs to the previous snapshot's membership; deleting a track removes all of its entries. diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index b525c111..baed84c6 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -42,6 +42,10 @@ shape for snapshot retrieval endpoints and is intended for the Workbench fronten **Characteristics:** - Preserves the release-track snapshot structure - Includes `members`, `staged`, `candidates`, and `quarantine` tier arrays when present +- Member and quarantine `object_modified` values are exact timestamps. + Standard candidate and staged entries may instead contain `"latest"`; the + response enriches them from the currently latest object revision without + replacing the stored selector. - Adds UI-friendly object details to tier entries - Suitable for Workbench UI rendering and release-track management workflows @@ -93,6 +97,9 @@ Standard STIX bundle format: - Self-contained: identities and marking definitions referenced by the exported objects are included automatically - `LinkById` tags in descriptions are converted to markdown citations +- If a draft export explicitly includes candidate or staged tiers, dynamic + `"latest"` selectors are resolved for that export request. Tagged member + contents remain exact. - Notes are never included (notes are Workbench-native objects, not STIX objects) - Suitable for external publication diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 54c335c3..b62b26b3 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -4,7 +4,7 @@ This document describes how object workflow states integrate with the release track versioning and release system. It addresses the critical challenge of managing thousands of objects being developed in parallel by multiple users while maintaining clean, production-ready tagged releases. -**Key Design Decision:** This system uses **release track-centric status with version pinning** to solve the "STIX freeze" problem. Each release track tracks its own workflow status for objects and pins to specific object versions, allowing the same object to be in different states across different release tracks and enabling work on future releases while current releases are frozen. +**Key Design Decision:** This system uses **release track-centric status with revision selection** to solve the "STIX freeze" problem. Each release track tracks its own workflow status for objects and may follow the latest revision or pin an exact revision while work is in flight. Release operations always freeze exact member revisions, allowing the same object to be in different states across tracks while completed releases remain immutable. **Note on Terminology:** We use **release track** instead of "collection" to avoid confusion with TAXII collections, MongoDB collections, STIX bundles, and `x-mitre-collection` SDOs. See [terminology.md](./terminology.md) for the complete terminology guide. @@ -27,14 +27,16 @@ The three workflow states tracked per release track: ### Version Pinning -Each tier entry includes **version pinning** via the `object_modified` timestamp: -- Release tracks track a reference to a **specific version** of an object (identified by its `stix.modified` timestamp) +Each tier entry includes a revision selector in `object_modified`: +- Candidate and staged entries may use an exact `stix.modified` timestamp or + the dynamic selector `"latest"` +- Member entries always identify a **specific version** of an object - Different release tracks can pin to different versions of the same object - This enables working on future object versions while a tagged release containing an earlier version is frozen ### Release Track Membership Tiers -Release tracks maintain objects in three distinct tiers, with each entry pinning to a specific object version: +Release tracks maintain objects in three distinct tiers: 1. **Candidates** (`candidates`) - Objects being worked on with track-scoped status 2. **Staged** (`staged`) - Reviewed objects (in this release track) ready for the next tagged release @@ -45,13 +47,13 @@ Release tracks maintain objects in three distinct tiers, with each entry pinning ``` Object version added to release track ↓ -Track-scoped status: work-in-progress → Added to candidates with version pin +Track-scoped status: work-in-progress → Added to candidates with exact or dynamic selector ↓ Track-scoped status: awaiting-review → Remains in candidates ↓ Track-scoped status: reviewed → Automatically promoted to staged ↓ -Snapshot tagged → staged entries moved to members +Snapshot tagged → dynamic staged selectors resolved and exact revisions moved to members ↓ Snapshot exported → members reflected in stix.x_mitre_contents of the output bundle ``` @@ -149,7 +151,7 @@ POST /api/release-tracks/:id/candidates }, { "object_ref": "attack-pattern--fff", - "object_modified": "2024-01-13T14:00:00Z", + "object_modified": "latest", "status": "work-in-progress", "added_to": "staged" // Auto-promoted if meets threshold } @@ -160,11 +162,12 @@ POST /api/release-tracks/:id/candidates **Business Logic:** 1. Validate all object_refs exist -2. Resolve `object_modified` timestamp: - - If provided: validate that specific version exists - - If omitted: use latest version (highest `stix.modified`) +2. Establish the `object_modified` selector: + - If an ISO timestamp is provided: retain that exact revision pin + - If `"latest"` is provided or `modified` is omitted: persist the dynamic + `"latest"` selector 3. Set initial track-scoped status (defaults to "work-in-progress") -4. Add to `workspace.candidates` with version pin +4. Add to `workspace.candidates` with the exact or dynamic selector 5. If status meets `candidacy_threshold`, auto-promote to `workspace.staged` 6. Update object's `workspace.referenced_by` array @@ -258,7 +261,7 @@ When promoting objects between tiers, conflicts can occur if multiple versions o - Promoting from `staged` to `members` (during tagging/release) when a different version already exists in `members` **Transitions can happen via:** -- **Manual candidate adds** via REST API endpoint (e.g., `POST /api/release-tracks/:id/candidates`) — adding without `modified` resolves the object's latest revision +- **Manual candidate adds** via REST API endpoint (e.g., `POST /api/release-tracks/:id/candidates`) — adding without `modified` creates a dynamic `"latest"` selector - **Demotion** back to candidates (`POST /api/release-tracks/:id/staged/demote`) - **Manual promotion** via REST API endpoint (e.g., `POST /api/release-tracks/:id/candidates/promote`) - **Auto-promotion** based on candidacy threshold (e.g., object status changes to `awaiting-review`) @@ -638,13 +641,19 @@ POST /api/release-tracks/:id/snapshots/latest/release **Business Logic:** 1. Validate no `AlreadyReleasedError` 2. Calculate next version -3. Move all entries from `staged` to `members` (preserving version pins) +3. Resolve every staged `"latest"` selector to the actual latest + `stix.modified` timestamp, then move exact entries into `members` 4. Update object documents: change tier in `workspace.referenced_by` from "staged" → "members" 5. Set `version` on release track 6. Add entry to `version_history` 7. Return summary showing what was promoted -**Note on Version Pins:** The `modified` timestamps are preserved during promotion. Released objects remain pinned to the specific version that was reviewed and staged. +**Note on Revision Selectors:** Explicit timestamps are preserved during +promotion. Dynamic staged selectors are frozen during release planning. +Released objects always contain exact timestamps; `"latest"` is never +persisted in `members`. A preview and a later commit each resolve independently, +so the commit may select a newer revision if the object changes between those +requests. ## Solving the STIX Freeze Problem @@ -1012,7 +1021,7 @@ See [virtual-tracks.md](virtual-tracks.md) for complete virtual track documentat 3. Virtual track snapshot creation (manual or scheduled) - Resolves latest (or pinned) version from each component - Creates draft snapshot with resolved composition - - Team receives notification to review + - Team coordinates review through its established operator workflow 4. Review, preview, and tag - Team reviews which component versions were included diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index f509f58c..55353ba9 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -29,8 +29,10 @@ The Release Tracks API supports two types of release tracks: - Examples: "GroupsMonthly", "TechniquesQuarterly", "SoftwareBiannual" **Virtual Release Tracks** - Computed aggregations of other release tracks (NEW) -- Compose content from multiple standard (or other virtual) tracks +- Compose content from multiple standard tracks; virtual-track nesting is not + supported - No duplicate object tracking - objects managed in source tracks only +- Purely compositional - virtual tracks cannot add native members of their own - Create snapshots manually or on schedule (never event-driven) - Always compose from tagged snapshots only (never drafts) - Examples: "EnterpriseTwiceAnnual" (aggregates Groups + Techniques + Software) @@ -112,7 +114,26 @@ There are three types of membership "standings": This presents a tenable solution to the classic "STIX freeze" dilemma wherein editors cannot begin working on the next-*next* (e.g., v20) release until all objects in the next (e.g., v19) release have been released. Staged objects are locked in for the imminent release, but editors are free to continue iterating on future object changes and can queue them up as candidates without affecting the permutation that has already been staged for the imminent release. -Candidates and staged objects alike can be be statically pinned to specific versions via `stix.id` and `stix.modified` couplings, or maintain dynamic/moving references to object versions by omitting `stix.modified`. In the latter, scenario, the release track will effectively "follow" the latest permutation of the relevant object until the moment a release snapshot is generated, at which point the latest permutation will become "locked in" to `x_mitre_contents` via the `stix.id` and `stix.modified` keys of the latest permutation of the object that existed at the time of the release. +Candidate requests may use `modified: "latest"` (or omit it) to create a +dynamic workflow reference. That selector remains `"latest"` while the entry +moves through `candidates` and `staged`; an explicitly supplied timestamp +remains an exact pin. The `track_latest` member-sync strategy likewise uses +dynamic candidate/staged references for revisions that should continue +following the object. + +Dynamic references are never supported in `members`. During a standard release +preview or commit, the server resolves every dynamic staged selector to the +object revision that is latest when that operation is handled. A successful +commit promotes those exact `(stix.id, stix.modified)` pairs into `members`, +making the released primary contents deterministic and immutable. Previewing +and committing are separate operations, so a newer object revision created +between them can legitimately produce a different plan; the committed release +records the revision resolved by the commit itself. + +Virtual snapshots are stricter still: they copy only exact member revisions +from tagged standard component snapshots. They never inherit `track_latest`, +and retrieving a persisted virtual snapshot does not re-resolve its component +tracks. ## Key Features @@ -123,14 +144,15 @@ Object versions automatically move between tiers based on release track-scoped w ``` Object version added to release track → track-scoped status = "work-in-progress" - → Added to workspace.candidates with version pin + → Added to workspace.candidates with an exact or "latest" revision selector Object status changed in release track → track-scoped status = "reviewed" - → Auto-promoted to workspace.staged (version pin preserved) + → Auto-promoted to workspace.staged (revision selector preserved) Snapshot tagged - → workspace.staged entries → stix.x_mitre_contents (version pins preserved) + → Resolve staged "latest" selectors + → Promote exact revisions into members / stix.x_mitre_contents ``` ### Configurable Thresholds diff --git a/docs/user/release-tracks/terminology.md b/docs/user/release-tracks/terminology.md index 7fe37a6d..9b90dfb3 100644 --- a/docs/user/release-tracks/terminology.md +++ b/docs/user/release-tracks/terminology.md @@ -173,16 +173,17 @@ Standard release tracks use three tiers to manage the object lifecycle from deve **Characteristics:** - When an object is first added to a release track, is it considered a candidate. It does not have full membership yet; if the snapshot were to be tagged and released right now, candidates would not be included. -- Each entry can either be statically pinned to a specific version (via its `object_modified` timestamp), or dynamically pinned to the latest version. +- Each entry can either use an exact `object_modified` timestamp or the + dynamic selector `"latest"`. - Each entry has a collection-scoped status: `work-in-progress`, `awaiting-review`, or `reviewed` - Objects in this tier are NOT included in published STIX bundles by default - Automatically promoted to staged tier when status reaches the candidacy threshold **Duplicate Rules:** -- Cannot contain exact duplicates (same `object_ref` + `object_modified` pair) -- **CAN** contain multiple versions of the same object (same `object_ref`, different `object_modified` timestamps) - - Example: Can have `attack-pattern--T1234, modified: 2024-01-15` AND `attack-pattern--T1234, modified: 2024-02-20` simultaneously - - However, only one version of a given object can be promoted to the `staged` tier and `members` tier +- Cannot contain identical selectors (same `object_ref` + + `object_modified` pair) +- Different selectors for the same object are governed by the configured + `into_candidates` conflict policy **Examples:** - "Add these 10 techniques as candidate objects" @@ -200,13 +201,18 @@ Standard release tracks use three tiers to manage the object lifecycle from deve When the release is exported as a `bundle`, all `members` will be included in the resultant bundle's `x_mitre_contents` array. -- Each `staged` entry includes a version pin (`object_modified` timestamp), which can either equal an ISO 8601 timestamp (designating a specific object version) or `"latest"` (designating a dynamic reference to the latest permutation of the relevant object) +- Each `staged` entry includes a revision selector (`object_modified`), which + can be an ISO 8601 timestamp (a specific object revision) or `"latest"` (a + dynamic reference) - Auto-promoted from candidates when objects meet the [candidacy threshold](./release-workflow.md#candidacy-threshold-configuration) - Moved to member objects tier (`members`) when the snapshot is tagged +- A `"latest"` selector is resolved during release planning; the resulting + member stores the exact `stix.modified` timestamp selected by that operation - NOT included in published STIX bundles until the snapshot is tagged **Duplicate Rules:** -- Cannot contain exact duplicates (same `object_ref` + `object_modified` pair) +- Cannot contain identical selectors (same `object_ref` + + `object_modified` pair) - **CANNOT** contain multiple versions of the same object - If a promotion would create a duplicate (different version of same object already in staged), conflict resolution policy applies @@ -223,7 +229,9 @@ When the release is exported as a `bundle`, all `members` will be included in th **Characteristics:** - Objects are considered "members" if they are contained in the `x_mitre_contents` array of the current snapshot. These are considered *already* released. -- Each entry is a version-pinned reference (`object_ref` + `object_modified`). Dynamic references (`object_modified: "latest"`) are not supported on member objects. +- Each entry is an exact revision pin (`object_ref` + timestamp-valued + `object_modified`). Dynamic references (`object_modified: "latest"`) are + not supported on member objects. - These objects are included in published STIX bundles - Represents the production-ready, published content - Only updated when a snapshot is tagged (staged objects are promoted to members) @@ -290,11 +298,12 @@ A **virtual release track** is a special type of release track that computes its **Characteristics:** - Does NOT manage objects through candidate/staged/released workflow -- Aggregates content from **component tracks** (standard or other virtual tracks) +- Aggregates content only from **standard component tracks** - Only references **tagged snapshots** from component tracks (never drafts) - Creates snapshots **manually** or **on schedule** (*never* event-driven; see [Types of Release Tracks](#types-of-release-tracks) for explanation) - All snapshots start as drafts and must be explicitly tagged -- Can optionally have **native objects** in addition to composed content (hybrid model) +- Is purely compositional and cannot own native objects; place additional + content in a standard component track **Examples:** - "EnterpriseTwiceAnnual" virtual track aggregates: diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 95547b8c..925e314f 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -75,6 +75,14 @@ This is analogous to Git's tagging system: - Git commits = release track snapshots (identified by `modified` key) - Git tags = tagged releases (identified by `version` key) +For a standard track, release planning also freezes workflow selectors. +Candidate entries are not released. Staged entries with an explicit timestamp +retain that exact revision; staged entries whose `object_modified` value is +`"latest"` are resolved to the actual latest `stix.modified` timestamp when +the preview or commit request is handled. Only exact revisions are promoted +into `members`, so the tagged release never contains a dynamic member +reference. + ### In-Place Tagging Strategy When you release a snapshot: @@ -109,9 +117,10 @@ Releases the most recent snapshot (highest `modified`) as a tagged release. Use `"version": "2.0"` instead of `increment` for an explicit version. The selectors are mutually exclusive: supplying both returns `400 Bad Request`, and the server never chooses one over the other. Omitting both -version selectors defaults to a minor increment. The `latest` selector is -resolved when the release request is handled. Callers that need to pin the -operation to one snapshot should use the `:modified` endpoint. +version selectors defaults to a minor increment. The `latest` path segment +selects whichever snapshot is latest when the release request is handled. +Callers that need to pin the operation to one snapshot should use the +`:modified` endpoint. **Examples:** diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index f3e2793c..6885e5ae 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -736,13 +736,6 @@ Each virtual track snapshot stores metadata about how it was composed: conflicts_resolved: [] }, - // Native objects (if any) - native_objects: { - candidates_count: 0, - staged_count: 0, - members_count: 0 - }, - // Final statistics summary: { total_objects: 870, @@ -929,7 +922,7 @@ the literal snapshot or publication artifact that would be tagged. The draft must have a non-null `composition_resolution`, proving that its members and quarantine tiers were materialized from its current composition. -### Get Virtual Track with Resolved Content +### Retrieve a Materialized Virtual Snapshot ```bash GET /api/release-tracks/:id/snapshots/latest?format=workbench&include=all @@ -938,37 +931,33 @@ GET /api/release-tracks/:id/snapshots/latest?format=workbench&include=all **Query params:** - `format`: `bundle` | `workbench` | `filesystemstore` (`filesystemstore` is not yet implemented and returns HTTP 501) - `include`: `members` | `quarantine` | `all` -- `resolve`: `true` (default) | `false` - Whether to resolve composition - -**Response when `resolve=true`:** -```json -{ - "id": "release-track--uuid-virtual", - "type": "virtual", - "snapshot_id": "2024-03-05T10:00:00.000Z", - "modified": "2024-03-05T10:00:00Z", - "version": null, - "name": "Enterprise ATT&CK", - "resolved_content": { - "members": [ - { - "object_ref": "intrusion-set--APT1", - "object_modified": "2024-02-01T10:00:00Z", - "source_track": "release-track--uuid-1", - "source_version": "5.2" - } - // ... all resolved objects - ], - "quarantine": [] - }, - - "composition_resolution": { - "resolved_at": "2024-03-05T10:00:00Z", - "component_snapshots": [...] - } -} -``` +There is no `resolve` query parameter and no `resolved_content` response +property. Composition is resolved eagerly when the virtual draft is created. +The concrete `members`, `quarantine`, and `composition_resolution` fields are +stored directly on that snapshot and are returned without consulting the +component tracks again. + +Every member and quarantined entry contains an exact +`(object_ref, object_modified)` pair. Standard candidate and staged entries may +persist the dynamic selector `"latest"`, but standard release planning resolves +it before promoting those entries into members. Direct standard member +replacement likewise resolves `"latest"` before persistence. A component +track's `track_latest` member-sync policy can create or move dynamic workflow +selectors in newer component drafts, but it cannot change the exact members +already present in a tagged component snapshot or in an existing virtual +snapshot. + +Consequently, while the track does not acquire a newer snapshot, +`GET /snapshots/latest` returns the same primary member revision set. +`GET /snapshots/:modified` identifies that persisted set directly. The +`latest` path segment selects the most recent snapshot; it is not a dynamic +object-revision selector. + +This guarantee applies to the persisted primary snapshot contents. +`format=bundle` also discovers secondary relationships and supporting objects +at export time, so the complete bundle graph is not currently reproducible. +See [Bundle Export](../../developer/release-tracks/bundle-export.md#relationship-and-secondary-object-consistency-boundary). ## Quarantine Management @@ -1006,59 +995,18 @@ Malformed requests and attempts against standard tracks return `400 Bad Request`. Selecting a revision that is not quarantined returns `404 Not Found` without creating a snapshot. -## Hybrid Model: Virtual Track + Native Objects - -Virtual tracks can optionally have **native objects** in addition to composed content. This is an advanced use case where a virtual track needs to include objects that don't exist in any component track: - -```javascript -{ - id: "release-track--uuid-virtual", - type: "virtual", - - // Composed from standard tracks - composition: { - component_tracks: [ - { - track_id: "release-track--uuid-1", - resolution_strategy: "latest_tagged", - priority: 1 - }, - { - track_id: "release-track--uuid-2", - resolution_strategy: "latest_tagged", - priority: 2 - } - ], - deduplication: { - strategy: "prioritize_latest_object" - } - }, - - // PLUS virtual track's own native members - native_members: [ - { - object_ref: "marking-definition--enterprise-only", - object_modified: "2024-01-01T10:00:00Z" - } - ], - - // Final result after sync - members: [ - // ... objects from component tracks - // ... plus native_members - ], - quarantine: [] -} -``` +## Pure Composition -**Use case:** Enterprise track includes Groups and Techniques from standard tracks, PLUS Enterprise-specific marking definitions or custom objects that don't belong in any component track. +Virtual tracks do not own native members and cannot compose other virtual +tracks. Every member must originate from a tagged snapshot of a standard +component track. This keeps one authoritative object lifecycle and one +membership authority for every contributed object. -**When virtual snapshot is created:** -1. Resolve composed content from component tracks (goes to `members` or `quarantine`) -2. Merge with virtual track's `native_members` (goes to `members`) -3. If any `native_members` conflict with composed objects, apply deduplication strategy - -**Note:** This is an advanced feature. Most virtual tracks should only use composition without native members. +If an aggregate needs content that does not belong in its existing component +tracks, create a dedicated standard track for that content and add it to the +virtual composition. Requests containing unsupported properties such as +`native_members`, or composition entries that reference a virtual track, +return `400 Bad Request`. ## Migration Strategy @@ -1143,47 +1091,12 @@ July 1: Enterprise scheduled snapshot triggers July 5: Team reviews draft, tags as Enterprise v14.1 ``` -## Performance Optimizations - -### 1. Snapshot Caching +## Implementation Characteristics -Since virtual snapshots are immutable once created, cache resolved content: - -```javascript -const cacheKey = `virtual-snapshot:${trackId}:${modified}:resolved`; +### 1. Eager, Parallel Component Resolution -const cached = await cache.get(cacheKey); -if (cached) return cached; - -const resolved = await resolveVirtualSnapshot(trackId, modified); -await cache.set(cacheKey, resolved, { ttl: 3600 }); // 1 hour cache -``` - -### 2. Lazy Resolution - -For `GET /api/release-tracks/:id/snapshots/latest` (latest snapshot), only resolve if: -- Query param `resolve=true` is specified -- Format requires resolution (e.g., `format=bundle`) - -Otherwise, return composition metadata without resolving: - -```javascript -if (!query.resolve && query.format === 'workbench') { - // Return composition config without resolving - return { - id: snapshot.id, - type: snapshot.type, - snapshot_id: snapshot.snapshot_id, - modified: snapshot.modified, - version: snapshot.version, - name: snapshot.name, - composition: snapshot.composition, - composition_resolution: snapshot.composition_resolution // Pre-computed - }; -} -``` - -### 3. Parallel Component Resolution +Virtual composition is resolved only during explicit or scheduled snapshot +creation. Component snapshots are fetched in parallel: Resolve component tracks in parallel: @@ -1195,7 +1108,7 @@ const resolutions = await Promise.all( ); ``` -### 4. Deduplication Optimization +### 2. Deduplication Use Set for O(1) duplicate detection: @@ -1212,6 +1125,11 @@ for (const obj of allObjects) { } ``` +The persisted snapshot is already the reusable composition result. No +cross-request snapshot cache is implemented. Caching should be considered only +if measured bundle-rendering latency or database load justifies the additional +invalidation and multi-instance consistency work. + ## Best Practices ### 1. Snapshot Before Tagging @@ -1264,27 +1182,6 @@ Add metadata to virtual track for documentation: } ``` -### 4. Monitor Component Track Releases - -Set up alerts when component tracks release: - -```javascript -eventBus.on('release-track:released', async (event) => { - // Find virtual tracks that reference this standard track - const virtualTracks = await findVirtualTracksByComponent(event.collectionId); - - // Notify virtual track owners - for (const vt of virtualTracks) { - await notificationService.send({ - to: vt.owner_email, - subject: `Component track ${event.collectionName} released v${event.version}`, - body: `Your virtual track "${vt.name}" references this component. ` + - `Consider creating a new snapshot to include the latest release.` - }); - } -}); -``` - ## Limitations ### 1. No Event-Driven Snapshots @@ -1293,7 +1190,10 @@ Virtual tracks do NOT automatically snapshot when component tracks release. **Rationale:** Prevents snapshot explosion when many component tracks release frequently. -**Alternative:** Use notifications + manual snapshots, or scheduled snapshots. +**Alternative:** Create snapshots manually or configure a cron/date schedule. +Component-release notifications are not implemented; they require an approved +operator workflow defining recipients, delivery channel, deduplication, and +the expected follow-up action. ### 2. No Workflow on Composed Objects From e2dd402d9bcf880ec84c224d39377812ef408a6c Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:09:42 -0400 Subject: [PATCH 30/55] fix(scheduler): recover scheduled virtual snapshots Repair the legacy collection-index scheduler regression, expand cron and date schedule coverage, recover persisted snapshots after interrupted occurrences, and include scheduler tests in the default test gate. --- app/scheduler/virtual-track-snapshots-task.js | 36 +++-- app/tests/scheduler/scheduler.spec.js | 27 +++- .../virtual-track-snapshots-task.spec.js | 138 ++++++++++++++++++ docs/admin/virtual-track-schedules.md | 6 + docs/developer/TODO.md | 49 +++++++ docs/developer/task-scheduler.md | 6 + package.json | 2 +- 7 files changed, 251 insertions(+), 13 deletions(-) diff --git a/app/scheduler/virtual-track-snapshots-task.js b/app/scheduler/virtual-track-snapshots-task.js index ef4eee98..0d6c930e 100644 --- a/app/scheduler/virtual-track-snapshots-task.js +++ b/app/scheduler/virtual-track-snapshots-task.js @@ -6,6 +6,7 @@ const config = require('../config/config'); const logger = require('../lib/logger'); const { createAutomationRunRecorder, serializeError } = require('../lib/automation-run-recorder'); const registryRepo = require('../repository/release-tracks/release-track-registry.repository'); +const dynamicRepo = require('../repository/release-tracks/release-track-dynamic.repository'); const occurrenceRepo = require('../repository/release-tracks/virtual-track-schedule-occurrence.repository'); const virtualTrackService = require('../services/release-tracks/virtual-track-service'); @@ -48,10 +49,10 @@ async function auditAttempt(occurrence, execute) { }); try { - const snapshot = await execute(); + const { snapshot, recovered } = await execute(); await recorder.recordItem({ - status: 'changed', - action: 'materialize_virtual_snapshot', + status: recovered ? 'unchanged' : 'changed', + action: recovered ? 'recover_scheduled_virtual_snapshot' : 'materialize_virtual_snapshot', target: { kind: 'release-track', document_id: occurrence.track_id, @@ -61,13 +62,18 @@ async function auditAttempt(occurrence, execute) { snapshot_modified: snapshot.modified, members_count: snapshot.members?.length || 0, quarantine_count: snapshot.quarantine?.length || 0, + recovered, }, }); await recorder.finish({ status: 'completed', - counts: { materialized: 1, failed: 0 }, + counts: recovered + ? { materialized: 0, recovered: 1, failed: 0 } + : { materialized: 1, failed: 0 }, summary: { - message: `Materialized scheduled virtual snapshot for ${occurrence.track_id}`, + message: recovered + ? `Recovered scheduled virtual snapshot for ${occurrence.track_id}` + : `Materialized scheduled virtual snapshot for ${occurrence.track_id}`, }, }); return snapshot; @@ -116,14 +122,26 @@ async function executeOccurrence(occurrence, now = new Date()) { } try { - const snapshot = await auditAttempt(claimed, () => - virtualTrackService.createVirtualSnapshot(claimed.track_id, { + const snapshot = await auditAttempt(claimed, async () => { + // A worker may have persisted the snapshot and exited before completing + // the occurrence ledger. Recover that durable result without recomputing + // composition, which may no longer be resolvable after the crash. + const existing = await dynamicRepo.getSnapshotByScheduledMaterialization( + claimed.track_id, + scheduledFor, + ); + if (existing) { + return { snapshot: existing, recovered: true }; + } + + const materialized = await virtualTrackService.createVirtualSnapshot(claimed.track_id, { scheduledMaterialization: { schedule_mode: claimed.schedule_mode, scheduled_for: scheduledFor, }, - }), - ); + }); + return { snapshot: materialized, recovered: false }; + }); await occurrenceRepo.complete(claimed.track_id, scheduledFor, snapshot.modified); return snapshot; } catch (err) { diff --git a/app/tests/scheduler/scheduler.spec.js b/app/tests/scheduler/scheduler.spec.js index be90003b..926eb469 100644 --- a/app/tests/scheduler/scheduler.spec.js +++ b/app/tests/scheduler/scheduler.spec.js @@ -1,5 +1,7 @@ const request = require('supertest'); const { expect } = require('expect'); +const sinon = require('sinon'); +const superagent = require('superagent'); const logger = require('../../lib/logger'); logger.level = 'debug'; @@ -7,7 +9,12 @@ logger.level = 'debug'; const database = require('../../lib/database-in-memory'); const databaseConfiguration = require('../../lib/database-configuration'); const login = require('../shared/login'); -const scheduler = require('../../scheduler/scheduler'); +const config = require('../../config/config'); + +// This spec exercises the collection-index synchronization task directly. +// Prevent the task module from registering a background job when it is loaded. +config.scheduler.enableScheduler = false; +const collectionIndexTask = require('../../scheduler/sync-collection-indexes-task'); // modified and created properties will be set before calling REST API const initialObjectData = { @@ -505,6 +512,7 @@ const initialObjectData = { describe('Scheduler', function () { let app; let passportCookie; + let remoteRequestStub; before(async function () { // Establish the database connection @@ -523,6 +531,16 @@ describe('Scheduler', function () { const timestamp = new Date().toISOString(); initialObjectData.collection_index.created = timestamp; initialObjectData.collection_index.modified = timestamp; + initialObjectData.workspace.update_policy.subscriptions = []; + + const remoteCollectionIndex = JSON.parse(JSON.stringify(initialObjectData.collection_index)); + remoteCollectionIndex.modified = new Date(Date.now() + 1000).toISOString(); + remoteRequestStub = sinon.stub(superagent, 'get').returns({ + accept: sinon.stub().resolves({ + text: JSON.stringify(remoteCollectionIndex), + }), + }); + const body = initialObjectData; await request(app) .post('/api/collection-indexes') @@ -532,16 +550,19 @@ describe('Scheduler', function () { }); it('Scheduled job runs when initiated manually', async function () { - const updatedCollections = await scheduler.runCheckCollectionIndexes(); + const updatedCollections = await collectionIndexTask.runCheckCollectionIndexes(); expect(updatedCollections).toHaveLength(1); + expect(remoteRequestStub.calledOnce).toBe(true); }); it('Scheduled job is skipped when initiated manually again', async function () { - const updatedCollections = await scheduler.runCheckCollectionIndexes(); + const updatedCollections = await collectionIndexTask.runCheckCollectionIndexes(); expect(updatedCollections).toHaveLength(0); + expect(remoteRequestStub.calledOnce).toBe(true); }); after(async function () { + sinon.restore(); await database.closeConnection(); }); }); diff --git a/app/tests/scheduler/virtual-track-snapshots-task.spec.js b/app/tests/scheduler/virtual-track-snapshots-task.spec.js index 2e33c389..885f57ff 100644 --- a/app/tests/scheduler/virtual-track-snapshots-task.spec.js +++ b/app/tests/scheduler/virtual-track-snapshots-task.spec.js @@ -128,6 +128,33 @@ describe('Scheduled virtual release-track materialization', function () { expect(await snapshotCount(virtual.id)).toBe(3); }); + it('materializes every due date while leaving future dates unregistered', async function () { + const component = await createComponent(); + const firstDue = new Date('2026-03-01T00:00:00.000Z'); + const secondDue = new Date('2026-03-15T12:00:00.000Z'); + const future = new Date('2026-04-01T00:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [firstDue.toISOString(), secondDue.toISOString(), future.toISOString()], + }); + + await task.reconcileSchedules(secondDue); + + expect(await snapshotCount(virtual.id)).toBe(3); + const occurrences = await VirtualTrackScheduleOccurrence.find({ + track_id: virtual.id, + }) + .sort({ scheduled_for: 1 }) + .lean() + .exec(); + expect(occurrences).toHaveLength(2); + expect(occurrences.map((occurrence) => occurrence.scheduled_for)).toEqual([ + firstDue, + secondDue, + ]); + expect(occurrences.every((occurrence) => occurrence.status === 'completed')).toBe(true); + }); + it('materializes duplicate cron delivery once', async function () { const component = await createComponent(); const virtual = await createVirtual(component.id, { @@ -151,6 +178,26 @@ describe('Scheduled virtual release-track materialization', function () { ).toBe(1); }); + it('registers cron tracks in UTC and removes their jobs after track deletion', async function () { + const component = await createComponent(); + const virtual = await createVirtual(component.id, { + mode: 'cron', + cron: '0 0 1 1,7 *', + }); + const jobName = `virtual-track-snapshot-materialization:${virtual.id}`; + + await task.reconcileSchedules(new Date('2026-07-15T00:00:00.000Z')); + + const job = schedule.scheduledJobs[jobName]; + expect(job).toBeDefined(); + expect(job.pendingInvocations[0].recurrenceRule._tz).toBe('Etc/UTC'); + + await releaseTracksService.deleteTrack(virtual.id); + await task.reconcileSchedules(new Date('2026-07-15T00:01:00.000Z')); + + expect(schedule.scheduledJobs[jobName]).toBeUndefined(); + }); + it('audits component failures and retries them during reconciliation', async function () { const component = await createComponent({ released: false }); const scheduledFor = new Date('2026-02-01T00:00:00.000Z'); @@ -203,6 +250,97 @@ describe('Scheduled virtual release-track materialization', function () { expect(runs.map((run) => run.status)).toEqual(['failed', 'completed']); }); + it('reclaims an expired occurrence that has not materialized a snapshot', async function () { + const component = await createComponent(); + const scheduledFor = new Date('2026-05-01T00:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [scheduledFor.toISOString()], + }); + const now = new Date('2026-05-01T00:10:00.000Z'); + + await VirtualTrackScheduleOccurrence.create({ + track_id: virtual.id, + schedule_mode: 'dates', + scheduled_for: scheduledFor, + status: 'running', + attempt_count: 1, + claimed_at: new Date('2026-05-01T00:00:00.000Z'), + claim_expires_at: new Date('2026-05-01T00:05:00.000Z'), + }); + + await task.reconcileSchedules(now); + + expect(await snapshotCount(virtual.id)).toBe(2); + expect( + await VirtualTrackScheduleOccurrence.findOne({ + track_id: virtual.id, + scheduled_for: scheduledFor, + }) + .lean() + .exec(), + ).toMatchObject({ + status: 'completed', + attempt_count: 2, + }); + }); + + it('completes an expired occurrence from its persisted snapshot without recomputing', async function () { + const component = await createComponent(); + const scheduledFor = new Date('2026-06-01T00:00:00.000Z'); + const virtual = await createVirtual(component.id, { + mode: 'dates', + dates: [scheduledFor.toISOString()], + }); + + await VirtualTrackScheduleOccurrence.create({ + track_id: virtual.id, + schedule_mode: 'dates', + scheduled_for: scheduledFor, + status: 'running', + attempt_count: 1, + claimed_at: new Date('2026-06-01T00:00:00.000Z'), + claim_expires_at: new Date('2026-06-01T00:05:00.000Z'), + }); + const materialized = await releaseTracksService.createVirtualSnapshot(virtual.id, { + scheduledMaterialization: { + schedule_mode: 'dates', + scheduled_for: scheduledFor, + }, + }); + + // A persisted scheduled snapshot is the authoritative result. Recovery + // must not depend on the component still being available. + await releaseTracksService.deleteTrack(component.id); + await task.reconcileSchedules(new Date('2026-06-01T00:10:00.000Z')); + + expect(await snapshotCount(virtual.id)).toBe(2); + expect( + await VirtualTrackScheduleOccurrence.findOne({ + track_id: virtual.id, + scheduled_for: scheduledFor, + }) + .lean() + .exec(), + ).toMatchObject({ + status: 'completed', + attempt_count: 2, + snapshot_modified: materialized.modified, + }); + + const recoveryRun = await mongoose.connection + .getClient() + .db() + .collection('automationRuns') + .findOne({ + 'scope.track_id': virtual.id, + status: 'completed', + }); + expect(recoveryRun).toMatchObject({ + counts: { materialized: 0, recovered: 1, failed: 0 }, + }); + }); + it('does not schedule or materialize manual tracks', async function () { const component = await createComponent(); const virtual = await createVirtual(component.id, { mode: 'manual' }); diff --git a/docs/admin/virtual-track-schedules.md b/docs/admin/virtual-track-schedules.md index 627366ea..7aad65cd 100644 --- a/docs/admin/virtual-track-schedules.md +++ b/docs/admin/virtual-track-schedules.md @@ -30,6 +30,12 @@ occurrences. The resulting snapshot also records Together, these controls prevent duplicate drafts across restarts, retry delivery, and multiple scheduler-enabled API instances. +If a worker persists the scheduled snapshot but exits before marking the +occurrence complete, the next worker treats that snapshot as the authoritative +result. It completes the occurrence from the persisted snapshot without +recomputing composition. The recovery attempt is audited as an unchanged +`recover_scheduled_virtual_snapshot` item with `counts.recovered: 1`. + ## Failures and retries An occurrence commonly fails when a component resolution has no matching diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index f413ec1f..fdc2c78c 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,54 @@ # Release Track TODOs +## Current implementation slice — Scheduler regression and virtual schedule coverage + +- [x] Repair the legacy collection-index scheduler spec so it imports the + refactored `sync-collection-indexes-task` module without auto-registering + background jobs during the test. +- [x] Add virtual-track coverage proving reconciliation registers scheduled + cron jobs in UTC and removes jobs for tracks that no longer exist. +- [x] Add date-schedule boundary coverage for multiple due dates and future + dates. +- [x] Add crash-window recovery coverage for a scheduled virtual snapshot that + was persisted before its occurrence ledger reached `completed`. +- [x] Add stale-claim recovery coverage and document any remaining + multi-process lease/fencing limitation. +- [x] Run the legacy scheduler spec, the virtual scheduler spec, the aggregate + scheduler suite, lint, and the complete `npm test` suite. +- [x] Record the coverage conclusion and propose a conventional commit message. + +Coverage conclusion (2026-07-30): + +- Functional coverage is sufficient for the current `manual`, `cron`, and + `dates` contracts. It exercises UTC cron registration and cleanup, duplicate + delivery, multiple due and future dates, missed-date recovery, retryable + component failures, expired claims, and recovery after snapshot persistence. +- Scheduler regressions now run under the default `npm test` and Cobertura + coverage gates instead of requiring a separate developer-only command. +- Remaining production hardening is explicitly tracked below; it does not + change the single-worker schedule contract covered by this slice. + +Verification result (2026-07-30): + +- The deterministic legacy collection-index scheduler spec passes (2), the + expanded virtual scheduler spec passes (8), and the aggregate scheduler + suite passes (10). +- Backend lint passes. The required clean full suite passes: OpenAPI 2, + config 21, API 945, middleware 24, and scheduler 10. +- Three roaming API-suite failures seen during earlier runs passed together in + isolation (23) before the clean full-suite run. + +### Remaining scheduled-materialization hardening + +- [ ] Add an owner token (fencing token) to occurrence claims, make terminal + updates conditional on the active token, and renew leases for work that may + exceed the claim duration. Add a true multi-worker regression proving that + an expired worker cannot overwrite the succeeding worker's result. +- [ ] Decide and document an operator policy for permanent failures. If + indefinite one-minute retries are not acceptable, add bounded exponential + backoff plus a terminal/dead-letter state and operator-visible recovery + controls. + ## Current implementation slice — Deterministic standard releases - [x] Preserve `modified: "latest"` and omitted candidate selectors as dynamic diff --git a/docs/developer/task-scheduler.md b/docs/developer/task-scheduler.md index 5a421054..70824e73 100644 --- a/docs/developer/task-scheduler.md +++ b/docs/developer/task-scheduler.md @@ -55,6 +55,12 @@ track-local index. The ledger prevents concurrent workers from doing the same work, while the snapshot index is the final idempotency guard after crashes or duplicate delivery. +The snapshot is authoritative if persistence succeeds before the worker can +complete the occurrence ledger. Reconciliation detects that persisted result, +marks the reclaimed occurrence complete, and does not recompute virtual +composition. This matters because component tracks may change or be removed +after the scheduled snapshot was already created. + Do not put release-track composition logic in the scheduler task. It delegates to `virtual-track-service`, which is also used by the explicit HTTP operation. diff --git a/package.json b/package.json index 6b446070..1c1ae037 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "prettier:fix": "npm run prettier -- --write", "format": "npm run prettier:fix && npm run lint:fix", "start": "node ./bin/www", - "test": "npm run test:openapi && npm run test:config && npm run test:api && npm run test:middleware", + "test": "npm run test:openapi && npm run test:config && npm run test:api && npm run test:middleware && npm run test:scheduler", "test:api": "mocha --timeout 20000 --recursive ./app/tests/api --exit", "test:config": "mocha --timeout 20000 --recursive ./app/tests/config --exit", "test:import": "mocha --timeout 20000 --recursive ./app/tests/import --exit", From 7fa26eda33058d1196ba81ff974d5624129489fe Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:25:35 -0400 Subject: [PATCH 31/55] fix(release-tracks): enforce unique release versions Enforce tagged-version uniqueness with a database partial index, return a typed conflict for concurrent release races, and add a fail-closed migration for existing and orphan track collections. --- app/exceptions/index.js | 11 ++ app/lib/error-handler.js | 2 + .../release-track-snapshot-schema.js | 12 +- .../release-track-dynamic.repository.js | 9 +- .../release-tracks-release.spec.js | 50 +++++ ...lease-version-uniqueness-migration.spec.js | 74 ++++++++ app/tests/middleware/error-handler.spec.js | 29 ++- docs/developer/TODO.md | 42 +++++ .../release-tracks/implementation-notes.md | 15 ++ docs/user/release-tracks/versioning.md | 10 + ...nforce-release-track-version-uniqueness.js | 173 ++++++++++++++++++ 11 files changed, 421 insertions(+), 6 deletions(-) create mode 100644 app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js create mode 100644 migrations/20260730040000-enforce-release-track-version-uniqueness.js diff --git a/app/exceptions/index.js b/app/exceptions/index.js index ad9a2977..a4db28ef 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -297,6 +297,16 @@ class AlreadyReleasedError extends CustomError { } } +class DuplicateReleaseVersionError extends CustomError { + constructor(trackId, version, options = {}) { + super(`Release track ${trackId} already has tagged version ${version}`, { + ...options, + track_id: trackId, + version, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -386,6 +396,7 @@ module.exports = { //** Version control errors */ AlreadyReleasedError, + DuplicateReleaseVersionError, TaggedSnapshotDeletionError, InvalidVersionError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index b0701431..6e4e292d 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -36,6 +36,7 @@ const { AlreadyRevokedError, SelfRevocationError, AlreadyReleasedError, + DuplicateReleaseVersionError, TaggedSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, @@ -132,6 +133,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof DuplicateNameError || err instanceof AlreadyRevokedError || err instanceof AlreadyReleasedError || + err instanceof DuplicateReleaseVersionError || err instanceof TaggedSnapshotDeletionError || err instanceof ReleaseConflictError || err instanceof VirtualSnapshotNotMaterializedError || diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index fee1a485..814a7190 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -413,8 +413,16 @@ const releaseTrackSnapshotSchema = new mongoose.Schema(releaseTrackSnapshotDefin // Primary lookup: find snapshot by track id + modified timestamp releaseTrackSnapshotSchema.index({ id: 1, modified: -1 }, { unique: true }); -// Find the latest tagged version -releaseTrackSnapshotSchema.index({ id: 1, version: 1 }); +// A tagged version identifies exactly one snapshot within a release track. +// Drafts are excluded so any number of snapshots may retain version: null. +releaseTrackSnapshotSchema.index( + { id: 1, version: 1 }, + { + name: 'unique_tagged_version', + unique: true, + partialFilterExpression: { version: { $type: 'string' } }, + }, +); // A scheduled occurrence may materialize at most one snapshot, including // after restart recovery or duplicate delivery by multiple scheduler nodes. diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index b761ee15..f1675d5b 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -4,6 +4,7 @@ const modelFactory = require('../../models/release-tracks/model-factory'); const { DatabaseError, DuplicateIdError, + DuplicateReleaseVersionError, BadlyFormattedParameterError, } = require('../../exceptions'); const logger = require('../../lib/logger'); @@ -266,8 +267,12 @@ class ReleaseTrackDynamicRepository { return saved.toObject(); } catch (err) { if (err.name === 'MongoServerError' && err.code === 11000) { + if (err.keyPattern?.version && typeof snapshotData.version === 'string') { + throw new DuplicateReleaseVersionError(trackId, snapshotData.version, { cause: err }); + } throw new DuplicateIdError({ details: `Snapshot with modified '${snapshotData.modified}' already exists for track '${trackId}'.`, + cause: err, }); } throw new DatabaseError(err); @@ -305,9 +310,7 @@ class ReleaseTrackDynamicRepository { return result; } catch (err) { if (err.name === 'MongoServerError' && err.code === 11000) { - throw new DuplicateIdError({ - details: `Version conflict while tagging snapshot for track '${trackId}'.`, - }); + throw new DuplicateReleaseVersionError(trackId, versionData.version, { cause: err }); } throw new DatabaseError(err); } diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 67b5df71..6362eff5 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -2,11 +2,13 @@ const request = require('supertest'); const { expect } = require('expect'); +const sinon = require('sinon'); const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const releaseHistoryService = require('../../../services/release-tracks/release-history-service'); const versioningService = require('../../../services/release-tracks/versioning-service'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); @@ -171,6 +173,54 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version_history.at(-1)).not.toHaveProperty('component_versions'); }); + it('allows only one concurrent release to claim a version', async function () { + const track = await createTrack('Concurrent Release Version'); + const newerDraft = await post(`/api/release-tracks/${track.id}/meta`, { + description: 'A distinct draft racing for the same release version', + }); + const originalHistoryLookup = releaseHistoryService.getTrackWideVersionHistory; + let waiting = 0; + let releaseBarrier; + const bothPlanned = new Promise((resolve) => { + releaseBarrier = resolve; + }); + const historyStub = sinon + .stub(releaseHistoryService, 'getTrackWideVersionHistory') + .callsFake(async (...args) => { + const history = await originalHistoryLookup(...args); + waiting += 1; + if (waiting === 2) releaseBarrier(); + await bothPlanned; + return history; + }); + + const release = (modified) => + request(app) + .post(`/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(modified)}/release`) + .send({ version: '2.0' }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + + let responses; + try { + responses = await Promise.all([release(track.modified), release(newerDraft.body.modified)]); + } finally { + historyStub.restore(); + } + expect(responses.map((response) => response.status).sort()).toEqual([200, 409]); + + const conflict = responses.find((response) => response.status === 409); + expect(conflict.body).toEqual({ + message: `Release track ${track.id} already has tagged version 2.0`, + track_id: track.id, + version: '2.0', + }); + + const tagged = await dynamicRepo.getAllSnapshots(track.id, { taggedOnly: true }); + expect(tagged.pagination.total).toBe(1); + expect(tagged.data[0].version).toBe('2.0'); + }); + it('freezes a dynamic staged reference to the latest revision during release', async function () { const revisionA = (await post('/api/techniques', buildTechnique('Dynamic Release A'), 201)) .body; diff --git a/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js b/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js new file mode 100644 index 00000000..895080d1 --- /dev/null +++ b/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js @@ -0,0 +1,74 @@ +'use strict'; + +const { expect } = require('expect'); +const mongoose = require('mongoose'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); +const migration = require('../../../../migrations/20260730040000-enforce-release-track-version-uniqueness'); + +const UNIQUE_INDEX = 'unique_tagged_version'; +const LEGACY_INDEX = 'id_1_version_1'; + +describe('Release-track tagged-version uniqueness migration', function () { + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + }); + + after(async function () { + await database.closeConnection(); + }); + + it('fails closed on legacy duplicates before replacing indexes and is rerunnable after repair', async function () { + const track = await releaseTracksService.createTrack({ + name: 'Legacy Duplicate Release Versions', + type: 'standard', + }); + const released = await releaseTracksService.releaseLatest(track.id, { + version: '1.0', + userAccountId: 'migration-test', + }); + const collection = mongoose.connection.db.collection(track.id); + + await collection.dropIndex(UNIQUE_INDEX); + await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX }); + + const duplicate = { ...released }; + delete duplicate._id; + duplicate.modified = new Date(new Date(released.modified).getTime() + 1000); + await collection.insertOne(duplicate); + await mongoose.connection.db + .collection('releaseTrackRegistry') + .deleteOne({ track_id: track.id }); + + await expect(migration.up(mongoose.connection.db)).rejects.toMatchObject({ + message: expect.stringContaining(`${track.id} version 1.0 (2 snapshots)`), + duplicates: [ + expect.objectContaining({ + track_id: track.id, + version: '1.0', + }), + ], + }); + + let indexes = await collection.indexes(); + expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(true); + expect(indexes.some((index) => index.name === UNIQUE_INDEX)).toBe(false); + + await collection.deleteOne({ modified: duplicate.modified }); + await migration.up(mongoose.connection.db); + await migration.up(mongoose.connection.db); + + indexes = await collection.indexes(); + expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(false); + expect(indexes.find((index) => index.name === UNIQUE_INDEX)).toMatchObject({ + key: { id: 1, version: 1 }, + unique: true, + partialFilterExpression: { version: { $type: 'string' } }, + }); + }); +}); diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index f84f85ae..0ab1829f 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -5,7 +5,12 @@ const sinon = require('sinon'); const logger = require('../../lib/logger'); const errorHandler = require('../../lib/error-handler'); -const { DatabaseError, DuplicateIdError, InvalidPostOperationError } = require('../../exceptions'); +const { + DatabaseError, + DuplicateIdError, + DuplicateReleaseVersionError, + InvalidPostOperationError, +} = require('../../exceptions'); describe('error-handler middleware', function () { beforeEach(function () { @@ -72,6 +77,28 @@ describe('error-handler middleware', function () { expect(next.called).toBe(false); }); + it('should return a structured conflict for DuplicateReleaseVersionError', function () { + const trackId = 'release-track--00000000-0000-4000-8000-000000000001'; + const err = new DuplicateReleaseVersionError(trackId, '2.0'); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(409)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: `Release track ${trackId} already has tagged version 2.0`, + track_id: trackId, + version: '2.0', + }), + ).toBe(true); + expect(next.called).toBe(false); + }); + it('should preserve wrapped error details for DatabaseError', function () { const err = new DatabaseError(new Error('Mongo connection failed')); const res = { diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index fdc2c78c..5ed481dd 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,47 @@ # Release Track TODOs +## Production-readiness branch — `fix/release-tracks-production-readiness` + +This branch implements the prioritized findings in +`.nocommit/project-review-release-tracks/15-recommendations.md`. Each numbered +recommendation is kept as a separate conventional commit so the merge request +can be reviewed or reverted item by item. + +### P0.1 — Enforce release version uniqueness + +- [x] Add a unique partial index for tagged `version` strings in every dynamic + release-track snapshot collection. +- [x] Convert duplicate-version races into a typed `409 Conflict` that + identifies the track and requested version. +- [x] Add a regression that releases two distinct drafts concurrently with the + same version and proves exactly one succeeds. +- [x] Add a rerunnable migration that fails closed on pre-existing duplicates + before replacing the legacy non-unique index. +- [x] Update release-version documentation and run focused, migration, + middleware, lint, and complete-suite verification. + +Verification result (2026-07-30): + +- The deterministic concurrent-release, migration, middleware, and isolated + roaming-failure group passes (39). +- The required clean full suite passes: OpenAPI 2, config 21, API 947, + middleware 25, and scheduler 10. +- The migration preflights the union of registry IDs and canonical orphan + release-track collection names before making any index changes. + +### Remaining prioritized recommendations + +- [ ] P0.2 — Make primary release membership fail closed. +- [ ] P0.3 — Make tagged-content immutability authoritative and durable. +- [ ] P0.4 — Correct destructive authorization and add durable audit records. +- [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. +- [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and + operator intervention. +- [ ] P0.7 — Establish and enforce a safe storage operating envelope. +- [ ] P0.8 — Harden deployment, database readiness, backup/restore, rollback, + and post-deploy verification. +- [ ] Address P1 recommendations in documented criticality order. + ## Current implementation slice — Scheduler regression and virtual schedule coverage - [x] Repair the legacy collection-index scheduler spec so it imports the diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 95d0da3c..43bfceea 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -13,6 +13,21 @@ db.objects.createIndex({ 'workspace.collections.staged': 1 }); db.objects.createIndex({ 'workspace.workflow.status': 1 }); ``` +Each release track also owns a dynamic snapshot collection. Tagged versions +use a unique partial index on `{ id: 1, version: 1 }`, restricted to documents +whose `version` is a string. Drafts therefore remain unlimited at +`version: null`, while the database—not an application-level preflight—decides +which concurrent release may claim a version. + +Migration `20260730040000-enforce-release-track-version-uniqueness` scans the +union of registered tracks and canonical `release-track--` collection +names before changing any indexes. Including orphan collections matters +because track creation predates transaction-backed registry coordination. If any +`(track_id, version)` has multiple tagged snapshots, it reports all offending +snapshot timestamps and performs no index changes. After operators repair the +data, rerunning the migration replaces the legacy non-unique index +idempotently. + ## Validation Rules - **Same revision selector** can only be in one tier per release-track snapshot diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 925e314f..3290e359 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -191,6 +191,16 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema 2. **Immutable once set** - Once a snapshot has `version` assigned, it cannot be changed 3. **Cannot re-tag** - A snapshot can only be tagged once (throws `AlreadyReleasedError` if attempted) 4. **Valid version format** - Must match `/^\d+\.\d+$/` (MAJOR.MINOR only, no patch component) +5. **Unique within the track** - Exactly one snapshot may hold a given tagged + version. If concurrent release requests race for the same version, one + succeeds and the other receives `409 Conflict` with the conflicting + `track_id` and `version`. + +Deployments upgrading from an earlier release run a database migration before +serving traffic. The migration checks every release-track collection for +pre-existing duplicate tagged versions and stops without changing indexes if +it finds any. Operators must resolve every reported track/version pair and +rerun the migration; the server does not guess which tagged snapshot to keep. ### First Tagged Release diff --git a/migrations/20260730040000-enforce-release-track-version-uniqueness.js b/migrations/20260730040000-enforce-release-track-version-uniqueness.js new file mode 100644 index 00000000..5aed38e6 --- /dev/null +++ b/migrations/20260730040000-enforce-release-track-version-uniqueness.js @@ -0,0 +1,173 @@ +'use strict'; + +/** + * Replace the legacy non-unique (id, version) index in every dynamic release + * track collection with a unique partial index over tagged snapshots. + * + * The migration preflights every collection before changing any indexes. If a + * deployment already contains duplicate tagged versions, migration stops and + * reports every offending track/version so an operator can repair the data + * deliberately. + */ + +const INDEX_NAME = 'unique_tagged_version'; +const LEGACY_INDEX_NAME = 'id_1_version_1'; +const CONCURRENCY = 8; +const TRACK_COLLECTION_PATTERN = + /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +async function mapWithConcurrency(items, mapper) { + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + await mapper(items[index]); + } + } + + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, () => worker())); +} + +async function collectionExists(db, trackId) { + return db.listCollections({ name: trackId }, { nameOnly: true }).hasNext(); +} + +async function findTrackCollectionIds(db) { + const [registeredTracks, collections] = await Promise.all([ + db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), + db.listCollections({}, { nameOnly: true }).toArray(), + ]); + + return Array.from( + new Set([ + ...registeredTracks.map((track) => track.track_id), + ...collections + .map((collection) => collection.name) + .filter((name) => TRACK_COLLECTION_PATTERN.test(name)), + ]), + ).sort(); +} + +async function findDuplicateVersions(db, trackIds) { + const duplicates = []; + + await mapWithConcurrency(trackIds, async (trackId) => { + if (!(await collectionExists(db, trackId))) return; + + const matches = await db + .collection(trackId) + .aggregate([ + { $match: { version: { $type: 'string' } } }, + { + $group: { + _id: { id: '$id', version: '$version' }, + count: { $sum: 1 }, + snapshots: { $push: '$modified' }, + }, + }, + { $match: { count: { $gt: 1 } } }, + { $sort: { '_id.version': 1 } }, + ]) + .toArray(); + + for (const match of matches) { + duplicates.push({ + track_id: trackId, + version: match._id.version, + snapshots: match.snapshots, + }); + } + }); + + return duplicates.sort( + (left, right) => + left.track_id.localeCompare(right.track_id) || left.version.localeCompare(right.version), + ); +} + +function isDesiredIndex(index) { + return ( + index?.name === INDEX_NAME && + index.unique === true && + index.key?.id === 1 && + index.key?.version === 1 && + index.partialFilterExpression?.version?.$type === 'string' + ); +} + +async function installUniqueIndex(db, trackId) { + if (!(await collectionExists(db, trackId))) return; + + const collection = db.collection(trackId); + const indexes = await collection.indexes(); + const desired = indexes.find((index) => index.name === INDEX_NAME); + if (isDesiredIndex(desired)) { + if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { + await collection.dropIndex(LEGACY_INDEX_NAME); + } + return; + } + + if (desired) await collection.dropIndex(INDEX_NAME); + if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { + await collection.dropIndex(LEGACY_INDEX_NAME); + } + + await collection.createIndex( + { id: 1, version: 1 }, + { + name: INDEX_NAME, + unique: true, + partialFilterExpression: { version: { $type: 'string' } }, + }, + ); +} + +module.exports = { + async up(db) { + const trackIds = await findTrackCollectionIds(db); + const duplicates = await findDuplicateVersions(db, trackIds); + + if (duplicates.length > 0) { + const summary = duplicates + .map( + (duplicate) => + `${duplicate.track_id} version ${duplicate.version} ` + + `(${duplicate.snapshots.length} snapshots)`, + ) + .join('; '); + const error = new Error( + `Duplicate tagged release versions detected; repair them before retrying migration: ${summary}`, + ); + error.duplicates = duplicates; + throw error; + } + + await mapWithConcurrency(trackIds, (trackId) => installUniqueIndex(db, trackId)); + }, + + async down(db) { + const trackIds = await findTrackCollectionIds(db); + + await mapWithConcurrency(trackIds, async (trackId) => { + if (!(await collectionExists(db, trackId))) return; + + const collection = db.collection(trackId); + const indexes = await collection.indexes(); + if (indexes.some((index) => index.name === INDEX_NAME)) { + await collection.dropIndex(INDEX_NAME); + } + if (!indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { + await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX_NAME }); + } + }); + }, + + _private: { + findTrackCollectionIds, + findDuplicateVersions, + installUniqueIndex, + isDesiredIndex, + }, +}; From aacfeffc60658612de979a732eb16f34c307789a Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:51:29 -0400 Subject: [PATCH 32/55] fix(release-tracks): reject unresolved primary revisions Validate exact primary content at request, release, clone, materialization, import, and export boundaries. Return structured missing references instead of persisting or rendering partial release data. --- .../definitions/components/release-tracks.yml | 29 ++ .../paths/release-tracks-paths.yml | 56 ++- app/exceptions/index.js | 20 ++ app/lib/error-handler.js | 4 + .../release-tracks/bundle-import-service.js | 42 ++- app/services/release-tracks/export-service.js | 79 +---- .../primary-revision-service.js | 151 +++++++++ .../release-tracks/release-tracks-service.js | 7 + .../release-tracks/snapshot-service.js | 28 +- .../release-tracks/standard-track-service.js | 27 +- .../release-tracks/versioning-service.js | 6 + .../release-tracks/virtual-track-service.js | 3 + .../primary-revision-integrity.spec.js | 320 ++++++++++++++++++ .../release-tracks-release.spec.js | 46 ++- .../release-tracks/snapshot-history.spec.js | 61 +++- app/tests/middleware/error-handler.spec.js | 54 +++ docs/developer/FRONTEND_TODO.md | 45 +++ docs/developer/TODO.md | 32 +- .../developer/release-tracks/bundle-export.md | 9 +- .../release-tracks/implementation-notes.md | 27 ++ docs/user/release-tracks/api-reference.md | 6 + docs/user/release-tracks/output-formats.md | 7 + docs/user/release-tracks/release-workflow.md | 13 +- 23 files changed, 915 insertions(+), 157 deletions(-) create mode 100644 app/services/release-tracks/primary-revision-service.js create mode 100644 app/tests/api/release-tracks/primary-revision-integrity.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 243ef13d..5ba8efe7 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -676,3 +676,32 @@ components: type: string format: date-time description: 'UTC occurrence timestamp; also serves as the idempotency key' + + object-revision-reference: + type: object + required: + - object_ref + - object_modified + properties: + object_ref: + type: string + description: 'STIX object ID' + object_modified: + type: string + format: date-time + description: 'Exact STIX revision timestamp' + + object-revision-error: + type: object + required: + - message + - missing_references + properties: + message: + type: string + description: 'Whether request input or persisted primary content failed validation' + missing_references: + type: array + minItems: 1 + items: + $ref: '#/components/schemas/object-revision-reference' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 9b1c4c64..daf6ff1f 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -256,8 +256,8 @@ paths: responses: '201': description: 'Release track created from bundle' - '501': - description: 'Not yet implemented' + '400': + description: 'A bundle primary object is unsupported, invalid, or cannot be persisted' /api/release-tracks/import: post: @@ -354,7 +354,7 @@ paths: '200': description: 'Contents updated successfully' '400': - description: 'Track is virtual or the contents request is invalid' + description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' /api/release-tracks/{id}/clone: post: @@ -374,6 +374,12 @@ paths: responses: '201': description: 'Release track cloned successfully' + '409': + description: 'The source snapshot references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' /api/release-tracks/{id}/snapshots/latest/release: post: @@ -412,7 +418,7 @@ paths: '400': description: 'Invalid release request' '409': - description: 'Already released or conflicting snapshot' + description: 'Already released, conflicting snapshot, or missing persisted primary revisions' /api/release-tracks/{id}/snapshots/latest/release/preview: get: @@ -488,7 +494,7 @@ paths: '200': description: 'Release preview generated' '409': - description: 'Virtual draft is unmaterialized or release is blocked by a conflict' + description: 'Virtual draft is unmaterialized, release is blocked by a conflict, or persisted primary revisions are missing' '501': description: 'Requested format is not yet implemented' @@ -558,6 +564,8 @@ paths: responses: '200': description: 'Candidates added successfully' + '400': + description: 'A requested exact or latest object revision does not exist' /api/release-tracks/{id}/candidates/review: post: @@ -654,6 +662,8 @@ paths: responses: '200': description: 'Candidate version pin updated successfully' + '400': + description: 'The requested replacement revision does not exist' # ============================================================================= # Staged objects @@ -873,6 +883,12 @@ paths: description: 'Virtual snapshot created successfully' '400': description: 'Track is not virtual or cannot resolve its composition' + '409': + description: 'A resolved component snapshot references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' /api/release-tracks/{id}/virtual/quarantine/promote: post: @@ -906,6 +922,12 @@ paths: description: 'Track is not virtual or the request body is invalid' '404': description: 'The selected exact revision is not quarantined' + '409': + description: 'The resulting virtual snapshot would reference missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' # ============================================================================= # Snapshot-specific operations @@ -1061,6 +1083,12 @@ paths: $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '404': description: 'Release track not found' + '409': + description: 'The selected snapshot representation references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' '501': description: 'Requested format is not yet implemented' @@ -1148,6 +1176,12 @@ paths: description: 'Snapshot retrieved successfully' '404': description: 'Snapshot not found' + '409': + description: 'The selected snapshot representation references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' '501': description: 'Requested format is not yet implemented' @@ -1234,7 +1268,7 @@ paths: '200': description: 'Contents updated successfully' '400': - description: 'Track is virtual or the contents request is invalid' + description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' /api/release-tracks/{id}/snapshots/{modified}/clone: post: @@ -1259,6 +1293,12 @@ paths: responses: '201': description: 'Release track cloned successfully' + '409': + description: 'The source snapshot references missing primary revisions' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' /api/release-tracks/{id}/snapshots/{modified}/release: post: @@ -1302,7 +1342,7 @@ paths: '400': description: 'Invalid release request' '409': - description: 'Already released or conflicting snapshot' + description: 'Already released, conflicting snapshot, or missing persisted primary revisions' /api/release-tracks/{id}/snapshots/{modified}/release/preview: get: @@ -1374,6 +1414,6 @@ paths: '200': description: 'Release preview generated' '409': - description: 'Virtual draft is unmaterialized or release is blocked by a conflict' + description: 'Release is blocked because the draft is unmaterialized, conflicting, or references missing primary revisions' '501': description: 'Requested format is not yet implemented' diff --git a/app/exceptions/index.js b/app/exceptions/index.js index a4db28ef..c387fd83 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -307,6 +307,24 @@ class DuplicateReleaseVersionError extends CustomError { } } +class InvalidObjectRevisionError extends CustomError { + constructor(missingReferences, options = {}) { + super('One or more object revisions do not exist', { + ...options, + missing_references: missingReferences, + }); + } +} + +class ReleaseContentIntegrityError extends CustomError { + constructor(missingReferences, options = {}) { + super('Release-track primary content is incomplete', { + ...options, + missing_references: missingReferences, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -397,11 +415,13 @@ module.exports = { //** Version control errors */ AlreadyReleasedError, DuplicateReleaseVersionError, + InvalidObjectRevisionError, TaggedSnapshotDeletionError, InvalidVersionError, //** Release track errors */ ReleaseConflictError, + ReleaseContentIntegrityError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 6e4e292d..0911c02e 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -37,9 +37,11 @@ const { SelfRevocationError, AlreadyReleasedError, DuplicateReleaseVersionError, + InvalidObjectRevisionError, TaggedSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, + ReleaseContentIntegrityError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, @@ -106,6 +108,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof ValidationError || err instanceof MitreIdentityWriteError || err instanceof InvalidVersionError || + err instanceof InvalidObjectRevisionError || err instanceof NoTaggedSnapshotsError || err instanceof InvalidComponentTypeError ) { @@ -136,6 +139,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof DuplicateReleaseVersionError || err instanceof TaggedSnapshotDeletionError || err instanceof ReleaseConflictError || + err instanceof ReleaseContentIntegrityError || err instanceof VirtualSnapshotNotMaterializedError || err instanceof MemberPinnedRevisionError || err instanceof ObjectHasValidationIssuesError || diff --git a/app/services/release-tracks/bundle-import-service.js b/app/services/release-tracks/bundle-import-service.js index 412fcf51..6740c064 100644 --- a/app/services/release-tracks/bundle-import-service.js +++ b/app/services/release-tracks/bundle-import-service.js @@ -20,6 +20,7 @@ const types = require('../../lib/types'); const logger = require('../../lib/logger'); const snapshotService = require('./snapshot-service'); +const primaryRevisionService = require('./primary-revision-service'); const { BadRequestError, DuplicateIdError } = require('../../exceptions'); // --------------------------------------------------------------------------- @@ -134,18 +135,24 @@ function sortByDependencyOrder(objects) { async function importObject(stixObj, serviceMap) { const service = serviceMap[stixObj.type]; if (!service) { - logger.warn( - `BundleImportService: No service for type "${stixObj.type}", skipping "${stixObj.id}"`, - ); - return { imported: false, ref: null }; + throw new BadRequestError({ + message: 'Bundle contains an unsupported primary object type', + details: { + object_ref: stixObj.id, + type: stixObj.type, + }, + }); } // Validate required fields if (!stixObj.id || !stixObj.modified) { - logger.warn( - `BundleImportService: Object missing id or modified, skipping: ${JSON.stringify({ id: stixObj.id, type: stixObj.type })}`, - ); - return { imported: false, ref: null }; + throw new BadRequestError({ + message: 'Bundle primary object is missing an exact revision identifier', + details: { + object_ref: stixObj.id, + type: stixObj.type, + }, + }); } const ref = { @@ -189,9 +196,18 @@ async function importObject(stixObj, serviceMap) { return { imported: false, ref }; } - // Non-duplicate errors are logged but don't abort the entire import + // A track must never retain a primary reference whose object failed to + // import. Previously this path logged the failure and returned the ref. logger.error(`BundleImportService: Failed to import "${stixObj.id}":`, err); - return { imported: false, ref }; + throw new BadRequestError({ + message: 'Failed to import a bundle primary object', + details: { + object_ref: stixObj.id, + object_modified: stixObj.modified, + type: stixObj.type, + }, + cause: err, + }); } } @@ -280,6 +296,12 @@ exports.createTrackFromBundle = async function createTrackFromBundle(bundleData) ); } + // Validate the authoritative member list before creating the dynamic track + // collection or registry entry. Successfully imported standalone objects + // remain available if a later primary is invalid, but no partial track is + // persisted. + memberEntries = (await primaryRevisionService.assertRequestEntries(memberEntries)).entries; + // ------------------------------------------------------------------ // Step 4: Create the release track // ------------------------------------------------------------------ diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index b02e8447..fb469a4b 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -25,48 +25,15 @@ const EventBus = require('../../lib/event-bus'); const Events = require('../../lib/event-constants'); const { selectRelationshipsForBundle } = require('../../lib/stix-bundle-relationships'); const revisionReference = require('../../lib/release-tracks/revision-reference'); +const primaryRevisionService = require('./primary-revision-service'); const { bundleTransformSchema, workbenchTransformSchema, filesystemStoreTransformSchema, } = require('../../lib/release-tracks/export-schemas'); -// --------------------------------------------------------------------------- -// Repository map — lazy-loaded to avoid circular dependency issues at startup. -// -// Maps STIX type prefixes to their corresponding repositories so we can -// batch-query each repository's `findManyByIdAndModified` in parallel. -// --------------------------------------------------------------------------- - -let _repoMap = null; - function getRepositoryMap() { - if (_repoMap) return _repoMap; - - _repoMap = { - [types.Technique]: require('../../repository/techniques-repository'), - [types.Tactic]: require('../../repository/tactics-repository'), - [types.Group]: require('../../repository/groups-repository'), - [types.Campaign]: require('../../repository/campaigns-repository'), - [types.Mitigation]: require('../../repository/mitigations-repository'), - [types.Matrix]: require('../../repository/matrix-repository'), - [types.Relationship]: require('../../repository/relationships-repository'), - [types.MarkingDefinition]: require('../../repository/marking-definitions-repository'), - [types.Identity]: require('../../repository/identities-repository'), - [types.Note]: require('../../repository/notes-repository'), - [types.DataSource]: require('../../repository/data-sources-repository'), - [types.DataComponent]: require('../../repository/data-components-repository'), - [types.Asset]: require('../../repository/assets-repository'), - [types.Analytic]: require('../../repository/analytics-repository'), - [types.DetectionStrategy]: require('../../repository/detection-strategies-repository'), - }; - - // Software types share a single repository - const softwareRepo = require('../../repository/software-repository'); - _repoMap[types.Malware] = softwareRepo; - _repoMap[types.Tool] = softwareRepo; - - return _repoMap; + return primaryRevisionService.getRepositoryMap(); } // ============================================================================= @@ -83,47 +50,7 @@ function getRepositoryMap() { * @returns {Promise>} Full Mongoose lean documents ({ stix, workspace, ... }) */ exports.hydrateMembers = async function hydrateMembers(entries) { - if (!entries || entries.length === 0) return []; - const resolvedEntries = await revisionReference.resolveEntries(entries); - const uniqueResolvedEntries = []; - const seenResolvedEntries = new Set(); - for (const entry of resolvedEntries) { - const key = `${entry.object_ref}::` + revisionReference.modifiedKey(entry.object_modified); - if (seenResolvedEntries.has(key)) continue; - seenResolvedEntries.add(key); - uniqueResolvedEntries.push(entry); - } - - // Group entries by STIX type prefix - const byType = {}; - for (const entry of uniqueResolvedEntries) { - const type = entry.object_ref.split('--')[0]; - if (!byType[type]) byType[type] = []; - byType[type].push(entry); - } - - const repoMap = getRepositoryMap(); - const hydrated = []; - - await Promise.all( - Object.entries(byType).map(async ([type, refs]) => { - const repo = repoMap[type]; - if (!repo) { - logger.warn( - `ExportService: No repository for type "${type}", skipping ${refs.length} object(s)`, - ); - return; - } - try { - const docs = await repo.findManyByIdAndModified(refs); - hydrated.push(...docs); - } catch (err) { - logger.error(`ExportService: Failed to hydrate ${refs.length} "${type}" object(s):`, err); - } - }), - ); - - return hydrated; + return (await primaryRevisionService.assertStoredEntries(entries)).documents; }; // ============================================================================= diff --git a/app/services/release-tracks/primary-revision-service.js b/app/services/release-tracks/primary-revision-service.js new file mode 100644 index 00000000..9b12b7ea --- /dev/null +++ b/app/services/release-tracks/primary-revision-service.js @@ -0,0 +1,151 @@ +'use strict'; + +// Authoritative hydration and existence validation for release-track primary +// content. Cross-service reads are intentionally centralized here so ingress, +// release planning, virtual materialization, import, and export share one +// exact-revision invariant. + +const types = require('../../lib/types'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); +const { InvalidObjectRevisionError, ReleaseContentIntegrityError } = require('../../exceptions'); + +let repositoryMap; + +function getRepositoryMap() { + if (repositoryMap) return repositoryMap; + + repositoryMap = { + [types.Technique]: require('../../repository/techniques-repository'), + [types.Tactic]: require('../../repository/tactics-repository'), + [types.Group]: require('../../repository/groups-repository'), + [types.Campaign]: require('../../repository/campaigns-repository'), + [types.Mitigation]: require('../../repository/mitigations-repository'), + [types.Matrix]: require('../../repository/matrix-repository'), + [types.Relationship]: require('../../repository/relationships-repository'), + [types.MarkingDefinition]: require('../../repository/marking-definitions-repository'), + [types.Identity]: require('../../repository/identities-repository'), + [types.Note]: require('../../repository/notes-repository'), + [types.DataSource]: require('../../repository/data-sources-repository'), + [types.DataComponent]: require('../../repository/data-components-repository'), + [types.Asset]: require('../../repository/assets-repository'), + [types.Analytic]: require('../../repository/analytics-repository'), + [types.DetectionStrategy]: require('../../repository/detection-strategies-repository'), + }; + + const softwareRepo = require('../../repository/software-repository'); + repositoryMap[types.Malware] = softwareRepo; + repositoryMap[types.Tool] = softwareRepo; + + return repositoryMap; +} + +function revisionKey(entry) { + return `${entry.object_ref}::${revisionReference.modifiedKey(entry.object_modified)}`; +} + +function serializeReference(entry) { + const modified = new Date(entry.object_modified); + return { + object_ref: entry.object_ref, + object_modified: Number.isNaN(modified.getTime()) + ? String(entry.object_modified) + : modified.toISOString(), + }; +} + +function uniqueEntries(entries) { + const seen = new Set(); + return entries.filter((entry) => { + const key = revisionKey(entry); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +/** + * Resolve dynamic selectors and hydrate every unique exact revision. + * Repository failures deliberately propagate; only a successful query with a + * missing result is classified as unresolved primary content. + */ +async function hydrateEntries(entries) { + if (!entries || entries.length === 0) { + return { entries: [], documents: [], missing: [] }; + } + + const resolvedEntries = uniqueEntries(await revisionReference.resolveEntries(entries)); + const byType = new Map(); + for (const entry of resolvedEntries) { + const type = entry.object_ref.split('--')[0]; + if (!byType.has(type)) byType.set(type, []); + byType.get(type).push(entry); + } + + const documentsByRevision = new Map(); + const unsupported = []; + const repositories = getRepositoryMap(); + + await Promise.all( + Array.from(byType.entries()).map(async ([type, refs]) => { + const repository = repositories[type]; + if (!repository) { + unsupported.push(...refs); + return; + } + + const documents = await repository.findManyByIdAndModified(refs); + for (const document of documents) { + documentsByRevision.set( + revisionKey({ + object_ref: document.stix.id, + object_modified: document.stix.modified, + }), + document, + ); + } + }), + ); + + const missing = [ + ...unsupported, + ...resolvedEntries.filter((entry) => !documentsByRevision.has(revisionKey(entry))), + ] + .filter( + (entry, index, all) => + all.findIndex((item) => revisionKey(item) === revisionKey(entry)) === index, + ) + .map(serializeReference); + const documents = resolvedEntries + .map((entry) => documentsByRevision.get(revisionKey(entry))) + .filter(Boolean); + + return { entries: resolvedEntries, documents, missing }; +} + +async function assertRequestEntries(entries) { + const result = await hydrateEntries(entries); + if (result.missing.length > 0) { + throw new InvalidObjectRevisionError(result.missing); + } + return result; +} + +async function assertStoredEntries(entries) { + const result = await hydrateEntries(entries); + if (result.missing.length > 0) { + throw new ReleaseContentIntegrityError(result.missing); + } + return result; +} + +module.exports = { + getRepositoryMap, + hydrateEntries, + assertRequestEntries, + assertStoredEntries, + _private: { + revisionKey, + serializeReference, + uniqueEntries, + }, +}; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 6aa14bb6..aa2bc8de 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -24,6 +24,7 @@ const standardTrackService = require('./standard-track-service'); const versioningService = require('./versioning-service'); const virtualTrackService = require('./virtual-track-service'); const exportService = require('./export-service'); +const primaryRevisionService = require('./primary-revision-service'); const ephemeralService = require('./ephemeral-service'); const bundleImportService = require('./bundle-import-service'); const memberSyncService = require('./member-sync-service'); @@ -173,6 +174,12 @@ function filterSnapshotTiers(snapshot, include) { } async function formatWorkbenchSnapshot(snapshot, options) { + const include = options?.include; + const selectedTiers = + !include || include === 'all' ? TIER_NAMES : [...new Set(['members', include])]; + await primaryRevisionService.assertStoredEntries( + selectedTiers.flatMap((tierName) => snapshot[tierName] || []), + ); const enriched = await addObjectInfoToSnapshot(snapshot); return filterSnapshotTiers(enriched, options?.include); } diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index b148a061..57bbffa5 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -20,8 +20,8 @@ const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); const versionUtils = require('../../lib/release-tracks/version-utils'); -const objectResolver = require('../../lib/release-tracks/object-resolver'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); +const primaryRevisionService = require('./primary-revision-service'); const { TrackNotFoundError, NotFoundError, @@ -75,23 +75,12 @@ function assertStandardTrack(snapshot) { * @returns {Promise>} */ async function resolveContentsMembers(contents) { - const latestByObjectRef = new Map(); - const resolveLatest = (objectRef) => { - if (!latestByObjectRef.has(objectRef)) { - latestByObjectRef.set(objectRef, objectResolver.resolveLatestModified(objectRef)); - } - return latestByObjectRef.get(objectRef); - }; - - return Promise.all( - contents.map(async (entry) => ({ - object_ref: entry.obj_ref, - object_modified: - entry.obj_modified === 'latest' - ? await resolveLatest(entry.obj_ref) - : new Date(entry.obj_modified), - })), - ); + const requested = contents.map((entry) => ({ + object_ref: entry.obj_ref, + object_modified: + entry.obj_modified === 'latest' ? entry.obj_modified : new Date(entry.obj_modified), + })); + return (await primaryRevisionService.assertRequestEntries(requested)).entries; } /** @@ -406,6 +395,9 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { delete clone.scheduled_materialization; const normalized = tierRevisionInvariant.normalizeSnapshot(clone); + await primaryRevisionService.assertStoredEntries( + tierRevisionInvariant.TIER_PRECEDENCE.flatMap((tier) => normalized.snapshot[tier] || []), + ); await modelFactory.ensureIndexes(newTrackId); const saved = await dynamicRepo.saveSnapshot(newTrackId, normalized.snapshot); diff --git a/app/services/release-tracks/standard-track-service.js b/app/services/release-tracks/standard-track-service.js index 47655146..60a94f0d 100644 --- a/app/services/release-tracks/standard-track-service.js +++ b/app/services/release-tracks/standard-track-service.js @@ -13,7 +13,7 @@ // ============================================================================= const snapshotService = require('./snapshot-service'); -const objectResolver = require('../../lib/release-tracks/object-resolver'); +const primaryRevisionService = require('./primary-revision-service'); const revisionReference = require('../../lib/release-tracks/revision-reference'); const conflictResolution = require('../../lib/release-tracks/conflict-resolution'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); @@ -103,16 +103,13 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId for (const raw of objectRefs) { const entry = normalizeObjectRef(raw); - // `latest` is a dynamic workflow-tier selector. Resolve it once here to - // validate that the object exists, but preserve the selector until the - // staged entry is frozen by a release operation. - let modified; - if (!entry.modified || entry.modified === 'latest') { - await objectResolver.resolveLatestModified(entry.id); - modified = revisionReference.LATEST; - } else { - modified = new Date(entry.modified); - } + // `latest` remains dynamic through the candidate/staged workflow. The + // shared primary-revision boundary resolves it only for existence + // validation and does not mutate the persisted selector. + const modified = + !entry.modified || entry.modified === 'latest' + ? revisionReference.LATEST + : new Date(entry.modified); const revision = { object_ref: entry.id, object_modified: modified }; const revisionKey = tierRevisionInvariant.revisionKey(revision); @@ -140,6 +137,8 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId : source; } + await primaryRevisionService.assertRequestEntries(newEntries); + // Same-object conflicts (the object_ref is already pinned in candidates at // a different revision) are resolved by the into_candidates policy. const conflictPolicy = source.config?.promotion_conflicts?.into_candidates || 'prefer_latest'; @@ -382,16 +381,18 @@ exports.updateCandidateVersion = async function updateCandidateVersion(trackId, const existingCandidates = source.candidates || []; let found = false; + let updatedEntry; const updatedCandidates = existingCandidates.map((candidate) => { if ( candidate.object_ref === objectRef && revisionReference.sameModified(candidate.object_modified, data.old_modified) ) { found = true; - return { + updatedEntry = { ...candidate, object_modified: revisionReference.normalize(data.new_modified), }; + return updatedEntry; } return candidate; }); @@ -404,6 +405,8 @@ exports.updateCandidateVersion = async function updateCandidateVersion(trackId, }); } + await primaryRevisionService.assertRequestEntries([updatedEntry]); + const snapshot = await snapshotService.cloneSnapshot(trackId, source, { candidates: updatedCandidates, }); diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 857cb238..6f902d2b 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -11,6 +11,7 @@ const conflictResolution = require('../../lib/release-tracks/conflict-resolution const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const revisionReference = require('../../lib/release-tracks/revision-reference'); const releaseHistoryService = require('./release-history-service'); +const primaryRevisionService = require('./primary-revision-service'); const logger = require('../../lib/logger'); const { AlreadyReleasedError, @@ -261,6 +262,11 @@ async function planLoadedSnapshot(trackId, snapshot, options) { } : snapshot; + await primaryRevisionService.assertStoredEntries([ + ...(releaseInput.members || []), + ...(releaseInput.staged || []), + ]); + return planRelease( trackId, releaseInput, diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 2697a63b..58a2bf30 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -17,6 +17,7 @@ // ============================================================================= const snapshotService = require('./snapshot-service'); +const primaryRevisionService = require('./primary-revision-service'); const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); const deduplicationStrategies = require('../../lib/release-tracks/deduplication-strategies'); @@ -488,6 +489,7 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op source, registryMap, ); + await primaryRevisionService.assertStoredEntries([...members, ...quarantined]); // Build overrides for the new snapshot const overrides = { @@ -559,6 +561,7 @@ exports.promoteQuarantinedObject = async function promoteQuarantinedObject(track const quarantine = (source.quarantine || []).filter( (entry) => entry.object_ref !== selected.object_ref, ); + await primaryRevisionService.assertStoredEntries([...members, ...quarantine]); const snapshot = await snapshotService.cloneSnapshot(trackId, source, { members, diff --git a/app/tests/api/release-tracks/primary-revision-integrity.spec.js b/app/tests/api/release-tracks/primary-revision-integrity.spec.js new file mode 100644 index 00000000..2b32c0a4 --- /dev/null +++ b/app/tests/api/release-tracks/primary-revision-integrity.spec.js @@ -0,0 +1,320 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const sinon = require('sinon'); +const { v4: uuidv4 } = require('uuid'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const Technique = require('../../../models/technique-model'); +const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const techniquesRepo = require('../../../repository/techniques-repository'); +const { DatabaseError } = require('../../../exceptions'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track primary revision integrity API', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + async function get(path, status = 200) { + return (await api('get', path, undefined, status)).body; + } + + async function createTechnique(name, previous) { + return post('/api/techniques', buildTechnique(name, previous), 201); + } + + async function createTrack(name, type = 'standard', extra = {}) { + return post('/api/release-tracks/new', { name, type, ...extra }, 201); + } + + function missingRevision(objectRef = `attack-pattern--${uuidv4()}`) { + return { + object_ref: objectRef, + object_modified: '2026-01-01T00:00:00.000Z', + }; + } + + async function deleteTechniqueRevision(technique) { + await Technique.deleteOne({ + 'stix.id': technique.stix.id, + 'stix.modified': new Date(technique.stix.modified), + }); + } + + it('rejects nonexistent exact candidate pins without creating a snapshot', async function () { + const track = await createTrack('Reject Missing Candidate'); + const missing = missingRevision(); + + const response = await api( + 'post', + `/api/release-tracks/${track.id}/candidates`, + { + object_refs: [{ id: missing.object_ref, modified: missing.object_modified }], + }, + 400, + ); + expect(response.body).toEqual({ + message: 'One or more object revisions do not exist', + missing_references: [missing], + }); + + expect((await dynamicRepo.getAllSnapshots(track.id)).pagination.total).toBe(1); + }); + + it('rejects a candidate pin update to a nonexistent revision', async function () { + const technique = await createTechnique('Reject Missing Candidate Update'); + const track = await createTrack('Reject Missing Candidate Update Track'); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + const missing = missingRevision(technique.stix.id); + + const response = await api( + 'post', + `/api/release-tracks/${track.id}/candidates/${technique.stix.id}/update-version`, + { + old_modified: technique.stix.modified, + new_modified: missing.object_modified, + }, + 400, + ); + expect(response.body.missing_references).toEqual([missing]); + + const latest = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(latest.candidates[0].object_modified).toBe(technique.stix.modified); + }); + + it('rejects direct member replacement atomically when one exact revision is missing', async function () { + const technique = await createTechnique('Reject Missing Direct Member'); + const track = await createTrack('Reject Missing Direct Member Track'); + const missing = missingRevision(); + + const response = await api( + 'post', + `/api/release-tracks/${track.id}/contents`, + { + x_mitre_contents: [ + { obj_ref: technique.stix.id, obj_modified: technique.stix.modified }, + { obj_ref: missing.object_ref, obj_modified: missing.object_modified }, + ], + }, + 400, + ); + expect(response.body.missing_references).toEqual([missing]); + expect((await dynamicRepo.getAllSnapshots(track.id)).pagination.total).toBe(1); + }); + + it('fails preview and release when a staged revision was deleted', async function () { + const technique = await createTechnique('Deleted Staged Revision'); + const track = await createTrack('Deleted Staged Revision Track'); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [technique.stix.id], + }); + await deleteTechniqueRevision(technique); + + const expectedMissing = { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }; + const preview = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest/release/preview`, + undefined, + 409, + ); + expect(preview.body).toEqual({ + message: 'Release-track primary content is incomplete', + missing_references: [expectedMissing], + }); + + const release = await api( + 'post', + `/api/release-tracks/${track.id}/snapshots/latest/release`, + {}, + 409, + ); + expect(release.body.missing_references).toEqual([expectedMissing]); + expect( + (await dynamicRepo.getAllSnapshots(track.id, { taggedOnly: true })).pagination.total, + ).toBe(0); + }); + + it('rejects cloning and export when a stored primary member is missing', async function () { + const technique = await createTechnique('Missing Stored Member'); + const track = await createTrack('Missing Stored Member Track'); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + await deleteTechniqueRevision(technique); + const registryCount = await ReleaseTrackRegistry.countDocuments(); + + const bundle = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest?format=bundle`, + undefined, + 409, + ); + expect(bundle.body.missing_references).toEqual([ + { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }, + ]); + + const workbench = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest`, + undefined, + 409, + ); + expect(workbench.body.missing_references).toEqual(bundle.body.missing_references); + + await api('post', `/api/release-tracks/${track.id}/clone`, {}, 409); + expect(await ReleaseTrackRegistry.countDocuments()).toBe(registryCount); + }); + + it('propagates repository hydration failures instead of returning a partial export', async function () { + const technique = await createTechnique('Failed Primary Hydration'); + const track = await createTrack('Failed Primary Hydration Track'); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + const hydrationStub = sinon + .stub(techniquesRepo, 'findManyByIdAndModified') + .rejects(new DatabaseError(new Error('injected hydration failure'))); + + try { + await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest?format=bundle`, + undefined, + 500, + ); + } finally { + hydrationStub.restore(); + } + }); + + it('aborts virtual materialization when a component member is missing', async function () { + const technique = await createTechnique('Missing Virtual Component Member'); + const component = await createTrack('Missing Virtual Component'); + await post(`/api/release-tracks/${component.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { + version: '1.0', + }); + const virtual = await createTrack('Missing Virtual Primary', 'virtual', { + composition: { + component_tracks: [ + { + track_id: component.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + }, + }); + await deleteTechniqueRevision(technique); + + const response = await api( + 'post', + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + {}, + 409, + ); + expect(response.body.missing_references).toEqual([ + { + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }, + ]); + expect((await dynamicRepo.getAllSnapshots(virtual.id)).pagination.total).toBe(1); + }); + + it('does not create a track when any bundle primary object cannot be imported', async function () { + const technique = await createTechnique('Existing Bundle Primary'); + const registryCount = await ReleaseTrackRegistry.countDocuments(); + const unsupported = { + type: 'x-unsupported-primary', + id: `x-unsupported-primary--${uuidv4()}`, + modified: '2026-01-01T00:00:00.000Z', + }; + + const response = await api( + 'post', + '/api/release-tracks/new-from-bundle', + { + type: 'bundle', + id: `bundle--${uuidv4()}`, + objects: [technique.stix, unsupported], + }, + 400, + ); + expect(response.body).toMatchObject({ + message: 'Bundle contains an unsupported primary object type', + details: { + object_ref: unsupported.id, + type: unsupported.type, + }, + }); + expect(await ReleaseTrackRegistry.countDocuments()).toBe(registryCount); + }); +}); diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 6362eff5..72f862dc 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -529,30 +529,41 @@ describe('Release-track release planning and commit API', function () { }); it('compares the latest virtual draft with its preceding tagged release', async function () { + const updatedOld = ( + await post('/api/techniques', buildTechnique('Virtual Preview Updated Old'), 201) + ).body; + const updatedNew = ( + await post('/api/techniques', buildTechnique('Virtual Preview Updated New', updatedOld), 201) + ).body; + const removed = (await post('/api/techniques', buildTechnique('Virtual Preview Removed'), 201)) + .body; + const added = (await post('/api/techniques', buildTechnique('Virtual Preview Added'), 201)) + .body; + const quarantined = ( + await post('/api/techniques', buildTechnique('Virtual Preview Quarantined'), 201) + ).body; const track = await createTrack('Virtual Release Preview', 'virtual'); const created = new Date(track.modified); const taggedModified = new Date(created.getTime() + 1000); const draftModified = new Date(created.getTime() + 2000); - const oldRevision = new Date(created.getTime() - 2000); - const newRevision = new Date(created.getTime() - 1000); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: taggedModified, version: '1.0', members: [ - memberEntry(virtualObjectRefs[0], oldRevision), - memberEntry(virtualObjectRefs[1], oldRevision), + memberEntry(updatedOld.stix.id, updatedOld.stix.modified), + memberEntry(removed.stix.id, removed.stix.modified), ], - quarantine: [quarantineEntry(virtualObjectRefs[3], oldRevision, track.id)], + quarantine: [quarantineEntry(quarantined.stix.id, quarantined.stix.modified, track.id)], }); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: draftModified, version: null, members: [ - memberEntry(virtualObjectRefs[0], newRevision), - memberEntry(virtualObjectRefs[2], newRevision), + memberEntry(updatedNew.stix.id, updatedNew.stix.modified), + memberEntry(added.stix.id, added.stix.modified), ], quarantine: [], composition_resolution: compositionResolution(draftModified), @@ -586,32 +597,43 @@ describe('Release-track release planning and commit API', function () { }); it('compares a historical virtual draft with the tagged release that preceded it', async function () { + const updatedOld = ( + await post('/api/techniques', buildTechnique('Historical Virtual Updated Old'), 201) + ).body; + const updatedNew = ( + await post( + '/api/techniques', + buildTechnique('Historical Virtual Updated New', updatedOld), + 201, + ) + ).body; + const laterMember = ( + await post('/api/techniques', buildTechnique('Historical Virtual Later Member'), 201) + ).body; const track = await createTrack('Historical Virtual Release Preview', 'virtual'); const created = new Date(track.modified); const firstTaggedModified = new Date(created.getTime() + 1000); const historicalDraftModified = new Date(created.getTime() + 2000); const laterTaggedModified = new Date(created.getTime() + 3000); - const oldRevision = new Date(created.getTime() - 2000); - const newRevision = new Date(created.getTime() - 1000); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: firstTaggedModified, version: '1.0', - members: [memberEntry(virtualObjectRefs[0], oldRevision)], + members: [memberEntry(updatedOld.stix.id, updatedOld.stix.modified)], }); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: historicalDraftModified, version: null, - members: [memberEntry(virtualObjectRefs[0], newRevision)], + members: [memberEntry(updatedNew.stix.id, updatedNew.stix.modified)], composition_resolution: compositionResolution(historicalDraftModified), }); await dynamicRepo.saveSnapshot(track.id, { ...snapshotBase(track), modified: laterTaggedModified, version: '2.0', - members: [memberEntry(virtualObjectRefs[3], newRevision)], + members: [memberEntry(laterMember.stix.id, laterMember.stix.modified)], }); const preview = await get( diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js index 7e1f2fe5..adc7b717 100644 --- a/app/tests/api/release-tracks/snapshot-history.spec.js +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -7,25 +7,19 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); -const objectRefs = [ - 'attack-pattern--00000000-0000-4000-8000-000000000001', - 'attack-pattern--00000000-0000-4000-8000-000000000002', - 'attack-pattern--00000000-0000-4000-8000-000000000003', - 'attack-pattern--00000000-0000-4000-8000-000000000004', - 'attack-pattern--00000000-0000-4000-8000-000000000005', - 'attack-pattern--00000000-0000-4000-8000-000000000006', -]; - -function memberEntry(index, modified) { +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; +const objectRevisions = []; + +function memberEntry(index) { return { - object_ref: objectRefs[index], - object_modified: modified, + object_ref: objectRevisions[index].id, + object_modified: objectRevisions[index].modified, }; } function stagedEntry(index, modified) { return { - ...memberEntry(index, modified), + ...memberEntry(index), object_status: 'reviewed', object_staged_at: modified, object_staged_by: 'snapshot-history-test', @@ -34,7 +28,7 @@ function stagedEntry(index, modified) { function candidateEntry(index, modified) { return { - ...memberEntry(index, modified), + ...memberEntry(index), object_status: 'work-in-progress', object_added_at: modified, object_added_by: 'snapshot-history-test', @@ -66,6 +60,9 @@ describe('GET /api/release-tracks/:id/snapshots', function () { app = await require('../../../index').initializeApp(); passportCookie = await login.loginAnonymous(app); + for (let index = 0; index < 6; index++) { + objectRevisions.push(await createTechnique(`Snapshot History Technique ${index + 1}`)); + } standardTrack = await createTrack('Snapshot History Standard', 'standard'); virtualTrack = await createTrack('Snapshot History Virtual', 'virtual'); @@ -77,7 +74,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(standardTrack), modified: standardTaggedModified, version: '1.0', - members: [memberEntry(0, standardTaggedModified), memberEntry(1, standardTaggedModified)], + members: [memberEntry(0), memberEntry(1)], staged: [stagedEntry(2, standardTaggedModified)], candidates: [ candidateEntry(3, standardTaggedModified), @@ -89,7 +86,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(standardTrack), modified: standardLatestModified, version: null, - members: [memberEntry(0, standardLatestModified)], + members: [memberEntry(0)], staged: [stagedEntry(1, standardLatestModified), stagedEntry(2, standardLatestModified)], candidates: [candidateEntry(3, standardLatestModified)], }); @@ -100,10 +97,10 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(virtualTrack), modified: virtualTaggedModified, version: '1.0', - members: [memberEntry(0, virtualTaggedModified), memberEntry(1, virtualTaggedModified)], + members: [memberEntry(0), memberEntry(1)], quarantine: [ { - ...memberEntry(2, virtualTaggedModified), + ...memberEntry(2), source_track_id: standardTrack.id, source_track_name: standardTrack.name, source_snapshot_version: '1.0', @@ -123,6 +120,34 @@ describe('GET /api/release-tracks/:id/snapshots', function () { return response.body; } + async function createTechnique(name) { + const timestamp = new Date().toISOString(); + const response = await request(app) + .post('/api/techniques') + .send({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201); + return { + id: response.body.stix.id, + modified: response.body.stix.modified, + }; + } + function get(path, status = 200) { return request(app) .get(path) diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index 0ab1829f..a553a44f 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -9,7 +9,9 @@ const { DatabaseError, DuplicateIdError, DuplicateReleaseVersionError, + InvalidObjectRevisionError, InvalidPostOperationError, + ReleaseContentIntegrityError, } = require('../../exceptions'); describe('error-handler middleware', function () { @@ -99,6 +101,58 @@ describe('error-handler middleware', function () { expect(next.called).toBe(false); }); + it('should return missing request revisions as a structured bad request', function () { + const missing = [ + { + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000001', + object_modified: '2026-01-01T00:00:00.000Z', + }, + ]; + const err = new InvalidObjectRevisionError(missing); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(400)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'One or more object revisions do not exist', + missing_references: missing, + }), + ).toBe(true); + expect(next.called).toBe(false); + }); + + it('should return missing stored revisions as a structured conflict', function () { + const missing = [ + { + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000001', + object_modified: '2026-01-01T00:00:00.000Z', + }, + ]; + const err = new ReleaseContentIntegrityError(missing); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(409)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'Release-track primary content is incomplete', + missing_references: missing, + }), + ).toBe(true); + expect(next.called).toBe(false); + }); + it('should preserve wrapped error details for DatabaseError', function () { const err = new DatabaseError(new Error('Mongo connection failed')); const res = { diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 1a1fe4b8..b0301b13 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -71,6 +71,51 @@ Done when: - Release-preview fixtures show dynamic staged input becoming exact would-be members, and committed-release fixtures contain no dynamic members. +## P0 — Surface fail-closed primary revision errors + +### [ ] Explain missing primary revisions instead of showing a generic failure + +The backend now verifies every exact `(object_ref, object_modified)` primary +reference at request ingress and again before it releases, clones, +materializes, or renders a snapshot. It no longer omits objects that could not +be hydrated. + +Two structured error cases are relevant to the UI: + +```ts +interface MissingPrimaryRevisions { + message: string; + missing_references: Array<{ + object_ref: string; + object_modified: string; + }>; +} +``` + +- HTTP `400` means the current request selected a revision that does not + exist. Candidate add/version-update and direct standard member replacement + flows should keep the dialog open, identify the missing selections, and let + the operator correct them. +- HTTP `409` means an existing draft or snapshot contains a dangling primary + reference. Snapshot retrieval, release preview/commit, cloning, virtual + materialization/quarantine promotion, and bundle export can return this + response. The UI should identify the affected revisions and explain that an + operator must repair the track/object data before continuing. + +Do not render a partial Workbench snapshot or treat a failed bundle request as +an empty export. + +Done when: + +- The release-track connector exposes `missing_references` on `400` and `409` + responses instead of flattening the response to a generic message. +- Candidate and direct-content forms keep their input state after a `400` and + highlight the missing revisions. +- Snapshot, release, clone, virtual-materialization, and export views present + an actionable integrity error for `409`. +- Tests cover multiple missing references and prove no partial snapshot or + bundle is rendered. + ## P0 — Align the Angular connector with the current routes ### [ ] Use only the explicit snapshot-retrieval endpoints diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 5ed481dd..26bb6c4e 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -29,9 +29,39 @@ Verification result (2026-07-30): - The migration preflights the union of registry IDs and canonical orphan release-track collection names before making any index changes. +### P0.2 — Make primary release membership fail closed + +- [x] Add one shared batch hydrator that resolves dynamic selectors, validates + every exact `(object_ref, object_modified)` pair, and reports all missing + primary revisions without swallowing repository failures. +- [x] Reject nonexistent exact candidate pins, candidate pin updates, direct + member replacement, track cloning, and virtual materialization before + snapshot persistence. +- [x] Revalidate existing and promoted members at release-preview and + release-commit boundaries; return a typed `409 Conflict` for corrupt stored + drafts. +- [x] Abort bundle import before creating a track when any authoritative + primary object failed to import or cannot be hydrated. +- [x] Abort bundle/workbench export when selected primary revisions cannot be + hydrated; return every missing reference instead of a partial result. +- [x] Add ingress, partial-import, deleted-staged-revision, virtual + materialization, and incomplete-export regressions. +- [x] Update user/developer documentation and run focused, lint, OpenAPI, and + complete-suite verification. + +Verification result (2026-07-30): + +- The shared integrity, release, export, virtual determinism/quarantine, and + middleware regression group passes (54); the complete release-track API + regression group passes (143). +- OpenAPI validation (2) and backend lint pass. +- The required full suite passes: OpenAPI 2, config 21, API 955, middleware + 27, and scheduler 10. +- Bruno documents the structured `400`/`409` integrity response on companion + branch `fix/release-tracks-production-readiness`. + ### Remaining prioritized recommendations -- [ ] P0.2 — Make primary release membership fail closed. - [ ] P0.3 — Make tagged-content immutability authoritative and durable. - [ ] P0.4 — Correct destructive authorization and add durable audit records. - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 074eebe9..5f9abb0c 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -104,7 +104,9 @@ Implemented in resolved for this export request, then the concrete `{object_ref, object_modified}` pairs are batch-fetched per STIX type via each repository's `findManyByIdAndModified`. The stored draft selectors are - not mutated. + not mutated. Hydration is fail-closed: if any selected primary revision is + missing, the request returns `409` with `missing_references` and emits no + partial bundle. Database failures propagate as server errors. 3. **Relationships** — the relationship service fetches the latest active relationship revisions whose `source_ref` and `target_ref` are both among the selected objects. Deprecated data-component `detects` relationships @@ -196,6 +198,11 @@ comma-separated and repeated-parameter forms reach the Zod layer, which normalizes and enforces the enums. Invalid values produce a 400 `InvalidQueryStringParameterError`. +Primary revision existence is validated separately in +`primary-revision-service.js`. This is intentionally a service-layer +invariant, because snapshot cloning, scheduled virtual materialization, and +release planning also enter through non-controller paths. + ### Regression tests - [release-tracks-bundle.spec.js](../../../app/tests/api/release-tracks/release-tracks-bundle.spec.js) diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 43bfceea..739b71f3 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -43,6 +43,33 @@ idempotently. and `version-utils.calculateNextVersion` repeats the invariant so internal release-planning callers cannot silently choose one selector. +### Primary revision integrity boundary + +`app/services/release-tracks/primary-revision-service.js` is the shared +existence and hydration boundary for primary snapshot content. It resolves +dynamic selectors, batches exact `(object_ref, object_modified)` reads by STIX +type, preserves request order, and reports every missing revision instead of +silently dropping it. + +The error contract distinguishes who can correct the problem: + +- Request ingress returns `400` with `missing_references` when candidate + selection or direct member replacement names a revision that does not exist. +- Operations over already-persisted content return `409` with + `missing_references` when a release preview/commit, track clone, virtual + materialization, quarantine promotion, or bundle export encounters a + dangling primary reference. +- Repository failures propagate as server errors. They are never interpreted + as an empty query result, because doing so could emit a partial release. + +Bundle bootstrap is also fail-closed. Every primary bundle object must have a +supported Workbench repository and must either be persisted successfully or +already exist as the exact revision being imported. The track registry and +initial snapshot are not created if any primary object fails. Import is not a +database transaction across the heterogeneous object collections, so objects +successfully created before a later failure may remain as ordinary Workbench +objects; no partial release track points at them. + ### Cross-tier revision enforcement `app/lib/release-tracks/tier-revision-invariant.js` owns selector identity diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 6a3d0584..b7dfb798 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -285,6 +285,12 @@ POST /api/release-tracks/new-from-bundle **Note:** All objects are added directly to the `members` tier. To add objects as candidates instead, use the standard [Create New Release Track](#create-new-release-track) endpoint followed by [Add Candidates](#add-candidates). +Bundle bootstrap is fail-closed. Unsupported primary object types, invalid +objects, and primary revisions that cannot be persisted cause HTTP `400`, and +the release track is not created. Objects successfully persisted before a +later object fails may remain available in Workbench, but no partial track or +snapshot is registered. + ### Import Release Track (Not Implemented) Comprehensively importing a release track would necessitate including the full snapshot history of the source release track. We don't presently have a solution for serializing an entire release track, including its snapshot history, into an atomic structure that can be exchanged between different Workbench deployments. diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index baed84c6..4a27bb59 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -47,6 +47,8 @@ shape for snapshot retrieval endpoints and is intended for the Workbench fronten response enriches them from the currently latest object revision without replacing the stored selector. - Adds UI-friendly object details to tier entries +- Fails with HTTP `409` and `missing_references` rather than returning + partially enriched tier content when a selected primary revision is missing - Suitable for Workbench UI rendering and release-track management workflows Use `include` to narrow tier arrays in `workbench` responses: @@ -100,6 +102,11 @@ Standard STIX bundle format: - If a draft export explicitly includes candidate or staged tiers, dynamic `"latest"` selectors are resolved for that export request. Tagged member contents remain exact. +- Bundle export is fail-closed for primary content. If any selected exact + revision no longer exists, the server returns HTTP `409` with every missing + `(object_ref, object_modified)` pair in `missing_references`; it never emits + a partial bundle. A repository/database failure is returned as a server + error rather than being mistaken for missing content. - Notes are never included (notes are Workbench-native objects, not STIX objects) - Suitable for external publication diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index b62b26b3..acdefd8e 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -58,6 +58,12 @@ Snapshot tagged → dynamic staged selectors resolved and exact revisions moved Snapshot exported → members reflected in stix.x_mitre_contents of the output bundle ``` +Immediately before previewing or committing a release, the server hydrates +every resulting exact member revision. If persisted track content points to a +revision that no longer exists, the operation returns HTTP `409` with +`missing_references` and does not tag the snapshot. This check protects both +standard and virtual releases from publishing incomplete primary membership. + ### STIX Freeze Solution Version pinning solves the "STIX freeze" problem: @@ -161,7 +167,9 @@ POST /api/release-tracks/:id/candidates ``` **Business Logic:** -1. Validate all object_refs exist +1. Validate that every selected exact revision exists. A missing exact pin, or + a `"latest"` selector for an object with no current revision, returns HTTP + `400` with `missing_references`; no snapshot is created. 2. Establish the `object_modified` selector: - If an ISO timestamp is provided: retain that exact revision pin - If `"latest"` is provided or `modified` is omitted: persist the dynamic @@ -171,6 +179,9 @@ POST /api/release-tracks/:id/candidates 5. If status meets `candidacy_threshold`, auto-promote to `workspace.staged` 6. Update object's `workspace.referenced_by` array +The same existence check applies when changing a candidate version pin and +when replacing a standard snapshot's member contents directly. + Importantly, candidate removal/deletion must occur separately using the `DELETE` operation: ```bash DELETE /api/release-tracks/:id/candidates From 173fff25cbeb1977df53d0138b4c1b1bb5d8db1c Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:25:07 -0400 Subject: [PATCH 33/55] test(release-tracks): evict dropped dynamic models Clear per-track Mongoose models when the in-memory database is dropped so later specs do not recreate indexes for collections that no longer exist. --- app/lib/database-in-memory.js | 17 ++++++++++++----- app/models/release-tracks/model-factory.js | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/app/lib/database-in-memory.js b/app/lib/database-in-memory.js index 66564f09..6c0efb16 100644 --- a/app/lib/database-in-memory.js +++ b/app/lib/database-in-memory.js @@ -25,7 +25,9 @@ exports.initializeConnection = async function () { // Bootstrap db connection logger.info('Mongoose attempting to connect to in memory database at ' + uri); try { - await mongoose.connect(uri); + if (mongoose.connection.readyState === 0) { + await mongoose.connect(uri); + } } catch (error) { handleError(error); } @@ -41,12 +43,17 @@ exports.initializeConnection = async function () { }; exports.closeConnection = async function () { - // Drop data and disconnect, but leave the mongod instance running for the - // next spec file. The mocha scripts run with --exit, so the process does - // not linger after the last spec. + // Drop data, but keep both mongod and the Mongoose connection alive for the + // next spec file. Disconnecting while an event listener is finishing can + // reset an otherwise unrelated Supertest request in a later suite. The + // mocha scripts run with --exit, so the process does not linger after the + // last spec. if (mongod && mongoose.connection.readyState !== 0) { await mongoose.connection.dropDatabase(); - await mongoose.connection.close(); + // Dynamic release-track collections no longer exist after the drop. + // Evict their models so the next spec does not rebuild indexes for every + // track created by all preceding specs in this process. + require('../models/release-tracks/model-factory').clearModels(); } }; diff --git a/app/models/release-tracks/model-factory.js b/app/models/release-tracks/model-factory.js index e4ba1771..c9383d7f 100644 --- a/app/models/release-tracks/model-factory.js +++ b/app/models/release-tracks/model-factory.js @@ -53,6 +53,20 @@ class ModelFactory { } } + /** + * Remove every cached dynamic release-track model. + * + * Test databases drop every dynamic collection between spec files. Keeping + * those models registered makes the next connection recreate indexes for + * every track used by every preceding spec, even though none of those + * collections still exists. + */ + clearModels() { + for (const trackId of [...this._cache.keys()]) { + this.removeModel(trackId); + } + } + /** * Ensure indexes are created on a release track's collection. * Call this after creating a new track to build the indexes defined in the schema. From a3734c1017568056ee65a87d5b9bb888c0297ad2 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:34:17 -0400 Subject: [PATCH 34/55] fix(release-tracks): make release protection durable Guard tagged revisions from authoritative snapshot history and make backref reconciliation required, durable, observable, and repairable after partial failures. --- .../definitions/components/release-tracks.yml | 22 ++ .../paths/release-tracks-paths.yml | 12 + app/exceptions/index.js | 11 + app/lib/error-handler.js | 4 +- app/lib/event-bus.js | 38 +++- .../release-track-reconciliation-model.js | 69 ++++++ app/repository/_base.repository.js | 13 ++ .../release-track-dynamic.repository.js | 37 ++++ ...release-track-reconciliation.repository.js | 105 +++++++++ app/services/meta-classes/base.service.js | 54 ++++- .../release-tracks/reconciliation-service.js | 139 ++++++++++++ .../release-tracks/snapshot-service.js | 5 +- .../tagged-membership-service.js | 68 ++++++ app/services/stix/attack-objects-service.js | 19 +- app/services/stix/relationships-service.js | 19 +- .../reconciliation-durability.spec.js | 208 ++++++++++++++++++ .../tagged-content-immutability.spec.js | 141 ++++++++++++ app/tests/middleware/error-handler.spec.js | 25 +++ docs/README.md | 1 + docs/admin/release-track-reconciliation.md | 79 +++++++ docs/developer/FRONTEND_TODO.md | 27 +++ docs/developer/TODO.md | 33 ++- docs/developer/event-bus-architecture.md | 26 ++- .../release-tracks/backref-reconciliation.md | 54 ++++- docs/user/release-tracks/object-backrefs.md | 15 +- docs/user/release-tracks/release-workflow.md | 7 + package.json | 1 + scripts/README.md | 21 ++ scripts/reconcileReleaseTrackBackrefs.js | 63 ++++++ 29 files changed, 1268 insertions(+), 48 deletions(-) create mode 100644 app/models/release-tracks/release-track-reconciliation-model.js create mode 100644 app/repository/release-tracks/release-track-reconciliation.repository.js create mode 100644 app/services/release-tracks/reconciliation-service.js create mode 100644 app/services/release-tracks/tagged-membership-service.js create mode 100644 app/tests/api/release-tracks/reconciliation-durability.spec.js create mode 100644 app/tests/api/release-tracks/tagged-content-immutability.spec.js create mode 100644 docs/admin/release-track-reconciliation.md create mode 100644 scripts/reconcileReleaseTrackBackrefs.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 5ba8efe7..1e5683a4 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -705,3 +705,25 @@ components: minItems: 1 items: $ref: '#/components/schemas/object-revision-reference' + + release-track-reconciliation-error: + type: object + required: + - message + - track_id + - reconciliation_id + properties: + message: + type: string + enum: + - 'Release-track membership protection could not be reconciled' + details: + type: string + description: 'Operator guidance; the track mutation may already be persisted' + track_id: + type: string + description: 'Release track whose object backrefs require repair' + reconciliation_id: + type: string + format: uuid + description: 'Durable reconciliation record to inspect or repair' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index daf6ff1f..4ed5c2d3 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -419,6 +419,12 @@ paths: description: 'Invalid release request' '409': description: 'Already released, conflicting snapshot, or missing persisted primary revisions' + '500': + description: 'The release may be tagged, but durable membership-protection reconciliation failed' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/snapshots/latest/release/preview: get: @@ -1343,6 +1349,12 @@ paths: description: 'Invalid release request' '409': description: 'Already released, conflicting snapshot, or missing persisted primary revisions' + '500': + description: 'The release may be tagged, but durable membership-protection reconciliation failed' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/snapshots/{modified}/release/preview: get: diff --git a/app/exceptions/index.js b/app/exceptions/index.js index c387fd83..8186a9ce 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -325,6 +325,16 @@ class ReleaseContentIntegrityError extends CustomError { } } +class ReleaseTrackReconciliationError extends CustomError { + constructor(trackId, reconciliationId, options = {}) { + super('Release-track membership protection could not be reconciled', { + ...options, + track_id: trackId, + reconciliation_id: reconciliationId, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -422,6 +432,7 @@ module.exports = { //** Release track errors */ ReleaseConflictError, ReleaseContentIntegrityError, + ReleaseTrackReconciliationError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 0911c02e..4e9d1b14 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -42,6 +42,7 @@ const { InvalidVersionError, ReleaseConflictError, ReleaseContentIntegrityError, + ReleaseTrackReconciliationError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, @@ -155,7 +156,8 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof TechniquesServiceError || err instanceof TacticsServiceError || err instanceof GenericServiceError || - err instanceof DatabaseError + err instanceof DatabaseError || + err instanceof ReleaseTrackReconciliationError ) { logger.error('Service error: %s', JSON.stringify(buildErrorResponse(err))); return res.status(500).send(buildErrorResponse(err)); diff --git a/app/lib/event-bus.js b/app/lib/event-bus.js index fa00dd3c..1766c604 100644 --- a/app/lib/event-bus.js +++ b/app/lib/event-bus.js @@ -42,7 +42,7 @@ class EventBus extends EventEmitter { * @param {object} payload - Data to pass to event handlers * @returns {Promise} */ - async emit(eventName, payload) { + async _dispatch(eventName, payload, options = {}) { const timestamp = new Date().toISOString(); // Log the event @@ -51,6 +51,12 @@ class EventBus extends EventEmitter { logger.debug(`EventBus: Emitting '${eventName}'`); const listeners = this.listeners(eventName); + if (listeners.length < (options.minimumListeners || 0)) { + throw new Error( + `Event '${eventName}' requires at least ${options.minimumListeners} listener(s); ` + + `found ${listeners.length}`, + ); + } if (listeners.length === 0) { logger.debug(`EventBus: No listeners for '${eventName}'`); return; @@ -79,12 +85,42 @@ class EventBus extends EventEmitter { logger.warn( `EventBus: ${failures.length}/${listeners.length} listeners failed for '${eventName}'`, ); + if (options.required) { + const error = new AggregateError( + failures.map((failure) => failure.reason), + `${failures.length}/${listeners.length} required listener(s) failed for '${eventName}'`, + ); + error.eventName = eventName; + error.failures = failures.map((failure) => failure.reason); + throw error; + } } // Return fulfilled handler results for callers that need them (e.g., WorkflowResult) return results.filter((r) => r.status === 'fulfilled' && r.value != null).map((r) => r.value); } + async emit(eventName, payload) { + return this._dispatch(eventName, payload); + } + + /** + * Emit an event whose listener side effects are part of the caller's + * success contract. Any listener failure rejects the emission. + * + * @param {string} eventName + * @param {object} payload + * @param {object} [options] + * @param {number} [options.minimumListeners] + * @returns {Promise} + */ + async emitRequired(eventName, payload, options = {}) { + return this._dispatch(eventName, payload, { + ...options, + required: true, + }); + } + /** * Log an event for debugging and auditing * @param {object} event - Event details diff --git a/app/models/release-tracks/release-track-reconciliation-model.js b/app/models/release-tracks/release-track-reconciliation-model.js new file mode 100644 index 00000000..397847fe --- /dev/null +++ b/app/models/release-tracks/release-track-reconciliation-model.js @@ -0,0 +1,69 @@ +'use strict'; + +const mongoose = require('mongoose'); +const { validateTrackId } = require('../../lib/release-tracks/release-track-validators'); + +const releaseTrackReconciliationSchema = new mongoose.Schema( + { + reconciliation_id: { + type: String, + required: true, + unique: true, + }, + track_id: { + type: String, + required: true, + validate: validateTrackId, + }, + requested_snapshot_modified: { + type: Date, + default: null, + }, + reconciled_snapshot_modified: { + type: Date, + default: null, + }, + source: { + type: String, + required: true, + enum: ['contents_changed', 'repair', 'full_scan'], + }, + status: { + type: String, + required: true, + enum: ['pending', 'completed', 'failed'], + default: 'pending', + }, + attempts: { + type: Number, + required: true, + default: 0, + min: 0, + }, + created_at: { + type: Date, + required: true, + }, + updated_at: { + type: Date, + required: true, + }, + completed_at: { + type: Date, + default: null, + }, + last_error: { + name: String, + message: String, + }, + }, + { + collection: 'releaseTrackReconciliations', + bufferCommands: false, + }, +); + +releaseTrackReconciliationSchema.index({ status: 1, updated_at: 1 }); +releaseTrackReconciliationSchema.index({ track_id: 1, created_at: -1 }); + +module.exports = mongoose.model('ReleaseTrackReconciliation', releaseTrackReconciliationSchema); diff --git a/app/repository/_base.repository.js b/app/repository/_base.repository.js index fd024a38..1cec5a72 100644 --- a/app/repository/_base.repository.js +++ b/app/repository/_base.repository.js @@ -591,6 +591,19 @@ class BaseRepository extends AbstractRepository { } } + /** + * Return every release-track ID present in denormalized object backrefs. + * Used only by administrative full-scan repair so deleted tracks with stale + * backrefs are included alongside registry-backed tracks. + */ + async distinctReleaseTrackIds() { + try { + return await this.model.distinct('workspace.release_tracks.id').exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + /** * Resolve specific object revisions to their document _ids. Lean, minimal * projection — used by release-track backref reconciliation. diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index f1675d5b..80926151 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -167,6 +167,43 @@ class ReleaseTrackDynamicRepository { } } + /** + * Find tagged snapshots whose members tier contains an object revision. + * Omitting objectModified matches every released revision for the STIX ID. + * This query reads the tagged snapshots themselves rather than relying on + * denormalized object backrefs or registry release metadata. + */ + async findTaggedSnapshotsContainingRevision(trackId, objectRef, objectModified) { + try { + const Model = this._getModel(trackId); + const memberMatch = { object_ref: objectRef }; + if (objectModified !== undefined) { + memberMatch.object_modified = new Date(objectModified); + } + + return await Model.find( + { + id: trackId, + version: { $type: 'string' }, + members: { $elemMatch: memberMatch }, + }, + { + id: 1, + type: 1, + name: 1, + modified: 1, + version: 1, + members: { $elemMatch: memberMatch }, + }, + ) + .sort({ modified: 1 }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async getAllSnapshots(trackId, options = {}) { try { const Model = this._getModel(trackId); diff --git a/app/repository/release-tracks/release-track-reconciliation.repository.js b/app/repository/release-tracks/release-track-reconciliation.repository.js new file mode 100644 index 00000000..154dec54 --- /dev/null +++ b/app/repository/release-tracks/release-track-reconciliation.repository.js @@ -0,0 +1,105 @@ +'use strict'; + +const { v4: uuidv4 } = require('uuid'); +const ReleaseTrackReconciliation = require('../../models/release-tracks/release-track-reconciliation-model'); +const { DatabaseError } = require('../../exceptions'); + +class ReleaseTrackReconciliationRepository { + async create({ trackId, snapshotModified, source }) { + const now = new Date(); + try { + const record = await ReleaseTrackReconciliation.create({ + reconciliation_id: uuidv4(), + track_id: trackId, + requested_snapshot_modified: snapshotModified || null, + source, + status: 'pending', + attempts: 0, + created_at: now, + updated_at: now, + }); + return record.toObject(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async startAttempt(reconciliationId) { + try { + return await ReleaseTrackReconciliation.findOneAndUpdate( + { reconciliation_id: reconciliationId }, + { + $inc: { attempts: 1 }, + $set: { + status: 'pending', + updated_at: new Date(), + completed_at: null, + last_error: null, + }, + }, + { new: true, lean: true }, + ).exec(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async complete(reconciliationId, snapshotModified) { + const now = new Date(); + try { + return await ReleaseTrackReconciliation.findOneAndUpdate( + { reconciliation_id: reconciliationId }, + { + $set: { + status: 'completed', + reconciled_snapshot_modified: snapshotModified || null, + updated_at: now, + completed_at: now, + last_error: null, + }, + }, + { new: true, lean: true }, + ).exec(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async fail(reconciliationId, error) { + try { + return await ReleaseTrackReconciliation.findOneAndUpdate( + { reconciliation_id: reconciliationId }, + { + $set: { + status: 'failed', + updated_at: new Date(), + completed_at: null, + last_error: { + name: error?.name || 'Error', + message: error?.message || String(error), + }, + }, + }, + { new: true, lean: true }, + ).exec(); + } catch (repositoryError) { + throw new DatabaseError(repositoryError); + } + } + + async findRepairable(limit = 100) { + try { + return await ReleaseTrackReconciliation.find({ + status: { $in: ['pending', 'failed'] }, + }) + .sort({ updated_at: 1, created_at: 1 }) + .limit(limit) + .lean() + .exec(); + } catch (error) { + throw new DatabaseError(error); + } + } +} + +module.exports = new ReleaseTrackReconciliationRepository(); diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 864ae03b..30a9b0c6 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -713,18 +713,56 @@ class BaseService extends ServiceWithHooks { * @param {Object} document - The stored document ({ workspace, stix }) * @param {string} operation - Verb for the error message ('updated'|'deleted') */ - static assertNotMemberPinned(document, operation) { - const memberPins = (document.workspace?.release_tracks || []).filter( + static async assertNotMemberPinned(document, operation) { + const currentMemberPins = (document.workspace?.release_tracks || []).filter( (entry) => entry.tier === 'members', ); - if (memberPins.length > 0) { + const taggedMembershipService = require('../release-tracks/tagged-membership-service'); + const taggedPins = await taggedMembershipService.findPinsForRevision( + document.stix.id, + document.stix.modified, + ); + + if (currentMemberPins.length > 0 || taggedPins.length > 0) { + const trackIds = [ + ...new Set([ + ...currentMemberPins.map((entry) => entry.id), + ...taggedPins.map((entry) => entry.track_id), + ]), + ]; throw new MemberPinnedRevisionError({ details: `Revision ${document.stix.id} (modified ` + `${new Date(document.stix.modified).toISOString()}) is pinned in the members tier of ` + - `release track(s) ${memberPins.map((entry) => entry.id).join(', ')} and cannot be ` + + `release track(s) ${trackIds.join(', ')} and cannot be ` + `${operation} in place. Create a new revision instead (set x_mitre_deprecated on a ` + `new revision to retire the object).`, + release_tracks: trackIds, + tagged_releases: taggedPins, + }); + } + } + + static async assertNoMemberPinnedVersions(stixId, currentMemberPinned, operation) { + const taggedMembershipService = require('../release-tracks/tagged-membership-service'); + const taggedPins = await taggedMembershipService.findPinsForObject(stixId); + const currentTrackIds = currentMemberPinned.flatMap((document) => + (document.workspace?.release_tracks || []) + .filter((entry) => entry.tier === 'members') + .map((entry) => entry.id), + ); + const trackIds = [ + ...new Set([...currentTrackIds, ...taggedPins.map((entry) => entry.track_id)]), + ]; + + if (trackIds.length > 0) { + throw new MemberPinnedRevisionError({ + details: + `Object ${stixId} has revision(s) pinned in the members tier of release track(s) ` + + `${trackIds.join(', ')} and cannot be ${operation}. Create a new revision instead ` + + `(set x_mitre_deprecated on a new revision to retire the object).`, + release_tracks: trackIds, + tagged_releases: taggedPins, }); } } @@ -946,7 +984,7 @@ class BaseService extends ServiceWithHooks { } // Members-pinned revisions are released content — immutable in place. - BaseService.assertNotMemberPinned(document, 'updated'); + await BaseService.assertNotMemberPinned(document, 'updated'); // TODO: diff analysis — detect field-level changes vs document // TODO: if no changes detected, short-circuit (no-op) @@ -1069,7 +1107,7 @@ class BaseService extends ServiceWithHooks { if (!existing) { return null; } - BaseService.assertNotMemberPinned(existing, 'deleted'); + await BaseService.assertNotMemberPinned(existing, 'deleted'); const document = await this.repository.findOneAndDelete(stixId, stixModified); @@ -1376,9 +1414,7 @@ class BaseService extends ServiceWithHooks { // Deleting all versions must not destroy a members-pinned revision const memberPinned = await this.repository.retrieveMemberPinnedVersionsLean(stixId); - for (const pinnedDocument of memberPinned) { - BaseService.assertNotMemberPinned(pinnedDocument, 'deleted'); - } + await BaseService.assertNoMemberPinnedVersions(stixId, memberPinned, 'deleted'); const result = await this.repository.deleteMany(stixId); if (result.deletedCount > 0) { diff --git a/app/services/release-tracks/reconciliation-service.js b/app/services/release-tracks/reconciliation-service.js new file mode 100644 index 00000000..394ea3f2 --- /dev/null +++ b/app/services/release-tracks/reconciliation-service.js @@ -0,0 +1,139 @@ +'use strict'; + +// Durable orchestration for workspace.release_tracks reconciliation. Each +// attempt is persisted before required EventBus listeners run. Repair always +// reconciles against the track's current latest snapshot, so replay is +// idempotent and cannot restore obsolete membership from an old event. + +const EventBus = require('../../lib/event-bus'); +const Events = require('../../lib/event-constants'); +const logger = require('../../lib/logger'); +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); +const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const reconciliationRepo = require('../../repository/release-tracks/release-track-reconciliation.repository'); +const attackObjectsRepo = require('../../repository/attack-objects-repository'); +const relationshipsRepo = require('../../repository/relationships-repository'); +const { ReleaseTrackReconciliationError } = require('../../exceptions'); + +// Reconciliation is also invoked by scheduler/migration tests and operator +// scripts that call the service facade without initializing Express. Loading +// both owning services guarantees the two required listeners are registered. +require('../stix/attack-objects-service'); +require('../stix/relationships-service'); + +async function dispatch(record, snapshot) { + await reconciliationRepo.startAttempt(record.reconciliation_id); + + try { + await EventBus.emitRequired( + Events.RELEASE_TRACK_CONTENTS_CHANGED, + { + trackId: record.track_id, + snapshot, + reconciliationId: record.reconciliation_id, + }, + { minimumListeners: 2 }, + ); + return await reconciliationRepo.complete(record.reconciliation_id, snapshot?.modified); + } catch (error) { + try { + await reconciliationRepo.fail(record.reconciliation_id, error); + } catch (recordError) { + logger.error( + `ReconciliationService: Failed to record reconciliation ${record.reconciliation_id} ` + + `failure: ${recordError.message}`, + ); + } + + throw new ReleaseTrackReconciliationError(record.track_id, record.reconciliation_id, { + details: + 'The release-track change was persisted, but one or more object backref protections ' + + 'failed. Run the release-track reconciliation repair command before retrying.', + cause: error, + }); + } +} + +async function currentSnapshot(trackId) { + const registry = await registryRepo.findByTrackId(trackId); + return registry ? dynamicRepo.getLatestSnapshot(trackId) : null; +} + +async function createAndDispatch(trackId, snapshot, source) { + const record = await reconciliationRepo.create({ + trackId, + snapshotModified: snapshot?.modified, + source, + }); + return dispatch(record, snapshot); +} + +exports.reconcileContentsChanged = function reconcileContentsChanged(trackId, snapshot) { + return createAndDispatch(trackId, snapshot, 'contents_changed'); +}; + +exports.repairOutstanding = async function repairOutstanding(options = {}) { + const records = await reconciliationRepo.findRepairable(options.limit || 100); + const results = []; + + for (const record of records) { + try { + const snapshot = await currentSnapshot(record.track_id); + const completed = await dispatch(record, snapshot); + results.push({ + reconciliation_id: record.reconciliation_id, + track_id: record.track_id, + status: completed.status, + }); + } catch (error) { + results.push({ + reconciliation_id: record.reconciliation_id, + track_id: record.track_id, + status: 'failed', + error: error.message, + }); + if (!options.continueOnError) throw error; + } + } + + return results; +}; + +exports.reconcileAll = async function reconcileAll(options = {}) { + const [registered, attackObjectTrackIds, relationshipTrackIds] = await Promise.all([ + registryRepo.findAll(), + attackObjectsRepo.distinctReleaseTrackIds(), + relationshipsRepo.distinctReleaseTrackIds(), + ]); + const trackIds = [ + ...new Set([ + ...registered.data.map((track) => track.track_id), + ...attackObjectTrackIds, + ...relationshipTrackIds, + ]), + ].sort(); + const results = []; + + for (const trackId of trackIds) { + try { + const snapshot = await currentSnapshot(trackId); + const completed = await createAndDispatch(trackId, snapshot, 'full_scan'); + results.push({ + reconciliation_id: completed.reconciliation_id, + track_id: trackId, + status: completed.status, + }); + } catch (error) { + results.push({ track_id: trackId, status: 'failed', error: error.message }); + if (!options.continueOnError) throw error; + } + } + + return results; +}; + +exports._private = { + createAndDispatch, + currentSnapshot, + dispatch, +}; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 57bbffa5..45ee635a 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -17,11 +17,10 @@ const registryRepo = require('../../repository/release-tracks/release-track-regi const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); const modelFactory = require('../../models/release-tracks/model-factory'); const logger = require('../../lib/logger'); -const EventBus = require('../../lib/event-bus'); -const EventConstants = require('../../lib/event-constants'); const versionUtils = require('../../lib/release-tracks/version-utils'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const primaryRevisionService = require('./primary-revision-service'); +const reconciliationService = require('./reconciliation-service'); const { TrackNotFoundError, NotFoundError, @@ -129,7 +128,7 @@ async function syncRegistryCounters(trackId) { * track (or its only snapshot) was deleted */ async function emitContentsChanged(trackId, snapshot) { - await EventBus.emit(EventConstants.RELEASE_TRACK_CONTENTS_CHANGED, { trackId, snapshot }); + await reconciliationService.reconcileContentsChanged(trackId, snapshot); } exports.emitContentsChanged = emitContentsChanged; diff --git a/app/services/release-tracks/tagged-membership-service.js b/app/services/release-tracks/tagged-membership-service.js new file mode 100644 index 00000000..ff4fcff5 --- /dev/null +++ b/app/services/release-tracks/tagged-membership-service.js @@ -0,0 +1,68 @@ +'use strict'; + +// Authoritative tagged-membership reads used by object mutation guards. +// workspace.release_tracks remains a useful denormalized current-snapshot +// pointer, but it is not authoritative for historical tagged releases and may +// be temporarily stale when reconciliation needs repair. + +const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); + +const QUERY_CONCURRENCY = 12; + +async function mapWithConcurrency(items, mapper) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + results[index] = await mapper(items[index], index); + } + } + + await Promise.all( + Array.from({ length: Math.min(QUERY_CONCURRENCY, items.length) }, () => worker()), + ); + return results; +} + +function pinFromSnapshot(snapshot) { + const member = snapshot.members[0]; + return { + track_id: snapshot.id, + track_type: snapshot.type, + track_name: snapshot.name, + version: snapshot.version, + snapshot_modified: snapshot.modified, + object_ref: member.object_ref, + object_modified: member.object_modified, + }; +} + +async function findPins(objectRef, objectModified) { + const tracks = (await registryRepo.findAll()).data; + const matchesByTrack = await mapWithConcurrency(tracks, async (track) => { + const snapshots = await dynamicRepo.findTaggedSnapshotsContainingRevision( + track.track_id, + objectRef, + objectModified, + ); + return snapshots.map(pinFromSnapshot); + }); + + return matchesByTrack.flat(); +} + +exports.findPinsForRevision = function findPinsForRevision(objectRef, objectModified) { + return findPins(objectRef, objectModified); +}; + +exports.findPinsForObject = function findPinsForObject(objectRef) { + return findPins(objectRef); +}; + +exports._private = { + mapWithConcurrency, + pinFromSnapshot, +}; diff --git a/app/services/stix/attack-objects-service.js b/app/services/stix/attack-objects-service.js index 51917ffc..eb2baf5c 100644 --- a/app/services/stix/attack-objects-service.js +++ b/app/services/stix/attack-objects-service.js @@ -234,19 +234,12 @@ class AttackObjectsService extends BaseService { */ static async handleReleaseTrackContentsChanged(payload) { const backrefReconciler = require('../../lib/release-tracks/backref-reconciler'); - - try { - await backrefReconciler.reconcile( - attackObjectsRepository, - payload.trackId, - payload.snapshot, - (objectRef) => !objectRef.startsWith('relationship--'), - ); - } catch (error) { - logger.error( - `AttackObjectsService: Error reconciling release track backrefs for ${payload.trackId}: ${error.message}`, - ); - } + return backrefReconciler.reconcile( + attackObjectsRepository, + payload.trackId, + payload.snapshot, + (objectRef) => !objectRef.startsWith('relationship--'), + ); } /** diff --git a/app/services/stix/relationships-service.js b/app/services/stix/relationships-service.js index 2e377219..8f32da1f 100644 --- a/app/services/stix/relationships-service.js +++ b/app/services/stix/relationships-service.js @@ -96,19 +96,12 @@ class RelationshipsService extends BaseService { */ static async handleReleaseTrackContentsChanged(payload) { const backrefReconciler = require('../../lib/release-tracks/backref-reconciler'); - - try { - await backrefReconciler.reconcile( - relationshipsRepository, - payload.trackId, - payload.snapshot, - (objectRef) => objectRef.startsWith('relationship--'), - ); - } catch (error) { - logger.error( - `RelationshipsService: Error reconciling release track backrefs for ${payload.trackId}: ${error.message}`, - ); - } + return backrefReconciler.reconcile( + relationshipsRepository, + payload.trackId, + payload.snapshot, + (objectRef) => objectRef.startsWith('relationship--'), + ); } /** diff --git a/app/tests/api/release-tracks/reconciliation-durability.spec.js b/app/tests/api/release-tracks/reconciliation-durability.spec.js new file mode 100644 index 00000000..2468b472 --- /dev/null +++ b/app/tests/api/release-tracks/reconciliation-durability.spec.js @@ -0,0 +1,208 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const sinon = require('sinon'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const Technique = require('../../../models/technique-model'); +const ReleaseTrackReconciliation = require('../../../models/release-tracks/release-track-reconciliation-model'); +const attackObjectsRepo = require('../../../repository/attack-objects-repository'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const reconciliationService = require('../../../services/release-tracks/reconciliation-service'); +const { DatabaseError } = require('../../../exceptions'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track durable backref reconciliation', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + afterEach(function () { + sinon.restore(); + }); + + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + async function getTechnique(technique) { + return ( + await api( + 'get', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 200, + ) + ).body; + } + + it('returns failure, persists the failed attempt, and repairs a committed release', async function () { + const technique = await post('/api/techniques', buildTechnique('Reconciliation Failure'), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Reconciliation Failure Track', type: 'standard' }, + 201, + ); + await post(`/api/release-tracks/${track.id}/candidates`, { + object_refs: [{ id: technique.stix.id, modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/candidates/promote`, { + object_refs: [technique.stix.id], + }); + + sinon + .stub(attackObjectsRepo, 'bulkWrite') + .rejects(new DatabaseError(new Error('injected backref write failure'))); + + const release = await api( + 'post', + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.0' }, + 500, + ); + expect(release.body).toMatchObject({ + message: 'Release-track membership protection could not be reconciled', + track_id: track.id, + reconciliation_id: expect.any(String), + }); + + const tagged = await dynamicRepo.getLatestTaggedSnapshot(track.id); + expect(tagged.version).toBe('1.0'); + + let record = await ReleaseTrackReconciliation.findOne({ + reconciliation_id: release.body.reconciliation_id, + }) + .lean() + .exec(); + expect(record).toMatchObject({ + track_id: track.id, + status: 'failed', + attempts: 1, + last_error: { + name: 'AggregateError', + message: expect.stringContaining('required listener'), + }, + }); + + let stored = await getTechnique(technique); + expect(stored.workspace.release_tracks).toEqual([ + expect.objectContaining({ id: track.id, tier: 'staged' }), + ]); + + sinon.restore(); + const results = await reconciliationService.repairOutstanding({ + limit: 100, + continueOnError: false, + }); + expect(results).toContainEqual({ + reconciliation_id: release.body.reconciliation_id, + track_id: track.id, + status: 'completed', + }); + + record = await ReleaseTrackReconciliation.findOne({ + reconciliation_id: release.body.reconciliation_id, + }) + .lean() + .exec(); + expect(record.status).toBe('completed'); + expect(record.attempts).toBe(2); + expect(record.completed_at).toBeInstanceOf(Date); + + stored = await getTechnique(technique); + expect(stored.workspace.release_tracks).toEqual([ + { + id: track.id, + type: 'standard', + tier: 'members', + status: 'reviewed', + }, + ]); + }); + + it('repairs legacy drift with an idempotent full scan', async function () { + const technique = await post('/api/techniques', buildTechnique('Full Scan Repair'), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Full Scan Repair Track', type: 'standard' }, + 201, + ); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + + await Technique.updateOne( + { + 'stix.id': technique.stix.id, + 'stix.modified': new Date(technique.stix.modified), + }, + { $pull: { 'workspace.release_tracks': { id: track.id } } }, + ); + expect((await getTechnique(technique)).workspace.release_tracks || []).toHaveLength(0); + + const first = await reconciliationService.reconcileAll({ continueOnError: false }); + expect(first).toContainEqual( + expect.objectContaining({ + track_id: track.id, + status: 'completed', + }), + ); + expect((await getTechnique(technique)).workspace.release_tracks).toEqual([ + expect.objectContaining({ id: track.id, tier: 'members' }), + ]); + + const second = await reconciliationService.reconcileAll({ continueOnError: false }); + expect(second).toContainEqual( + expect.objectContaining({ + track_id: track.id, + status: 'completed', + }), + ); + expect((await getTechnique(technique)).workspace.release_tracks).toHaveLength(1); + }); +}); diff --git a/app/tests/api/release-tracks/tagged-content-immutability.spec.js b/app/tests/api/release-tracks/tagged-content-immutability.spec.js new file mode 100644 index 00000000..e0c03567 --- /dev/null +++ b/app/tests/api/release-tracks/tagged-content-immutability.spec.js @@ -0,0 +1,141 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function buildTechnique(name, previous) { + const timestamp = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track authoritative tagged-content immutability', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + it('blocks mutation from historical tagged membership when current backrefs are absent', async function () { + const technique = await post('/api/techniques', buildTechnique('Historical Member'), 201); + const replacement = await post('/api/techniques', buildTechnique('Current Draft Member'), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Historical Immutability', type: 'standard' }, + 201, + ); + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + }); + await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '1.0' }); + + // A newer draft removes the member, so latest-snapshot reconciliation + // deliberately removes the object's denormalized backref. The historical + // tagged snapshot remains the immutable authority. + await post(`/api/release-tracks/${track.id}/contents`, { + x_mitre_contents: [{ obj_ref: replacement.stix.id, obj_modified: replacement.stix.modified }], + }); + const current = ( + await api( + 'get', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 200, + ) + ).body; + expect(current.workspace.release_tracks || []).toHaveLength(0); + + // Clear the registry's denormalized tagged-release catalogue as well. + // The guard must query tagged snapshots, not either derived index. + await ReleaseTrackRegistry.updateOne( + { track_id: track.id }, + { + $set: { + tagged_releases: [], + tagged_release_count: 0, + latest_tagged_version: null, + }, + }, + ); + + const updated = buildTechnique('Historical Member (edited)', technique); + updated.stix.modified = technique.stix.modified; + const putResponse = await api( + 'put', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + updated, + 409, + ); + expect(putResponse.body.release_tracks).toEqual([track.id]); + expect(putResponse.body.tagged_releases).toEqual([ + expect.objectContaining({ + track_id: track.id, + version: '1.0', + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }), + ]); + + await api( + 'delete', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 409, + ); + await api('delete', `/api/techniques/${technique.stix.id}`, undefined, 409); + + await api( + 'get', + `/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`, + undefined, + 200, + ); + }); +}); diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index a553a44f..05549787 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -12,6 +12,7 @@ const { InvalidObjectRevisionError, InvalidPostOperationError, ReleaseContentIntegrityError, + ReleaseTrackReconciliationError, } = require('../../exceptions'); describe('error-handler middleware', function () { @@ -175,4 +176,28 @@ describe('error-handler middleware', function () { expect(Object.keys(err)).not.toContain('cause'); expect(next.called).toBe(false); }); + + it('should return durable reconciliation identifiers on protection failures', function () { + const err = new ReleaseTrackReconciliationError('release-track--track', 'repair-id', { + details: 'Run repair.', + }); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(500)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'Release-track membership protection could not be reconciled', + details: 'Run repair.', + track_id: 'release-track--track', + reconciliation_id: 'repair-id', + }), + ).toBe(true); + expect(next.called).toBe(false); + }); }); diff --git a/docs/README.md b/docs/README.md index bf0ca6bf..65a8e53f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -58,6 +58,7 @@ Configuration, deployment, and identity provider setup. - [Configuration](admin/configuration.md): Complete configuration guide (environment variables, JSON files) - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records - [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability +- [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures ### Authentication diff --git a/docs/admin/release-track-reconciliation.md b/docs/admin/release-track-reconciliation.md new file mode 100644 index 00000000..53ca08c3 --- /dev/null +++ b/docs/admin/release-track-reconciliation.md @@ -0,0 +1,79 @@ +# Release-Track Membership Reconciliation + +Release-track snapshots are authoritative. Object documents carry +`workspace.release_tracks` as a derived current-snapshot index used for +navigation and mutation protection. + +## Durable records + +Every snapshot membership change creates a document in +`releaseTrackReconciliations` before the server updates object backrefs. + +Important fields: + +| Field | Meaning | +|---|---| +| `reconciliation_id` | Stable UUID returned to API callers when reconciliation fails | +| `track_id` | Track whose latest snapshot is being reconciled | +| `requested_snapshot_modified` | Snapshot current when the record was created; null means track deletion | +| `reconciled_snapshot_modified` | Snapshot actually used by the successful attempt | +| `source` | `contents_changed`, `repair`, or `full_scan` | +| `status` | `pending`, `completed`, or `failed` | +| `attempts` | Number of listener dispatch attempts | +| `last_error` | Most recent failure name and message | + +API HTTP `500` responses containing a `reconciliation_id` mean the +release-track mutation may already be persisted. In particular, a release may +already be tagged. Inspect the track before repeating any mutation. + +## Inspect failures + +```javascript +db.releaseTrackReconciliations.find({ + status: { $in: ["pending", "failed"] } +}).sort({ updated_at: 1 }).pretty() +``` + +Inspect one response identifier: + +```javascript +db.releaseTrackReconciliations.findOne({ + reconciliation_id: "" +}) +``` + +## Repair outstanding attempts + +```bash +DATABASE_URL=mongodb://... npm run repair:release-track-backrefs +``` + +The default repairs up to 100 oldest pending/failed records. Set a bound: + +```bash +DATABASE_URL=mongodb://... npm run repair:release-track-backrefs -- --limit=500 +``` + +Each retry reads the track's current latest snapshot. It does not replay an +obsolete snapshot payload, so repeated repair is idempotent. + +## Full scan + +Run a full scan after an unclean shutdown or when legacy drift is suspected: + +```bash +DATABASE_URL=mongodb://... npm run repair:release-track-backrefs -- --all +``` + +This unions registered track IDs with IDs found in object and relationship +backrefs. Existing tracks are reconciled to their current latest snapshots; +backrefs for tracks that no longer exist are removed. + +The command prints JSON and exits nonzero if any track still fails. Preserve +failed records and command output for incident review. + +## Known crash window + +Snapshot persistence and reconciliation-record creation do not share a MongoDB +transaction. A hard crash between those writes can leave no pending record. +The full scan is the recovery mechanism for that narrow interval. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index b0301b13..0bfed042 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -116,6 +116,33 @@ Done when: - Tests cover multiple missing references and prove no partial snapshot or bundle is rendered. +### [ ] Handle persisted mutations whose membership reconciliation failed + +A release-track mutation can persist its snapshot before a downstream object +backref write fails. The server now returns HTTP `500` instead of reporting +success and includes: + +```ts +{ + message: 'Release-track membership protection could not be reconciled'; + track_id: string; + reconciliation_id: string; + details?: string; +} +``` + +For a release request, the snapshot may already be tagged. Do not +automatically retry the POST: refresh snapshot history first, show the +reconciliation ID, and direct the operator to an administrator if protection +repair is still pending. + +Done when: + +- The connector preserves `track_id` and `reconciliation_id` from this `500`. +- Release and mutation dialogs explain that persistence may have succeeded + and do not offer a blind retry. +- The UI refreshes the relevant track before enabling another action. + ## P0 — Align the Angular connector with the current routes ### [ ] Use only the explicit snapshot-retrieval endpoints diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 26bb6c4e..0c0995e4 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -62,7 +62,38 @@ Verification result (2026-07-30): ### Remaining prioritized recommendations -- [ ] P0.3 — Make tagged-content immutability authoritative and durable. +### P0.3 — Make tagged-content immutability authoritative and durable + +- [x] Guard object revision update/delete and delete-all by querying tagged + snapshot membership, even when `workspace.release_tracks` is missing or + stale. +- [x] Make release-track backref reconciliation failures propagate to the + triggering request so a release is never reported as fully successful when + protection writes failed. +- [x] Persist every reconciliation attempt and its terminal outcome so + process crashes and partial listener failures remain operator-visible. +- [x] Provide an idempotent repair command for failed/pending reconciliation + records and a full-scan mode for legacy drift. +- [x] Add failure-injection, missing-backref, repair, and historical-release + regressions; update user/developer/admin documentation. +- [x] Run focused, lint, OpenAPI, and complete-suite verification. + +Verification result (2026-07-30): + +- Lint and OpenAPI validation pass. +- The complete release-track API group passes (146), including + failure-injection, repair, and authoritative historical-membership + regressions. Scheduler/date/cron integration (15) and middleware (11) + focused groups pass. +- Repeated complete-suite runs execute all 958 API cases and consistently + pass the release-track cases. The repository's documented roaming + Supertest transport flake still moves among unrelated isolated-pass cases + (socket resets, transient status mismatches, or timeouts). Dynamic + release-track models are now evicted between dropped test databases and the + Mongoose connection is reused, reducing the API run from roughly six + minutes to roughly one minute; the remaining unrelated transport flake is + tracked separately from this completed integrity change. + - [ ] P0.4 — Correct destructive authorization and add durable audit records. - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. - [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and diff --git a/docs/developer/event-bus-architecture.md b/docs/developer/event-bus-architecture.md index 66ad2fc0..8112ce2f 100644 --- a/docs/developer/event-bus-architecture.md +++ b/docs/developer/event-bus-architecture.md @@ -105,6 +105,30 @@ Each STIX document has two top-level keys: ### 4. Event Bus Messaging +The default `EventBus.emit()` method waits for every listener with +`Promise.allSettled()`, logs individual failures, and returns successful +listener values. It is appropriate when a listener is advisory or when the +caller has a separate recovery contract. + +Use `EventBus.emitRequired()` when listener-owned writes are part of the +caller's success contract. It still lets every listener finish, but rejects +when a listener fails or when fewer than the declared `minimumListeners` are +registered. The caller must make the failure durable when the originating +write has already been persisted. + +Release-track membership reconciliation is the first required-event workflow: + +1. Persist the snapshot mutation. +2. Create a pending `releaseTrackReconciliations` record. +3. Call `emitRequired()` for the attack-object and relationship backref + owners. +4. Mark the record completed, or mark it failed and return a structured + service error containing its reconciliation ID. + +See +[backref-reconciliation.md](release-tracks/backref-reconciliation.md) and the +[operator repair procedure](../admin/release-track-reconciliation.md). + **Event Naming Convention:** ``` @@ -160,7 +184,7 @@ Where `{type}` is the STIX type (e.g., `attack-pattern`, `x-mitre-analytic`, `x- | `x-mitre-detection-strategy::analytics-referenced` | DetectionStrategiesService | When detection strategy references analytics (create/update) | `{ detectionStrategyId, detectionStrategy, analyticIds }` | AnalyticsService | | `x-mitre-detection-strategy::analytics-removed` | DetectionStrategiesService | When analytics removed from detection strategy | `{ detectionStrategyId, analyticIds }` | AnalyticsService | | `x-mitre-analytic::parent-changed` | AnalyticsService | When analytic's parent detection strategy changes | `{ analyticId, oldParentId, newParentId, analytic }` | (Future: for cascading updates) | -| `release-track::contents-changed` | snapshot-service / versioning-service | After any persisted change to a track's latest snapshot (or track/snapshot deletion) | `{ trackId, snapshot }` (`snapshot` null when the track or its only snapshot was deleted) | AttackObjectsService, RelationshipsService (reconcile `workspace.release_tracks` backrefs; see [backref-reconciliation.md](release-tracks/backref-reconciliation.md)) | +| `release-track::contents-changed` | snapshot-service / versioning-service | After a durable reconciliation record is created for any persisted change to a track's latest snapshot (or track/snapshot deletion) | `{ trackId, snapshot, reconciliationId }` (`snapshot` null when the track or its only snapshot was deleted) | AttackObjectsService, RelationshipsService (required listeners that reconcile `workspace.release_tracks` backrefs; see [backref-reconciliation.md](release-tracks/backref-reconciliation.md)) | ## Workflow Examples diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index e3b427e3..6e2d3a9d 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -38,7 +38,10 @@ snapshot-service.deleteSnapshot │ bundle import, updateContents, ...) snapshot-service.deleteTrack │ versioning-service.releaseLatest/releaseByModified ┘ (staged → members via tagSnapshotInPlace) │ - ▼ awaited EventBus.emit release-track::contents-changed { trackId, snapshot } + ▼ persist releaseTrackReconciliations record (pending) + │ + ▼ awaited EventBus.emitRequired release-track::contents-changed + │ { trackId, snapshot, reconciliationId } │ snapshot = track's latest snapshot, │ or null when the track (or its only │ snapshot) was deleted @@ -63,8 +66,17 @@ can reference its ID yet. `releaseByModified` may tag an older snapshot; the rel path therefore re-reads the *latest* snapshot before emitting rather than using the tagged one. -Emissions are awaited (the request/response-blocking convention), so backrefs -are consistent by the time the triggering API call returns. +Emissions are awaited and required. The EventBus rejects when either owning +listener fails or is not registered. A successful response therefore means +both object collections were reconciled. A failure returns HTTP `500` with +the durable `reconciliation_id`; the release-track mutation may already be +persisted and must not be retried blindly. + +Every attempt is written to `releaseTrackReconciliations` before listeners +run. Records move through `pending`, `completed`, or `failed` and retain the +requested snapshot, attempt count, timestamps, and last error. If recording +completion fails after the listeners succeeded, the record remains pending; +replaying it is safe because reconciliation is idempotent. ## Reconciliation algorithm @@ -133,5 +145,37 @@ backref to the newly latest revision without rewriting the stored selector revision is later re-created, its backref is restored on the next contents-changed event for that track, not immediately. - **Historical snapshots.** Backrefs describe only the *latest* snapshot per - track. Membership in older snapshots remains discoverable only from the - track side. + track. Object mutation guards do not trust that derived view: they query + every registered track's tagged snapshots for the exact revision before an + in-place update or delete. Historical tagged membership therefore remains + immutable even after the latest draft removes the object or the registry's + tagged-release catalogue is stale. +- **Crash window before record creation.** Snapshot persistence and the + central reconciliation record are not in one MongoDB transaction. A hard + process failure in that narrow interval can leave no pending record. + Operators should run the full-scan repair after an unclean shutdown; it + compares every registered track and every track ID found in object + backrefs against current latest snapshots. + +## Repair + +Repair outstanding `pending`/`failed` attempts: + +```bash +npm run repair:release-track-backrefs +``` + +Limit one invocation with `--limit`: + +```bash +npm run repair:release-track-backrefs -- --limit=500 +``` + +Perform a full idempotent scan, including stale backrefs for deleted tracks: + +```bash +npm run repair:release-track-backrefs -- --all +``` + +The command exits nonzero if any track still fails and prints a JSON summary +with track and reconciliation identifiers. diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index ad7e0049..7d960a68 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -81,7 +81,11 @@ Release tracks are never blind to changes in the objects they pin: `409 Conflict` — released content cannot be changed or destroyed under the track. Make changes by creating a new revision (`POST`); retire an object by creating a new revision with `x_mitre_deprecated: true`. Revision sync - captures either one. + captures either one. This guard checks tagged snapshots authoritatively, not + only the current `workspace.release_tracks` value. A revision remains + protected when it belongs only to a historical tagged release, when a newer + draft has removed it, or when a reconciliation failure temporarily omitted + its backref. - **Candidate/staged-pinned revisions can be edited in place, but the track sees it.** An in-place `PUT` (including one that only sets `x_mitre_deprecated`) marks the pinned entry `modified-in-place`: the @@ -112,3 +116,12 @@ POST /api/release-tracks/:id/candidates/promote → { tier: "staged", sta POST /api/release-tracks/:id/snapshots/latest/release → { tier: "members", status: "reviewed" } DELETE /api/release-tracks/:id → entry removed ``` + +## Reconciliation failures + +Track mutations reconcile object backrefs before reporting success. If either +object collection cannot be updated, the API returns HTTP `500` with +`track_id` and `reconciliation_id`. The track mutation may already have been +persisted—including a release tag—so do not repeat it blindly. Give the +reconciliation ID to an administrator, who can inspect the durable failure +record and run the idempotent repair command. diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index acdefd8e..bbae7b7d 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -64,6 +64,13 @@ revision that no longer exists, the operation returns HTTP `409` with `missing_references` and does not tag the snapshot. This check protects both standard and virtual releases from publishing incomplete primary membership. +After tagging, the server reconciles the member protections stored on object +documents. A successful response means both object collections were updated. +HTTP `500` with a `reconciliation_id` means the release may already be tagged, +but one or more protection writes failed. Do not repeat the release request +without checking the selected snapshot first; an administrator can safely +replay the idempotent reconciliation using that durable record. + ### STIX Freeze Solution Version pinning solves the "STIX freeze" problem: diff --git a/package.json b/package.json index 1c1ae037..deeb337a 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "test:fuzz": "mocha --timeout 10000 --recursive ./app/tests/fuzz --exit", "test:scheduler": "mocha --timeout 60000 --recursive ./app/tests/scheduler --exit", "test:file": "mocha --timeout 10000 --exit", + "repair:release-track-backrefs": "node scripts/reconcileReleaseTrackBackrefs.js", "check:lockfile": "bash scripts/check-package-lock.sh" }, "dependencies": { diff --git a/scripts/README.md b/scripts/README.md index e4928f56..e874529b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1 +1,22 @@ This directory holds utility scripts that are used for system configuration during software development. + +## Release-track backref repair + +`reconcileReleaseTrackBackrefs.js` repairs durable failed or pending +`workspace.release_tracks` reconciliation attempts: + +```bash +npm run repair:release-track-backrefs +npm run repair:release-track-backrefs -- --limit=500 +``` + +Use `--all` after an unclean shutdown or when legacy drift is suspected. It +reconciles all registry tracks and removes stale backrefs whose track no +longer exists: + +```bash +npm run repair:release-track-backrefs -- --all +``` + +The script requires the normal `DATABASE_URL`, prints a JSON result, and exits +nonzero when any repair still fails. diff --git a/scripts/reconcileReleaseTrackBackrefs.js b/scripts/reconcileReleaseTrackBackrefs.js new file mode 100644 index 00000000..e7e9d49d --- /dev/null +++ b/scripts/reconcileReleaseTrackBackrefs.js @@ -0,0 +1,63 @@ +#!/usr/bin/env node + +'use strict'; + +const mongoose = require('mongoose'); + +function parseOptions(argv) { + const all = argv.includes('--all'); + const limitArgument = argv.find((argument) => argument.startsWith('--limit=')); + const limit = limitArgument ? Number(limitArgument.split('=')[1]) : 100; + + if (!Number.isInteger(limit) || limit < 1 || limit > 10000) { + throw new Error('--limit must be an integer between 1 and 10000'); + } + + return { all, limit }; +} + +async function run() { + const options = parseOptions(process.argv.slice(2)); + await require('../app/lib/database-connection').initializeConnection(); + + // Loading the owning services registers both required reconciliation + // listeners before the repair dispatches any events. + require('../app/services/stix/attack-objects-service'); + require('../app/services/stix/relationships-service'); + const reconciliationService = require('../app/services/release-tracks/reconciliation-service'); + + const results = options.all + ? await reconciliationService.reconcileAll({ continueOnError: true }) + : await reconciliationService.repairOutstanding({ + limit: options.limit, + continueOnError: true, + }); + const failed = results.filter((result) => result.status === 'failed'); + + console.log( + JSON.stringify( + { + mode: options.all ? 'full_scan' : 'outstanding', + processed: results.length, + completed: results.length - failed.length, + failed: failed.length, + results, + }, + null, + 2, + ), + ); + + if (failed.length > 0) { + process.exitCode = 1; + } +} + +run() + .catch((error) => { + console.error(error); + process.exitCode = 1; + }) + .finally(async () => { + await mongoose.disconnect(); + }); From 256f86e18db5603fd32cbb258cbf0a0669ea6f2f Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:52:36 -0400 Subject: [PATCH 35/55] fix(release-tracks): protect destructive operations Require administrator authorization and exact track confirmation for member replacement and full-track deletion. Persist actor-attributed audit events before execution and expose durable identifiers when finalization fails. --- .../definitions/components/release-tracks.yml | 20 ++ .../paths/release-tracks-paths.yml | 58 ++++- app/controllers/release-tracks-controller.js | 34 ++- app/exceptions/index.js | 11 + app/lib/error-handler.js | 4 +- .../release-track-audit-event-model.js | 42 ++++ .../release-track-audit-event.repository.js | 75 ++++++ app/routes/release-tracks-routes.js | 4 +- .../destructive-audit-service.js | 45 ++++ .../release-tracks/release-tracks-service.js | 53 +++- .../destructive-authorization.spec.js | 226 ++++++++++++++++++ .../primary-revision-integrity.spec.js | 8 +- .../reconciliation-durability.spec.js | 2 +- .../release-tracks-backrefs.spec.js | 7 +- .../release-tracks-bundle.spec.js | 2 +- .../release-tracks-change-capture.spec.js | 2 +- .../release-tracks-release.spec.js | 16 +- .../release-tracks-tier-invariant.spec.js | 2 +- .../api/release-tracks/release-tracks.spec.js | 1 + .../release-tracks/releases-by-object.spec.js | 2 +- .../tagged-content-immutability.spec.js | 4 +- .../virtual-deduplication.spec.js | 2 +- .../virtual-determinism.spec.js | 2 +- .../virtual-domain-filters.spec.js | 2 +- .../virtual-object-type-filters.spec.js | 2 +- .../release-tracks/virtual-quarantine.spec.js | 2 +- app/tests/middleware/error-handler.spec.js | 25 ++ docs/README.md | 2 + docs/admin/release-track-audit.md | 50 ++++ docs/developer/FRONTEND_TODO.md | 27 +++ docs/developer/TODO.md | 26 +- .../developer/release-tracks/authorization.md | 41 ++++ docs/user/release-tracks/api-reference.md | 28 ++- 33 files changed, 778 insertions(+), 49 deletions(-) create mode 100644 app/models/release-tracks/release-track-audit-event-model.js create mode 100644 app/repository/release-tracks/release-track-audit-event.repository.js create mode 100644 app/services/release-tracks/destructive-audit-service.js create mode 100644 app/tests/api/release-tracks/destructive-authorization.spec.js create mode 100644 docs/admin/release-track-audit.md create mode 100644 docs/developer/release-tracks/authorization.md diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 1e5683a4..d1549685 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -727,3 +727,23 @@ components: type: string format: uuid description: 'Durable reconciliation record to inspect or repair' + + release-track-audit-error: + type: object + required: + - message + - track_id + - audit_event_id + properties: + message: + type: string + enum: + - 'Release-track audit recording could not be finalized' + details: + type: string + description: 'Operator guidance; the destructive operation may already be persisted' + track_id: + type: string + audit_event_id: + type: string + format: uuid diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 4ed5c2d3..94917925 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -283,7 +283,9 @@ paths: operationId: 'release-tracks-delete' description: | Delete an entire release track including all snapshots and version history. - This operation cannot be undone. + This administrator-only operation cannot be undone. The caller must + confirm the exact target with confirm_track_id. The server writes a + durable audit event before deletion begins. tags: - 'Release Tracks' parameters: @@ -293,11 +295,29 @@ paths: description: 'Release track ID' schema: type: string + - name: confirm_track_id + in: query + required: true + description: 'Must exactly equal the id path parameter' + schema: + type: string responses: '204': description: 'Release track deleted successfully' '404': description: 'Release track not found' + '400': + description: 'Missing or mismatched destructive confirmation' + '401': + description: 'Administrator role required' + '500': + description: 'The deletion may be persisted, but its durable audit or membership reconciliation failed' + content: + application/json: + schema: + oneOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' # ============================================================================= # Latest snapshot operations @@ -341,6 +361,8 @@ paths: Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. Creates a new snapshot clone. + This administrator-only operation requires confirm_track_id to exactly + match the target track and writes a durable audit event. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -350,11 +372,27 @@ paths: required: true schema: type: string + - name: confirm_track_id + in: query + required: true + description: 'Must exactly equal the id path parameter' + schema: + type: string responses: '200': description: 'Contents updated successfully' '400': description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' + '401': + description: 'Administrator role required' + '500': + description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' + content: + application/json: + schema: + oneOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/clone: post: @@ -1256,6 +1294,8 @@ paths: store exact revision pins. Exact revisions already present in another tier are retained only in members; different revisions of the same object remain valid across tiers. + This administrator-only operation requires confirm_track_id to exactly + match the target track and writes a durable audit event. Request body validated via Zod in controller. tags: - 'Release Tracks' @@ -1270,11 +1310,27 @@ paths: required: true schema: type: string + - name: confirm_track_id + in: query + required: true + description: 'Must exactly equal the id path parameter' + schema: + type: string responses: '200': description: 'Contents updated successfully' '400': description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' + '401': + description: 'Administrator role required' + '500': + description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' + content: + application/json: + schema: + oneOf: + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' + - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' /api/release-tracks/{id}/snapshots/{modified}/clone: post: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 3bc462c5..fb61574b 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -79,6 +79,25 @@ function parseOptionalQueryStrict(value, schema, defaultValue, parameterName) { }); } +function requireDestructiveConfirmation(req) { + if (req.query.confirm_track_id !== req.params.id) { + throw new BadRequestError({ + message: 'Destructive release-track confirmation is required', + details: `Set confirm_track_id to the exact target track ID '${req.params.id}'.`, + parameter_name: 'confirm_track_id', + expected_track_id: req.params.id, + }); + } +} + +function destructiveActor(req) { + return { + user_account_id: req.user?.userAccountId, + role: req.user?.role, + authentication_strategy: req.user?.strategy, + }; +} + function rejectFilesystemStoreFormat(format, methodName) { if (format !== 'filesystemstore') return null; @@ -445,6 +464,7 @@ exports.updateMetadataByLatest = async function updateMetadataByLatest(req, res, /** POST /api/release-tracks/:id/contents */ exports.updateContentsByLatest = async function updateContentsByLatest(req, res, next) { try { + requireDestructiveConfirmation(req); const bodyResult = updateContentsBodySchema.safeParse(req.body); if (!bodyResult.success) { return next( @@ -458,7 +478,8 @@ exports.updateContentsByLatest = async function updateContentsByLatest(req, res, const result = await releaseTracksService.updateContents( req.params.id, bodyResult.data, - req.user?.userAccountId, + destructiveActor(req), + req.query.confirm_track_id, ); logger.debug(`Success: Updated contents for track ${req.params.id}`); return res.status(200).send(result); @@ -521,7 +542,12 @@ exports.cloneByLatest = async function cloneByLatest(req, res, next) { /** DELETE /api/release-tracks/:id */ exports.deleteReleaseTrack = async function deleteReleaseTrack(req, res, next) { try { - await releaseTracksService.deleteTrack(req.params.id); + requireDestructiveConfirmation(req); + await releaseTracksService.deleteTrack( + req.params.id, + destructiveActor(req), + req.query.confirm_track_id, + ); logger.debug(`Success: Deleted track ${req.params.id}`); return res.status(204).end(); } catch (err) { @@ -589,6 +615,7 @@ exports.updateMetadataByModified = async function updateMetadataByModified(req, /** POST /api/release-tracks/:id/snapshots/:modified/contents */ exports.updateContentsByModified = async function updateContentsByModified(req, res, next) { try { + requireDestructiveConfirmation(req); const bodyResult = updateContentsBodySchema.safeParse(req.body); if (!bodyResult.success) { return next( @@ -603,7 +630,8 @@ exports.updateContentsByModified = async function updateContentsByModified(req, req.params.id, req.params.modified, bodyResult.data, - req.user?.userAccountId, + destructiveActor(req), + req.query.confirm_track_id, ); logger.debug(`Success: Updated contents for snapshot ${req.params.modified}`); return res.status(200).send(result); diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 8186a9ce..8be2df56 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -335,6 +335,16 @@ class ReleaseTrackReconciliationError extends CustomError { } } +class ReleaseTrackAuditError extends CustomError { + constructor(trackId, auditEventId, options = {}) { + super('Release-track audit recording could not be finalized', { + ...options, + track_id: trackId, + audit_event_id: auditEventId, + }); + } +} + class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { super(`Tagged snapshot version ${version} cannot be deleted`, options); @@ -433,6 +443,7 @@ module.exports = { ReleaseConflictError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, + ReleaseTrackAuditError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 4e9d1b14..c187dc47 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -43,6 +43,7 @@ const { ReleaseConflictError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, + ReleaseTrackAuditError, NoTaggedSnapshotsError, InvalidComponentTypeError, VirtualSnapshotNotMaterializedError, @@ -157,7 +158,8 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof TacticsServiceError || err instanceof GenericServiceError || err instanceof DatabaseError || - err instanceof ReleaseTrackReconciliationError + err instanceof ReleaseTrackReconciliationError || + err instanceof ReleaseTrackAuditError ) { logger.error('Service error: %s', JSON.stringify(buildErrorResponse(err))); return res.status(500).send(buildErrorResponse(err)); diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js new file mode 100644 index 00000000..ded1fa76 --- /dev/null +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -0,0 +1,42 @@ +'use strict'; + +const mongoose = require('mongoose'); +const { validateTrackId } = require('../../lib/release-tracks/release-track-validators'); + +const releaseTrackAuditEventSchema = new mongoose.Schema( + { + event_id: { type: String, required: true, unique: true }, + action: { + type: String, + required: true, + enum: ['replace_members_latest', 'replace_members_historical', 'delete_track'], + }, + track_id: { type: String, required: true, validate: validateTrackId }, + status: { + type: String, + required: true, + enum: ['pending', 'completed', 'failed'], + default: 'pending', + }, + actor: { type: mongoose.Schema.Types.Mixed, required: true }, + confirmation: { type: String, required: true }, + request: { type: mongoose.Schema.Types.Mixed, default: {} }, + result: { type: mongoose.Schema.Types.Mixed, default: null }, + error: { + name: String, + message: String, + }, + started_at: { type: Date, required: true }, + finished_at: { type: Date, default: null }, + }, + { + collection: 'releaseTrackAuditEvents', + bufferCommands: false, + }, +); + +releaseTrackAuditEventSchema.index({ track_id: 1, started_at: -1 }); +releaseTrackAuditEventSchema.index({ action: 1, started_at: -1 }); +releaseTrackAuditEventSchema.index({ 'actor.user_account_id': 1, started_at: -1 }); + +module.exports = mongoose.model('ReleaseTrackAuditEvent', releaseTrackAuditEventSchema); diff --git a/app/repository/release-tracks/release-track-audit-event.repository.js b/app/repository/release-tracks/release-track-audit-event.repository.js new file mode 100644 index 00000000..884e1679 --- /dev/null +++ b/app/repository/release-tracks/release-track-audit-event.repository.js @@ -0,0 +1,75 @@ +'use strict'; + +const { v4: uuidv4 } = require('uuid'); +const ReleaseTrackAuditEvent = require('../../models/release-tracks/release-track-audit-event-model'); +const { DatabaseError } = require('../../exceptions'); + +class ReleaseTrackAuditEventRepository { + async create({ action, trackId, actor, confirmation, request }) { + try { + const event = await ReleaseTrackAuditEvent.create({ + event_id: uuidv4(), + action, + track_id: trackId, + status: 'pending', + actor, + confirmation, + request, + started_at: new Date(), + }); + return event.toObject(); + } catch (error) { + throw new DatabaseError(error); + } + } + + async complete(eventId, result) { + try { + const event = await ReleaseTrackAuditEvent.findOneAndUpdate( + { event_id: eventId }, + { + $set: { + status: 'completed', + result: result || null, + error: null, + finished_at: new Date(), + }, + }, + { new: true, lean: true }, + ).exec(); + if (!event) { + throw new Error(`Release-track audit event ${eventId} no longer exists`); + } + return event; + } catch (error) { + throw new DatabaseError(error); + } + } + + async fail(eventId, error) { + try { + const event = await ReleaseTrackAuditEvent.findOneAndUpdate( + { event_id: eventId }, + { + $set: { + status: 'failed', + error: { + name: error?.name || 'Error', + message: error?.message || String(error), + }, + finished_at: new Date(), + }, + }, + { new: true, lean: true }, + ).exec(); + if (!event) { + throw new Error(`Release-track audit event ${eventId} no longer exists`); + } + return event; + } catch (repositoryError) { + throw new DatabaseError(repositoryError); + } + } +} + +module.exports = new ReleaseTrackAuditEventRepository(); diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 263caf51..40838dc7 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -263,7 +263,7 @@ router .route('/release-tracks/:id/snapshots/:modified/contents') .post( authn.authenticate, - authz.requireRole(authz.editorOrHigher), + authz.requireRole(authz.admin), releaseTracksController.updateContentsByModified, ); @@ -324,7 +324,7 @@ router .route('/release-tracks/:id') .delete( authn.authenticate, - authz.requireRole(authz.editorOrHigher), + authz.requireRole(authz.admin), releaseTracksController.deleteReleaseTrack, ); diff --git a/app/services/release-tracks/destructive-audit-service.js b/app/services/release-tracks/destructive-audit-service.js new file mode 100644 index 00000000..1b399951 --- /dev/null +++ b/app/services/release-tracks/destructive-audit-service.js @@ -0,0 +1,45 @@ +'use strict'; + +const logger = require('../../lib/logger'); +const auditRepo = require('../../repository/release-tracks/release-track-audit-event.repository'); +const { ReleaseTrackAuditError } = require('../../exceptions'); + +function snapshotResult(snapshot) { + if (!snapshot) return null; + return { + snapshot_modified: snapshot.modified, + version: snapshot.version ?? null, + members_count: snapshot.members?.length || 0, + }; +} + +exports.execute = async function execute(options, operation) { + const event = await auditRepo.create(options); + let operationCompleted = false; + + try { + const result = await operation(); + operationCompleted = true; + await auditRepo.complete(event.event_id, options.result?.(result) ?? snapshotResult(result)); + return result; + } catch (error) { + if (!operationCompleted) { + try { + await auditRepo.fail(event.event_id, error); + } catch (auditError) { + logger.error( + `DestructiveAuditService: Failed to record ${event.event_id} failure: ` + + auditError.message, + ); + } + throw error; + } + + throw new ReleaseTrackAuditError(options.trackId, event.event_id, { + details: + 'The destructive release-track operation completed, but its audit record could not be ' + + 'finalized. Inspect the track and audit event before retrying.', + cause: error, + }); + } +}; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index aa2bc8de..7a2ebab0 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -29,6 +29,7 @@ const ephemeralService = require('./ephemeral-service'); const bundleImportService = require('./bundle-import-service'); const memberSyncService = require('./member-sync-service'); const releaseHistoryService = require('./release-history-service'); +const destructiveAuditService = require('./destructive-audit-service'); const attackObjectsService = require('../stix/attack-objects-service'); const userAccountsService = require('../system/user-accounts-service'); const revisionReference = require('../../lib/release-tracks/revision-reference'); @@ -40,6 +41,16 @@ function notImplemented(methodName) { throw new NotImplementedError(MODULE, methodName); } +function destructiveIdentity(trackId, actor, confirmation) { + return { + actor: actor || { + kind: 'system', + name: 'internal-service', + }, + confirmation: confirmation || trackId, + }; +} + function rejectFilesystemStoreFormat(format, methodName) { if (format !== 'filesystemstore') return; @@ -287,17 +298,38 @@ exports.updateMetadataByModified = function updateMetadataByModified( return snapshotService.updateMetadataByModified(trackId, modified, updates, userId); }; -exports.updateContents = function updateContents(trackId, contents, userId) { - return snapshotService.updateContents(trackId, contents, userId); +exports.updateContents = function updateContents(trackId, contents, actor, confirmation) { + return destructiveAuditService.execute( + { + action: 'replace_members_latest', + trackId, + ...destructiveIdentity(trackId, actor, confirmation), + request: { members_count: contents.x_mitre_contents.length }, + }, + () => snapshotService.updateContents(trackId, contents, actor?.user_account_id), + ); }; exports.updateContentsByModified = function updateContentsByModified( trackId, modified, contents, - userId, + actor, + confirmation, ) { - return snapshotService.updateContentsByModified(trackId, modified, contents, userId); + return destructiveAuditService.execute( + { + action: 'replace_members_historical', + trackId, + ...destructiveIdentity(trackId, actor, confirmation), + request: { + source_snapshot_modified: modified, + members_count: contents.x_mitre_contents.length, + }, + }, + () => + snapshotService.updateContentsByModified(trackId, modified, contents, actor?.user_account_id), + ); }; exports.cloneTrack = function cloneTrack(trackId, options) { @@ -308,8 +340,17 @@ exports.cloneFromSnapshot = function cloneFromSnapshot(trackId, modified, option return snapshotService.cloneFromSnapshot(trackId, modified, options); }; -exports.deleteTrack = function deleteTrack(trackId) { - return snapshotService.deleteTrack(trackId); +exports.deleteTrack = function deleteTrack(trackId, actor, confirmation) { + return destructiveAuditService.execute( + { + action: 'delete_track', + trackId, + ...destructiveIdentity(trackId, actor, confirmation), + request: {}, + result: () => ({ deleted: true }), + }, + () => snapshotService.deleteTrack(trackId), + ); }; exports.deleteSnapshot = function deleteSnapshot(trackId, modified) { diff --git a/app/tests/api/release-tracks/destructive-authorization.spec.js b/app/tests/api/release-tracks/destructive-authorization.spec.js new file mode 100644 index 00000000..8dd05d0e --- /dev/null +++ b/app/tests/api/release-tracks/destructive-authorization.spec.js @@ -0,0 +1,226 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const sinon = require('sinon'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const UserAccount = require('../../../models/user-account-model'); +const ReleaseTrackAuditEvent = require('../../../models/release-tracks/release-track-audit-event-model'); +const auditRepository = require('../../../repository/release-tracks/release-track-audit-event.repository'); +const systemConfigurationService = require('../../../services/system/system-configuration-service'); + +const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; + +function techniquePayload() { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name: 'Destructive authorization member', + description: 'Member used by destructive authorization tests.', + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; +} + +describe('Release-track destructive authorization and audit', function () { + let app; + let passportCookie; + let anonymousUser; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + anonymousUser = await systemConfigurationService.retrieveAnonymousUserAccount(); + }); + + after(async function () { + sinon.restore(); + await UserAccount.updateOne({ id: anonymousUser.id }, { $set: { role: 'admin' } }); + await database.closeConnection(); + }); + + afterEach(function () { + sinon.restore(); + }); + + async function setRole(role) { + await UserAccount.updateOne({ id: anonymousUser.id }, { $set: { role } }); + } + + function api(method, path, body, status, query) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (query) call.query(query); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200, query) { + return (await api('post', path, body, status, query)).body; + } + + it('requires admin role, exact confirmation, and durable outcome records', async function () { + await setRole('admin'); + const technique = await post('/api/techniques', techniquePayload(), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Destructive authorization standard', type: 'standard' }, + 201, + ); + const contents = { + x_mitre_contents: [ + { + obj_ref: technique.stix.id, + obj_modified: technique.stix.modified, + }, + ], + }; + + await setRole('editor'); + await api('post', `/api/release-tracks/${track.id}/contents`, contents, 401, { + confirm_track_id: track.id, + }); + await api( + 'post', + `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, + contents, + 401, + { confirm_track_id: track.id }, + ); + await api('delete', `/api/release-tracks/${track.id}`, undefined, 401, { + confirm_track_id: track.id, + }); + expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); + + await setRole('admin'); + await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400); + await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400, { + confirm_track_id: 'release-track--00000000-0000-4000-8000-000000000099', + }); + await api('delete', `/api/release-tracks/${track.id}`, undefined, 400); + expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); + + const latest = await post(`/api/release-tracks/${track.id}/contents`, contents, 200, { + confirm_track_id: track.id, + }); + await post( + `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, + contents, + 200, + { confirm_track_id: track.id }, + ); + + const virtual = await post( + '/api/release-tracks/new', + { name: 'Destructive authorization virtual', type: 'virtual' }, + 201, + ); + await api('post', `/api/release-tracks/${virtual.id}/contents`, contents, 400, { + confirm_track_id: virtual.id, + }); + + await api('delete', `/api/release-tracks/${track.id}`, undefined, 204, { + confirm_track_id: track.id, + }); + + const events = await ReleaseTrackAuditEvent.find().sort({ started_at: 1 }).lean().exec(); + expect(events).toHaveLength(4); + expect(events.map((event) => [event.action, event.status])).toEqual([ + ['replace_members_latest', 'completed'], + ['replace_members_historical', 'completed'], + ['replace_members_latest', 'failed'], + ['delete_track', 'completed'], + ]); + expect(events[0]).toMatchObject({ + track_id: track.id, + confirmation: track.id, + actor: { + user_account_id: anonymousUser.id, + role: 'admin', + authentication_strategy: 'anonymId', + }, + request: { members_count: 1 }, + result: { + snapshot_modified: new Date(latest.modified), + members_count: 1, + }, + }); + expect(events[2].track_id).toBe(virtual.id); + expect(events[2].error.message).toContain( + 'Direct contents updates are only available for standard release tracks', + ); + expect(events[3].result).toEqual({ deleted: true }); + }); + + it('reports an audit-finalization failure without hiding the persisted mutation', async function () { + await setRole('admin'); + const technique = await post('/api/techniques', techniquePayload(), 201); + const track = await post( + '/api/release-tracks/new', + { name: 'Audit finalization failure standard', type: 'standard' }, + 201, + ); + const contents = { + x_mitre_contents: [ + { + obj_ref: technique.stix.id, + obj_modified: technique.stix.modified, + }, + ], + }; + + sinon.stub(auditRepository, 'complete').rejects(new Error('injected audit update failure')); + const response = await api('post', `/api/release-tracks/${track.id}/contents`, contents, 500, { + confirm_track_id: track.id, + }); + auditRepository.complete.restore(); + + expect(response.body).toMatchObject({ + message: 'Release-track audit recording could not be finalized', + track_id: track.id, + }); + expect(response.body.audit_event_id).toEqual(expect.any(String)); + + const latest = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest`, + undefined, + 200, + ); + expect(latest.body.members).toHaveLength(1); + expect(latest.body.members[0]).toMatchObject({ + object_ref: technique.stix.id, + object_modified: technique.stix.modified, + }); + + const pendingEvent = await ReleaseTrackAuditEvent.findOne({ + event_id: response.body.audit_event_id, + }) + .lean() + .exec(); + expect(pendingEvent).toMatchObject({ + action: 'replace_members_latest', + track_id: track.id, + status: 'pending', + }); + expect(pendingEvent.finished_at).toBeNull(); + }); +}); diff --git a/app/tests/api/release-tracks/primary-revision-integrity.spec.js b/app/tests/api/release-tracks/primary-revision-integrity.spec.js index 2b32c0a4..81c53614 100644 --- a/app/tests/api/release-tracks/primary-revision-integrity.spec.js +++ b/app/tests/api/release-tracks/primary-revision-integrity.spec.js @@ -145,7 +145,7 @@ describe('Release-track primary revision integrity API', function () { const response = await api( 'post', - `/api/release-tracks/${track.id}/contents`, + `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [ { obj_ref: technique.stix.id, obj_modified: technique.stix.modified }, @@ -199,7 +199,7 @@ describe('Release-track primary revision integrity API', function () { it('rejects cloning and export when a stored primary member is missing', async function () { const technique = await createTechnique('Missing Stored Member'); const track = await createTrack('Missing Stored Member Track'); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); await deleteTechniqueRevision(technique); @@ -233,7 +233,7 @@ describe('Release-track primary revision integrity API', function () { it('propagates repository hydration failures instead of returning a partial export', async function () { const technique = await createTechnique('Failed Primary Hydration'); const track = await createTrack('Failed Primary Hydration Track'); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); const hydrationStub = sinon @@ -255,7 +255,7 @@ describe('Release-track primary revision integrity API', function () { it('aborts virtual materialization when a component member is missing', async function () { const technique = await createTechnique('Missing Virtual Component Member'); const component = await createTrack('Missing Virtual Component'); - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { diff --git a/app/tests/api/release-tracks/reconciliation-durability.spec.js b/app/tests/api/release-tracks/reconciliation-durability.spec.js index 2468b472..52e0792e 100644 --- a/app/tests/api/release-tracks/reconciliation-durability.spec.js +++ b/app/tests/api/release-tracks/reconciliation-durability.spec.js @@ -172,7 +172,7 @@ describe('Release-track durable backref reconciliation', function () { { name: 'Full Scan Repair Track', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index e862a3a3..20361bf8 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -196,6 +196,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { it('deleting the track removes its backrefs', async function () { await request(app) .delete(`/api/release-tracks/${trackId}`) + .query({ confirm_track_id: trackId }) .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(204); @@ -301,7 +302,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const trackId = await createTrack('Backref Contents Track'); const contentsSnapshot = await postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }, @@ -364,7 +365,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const trackId = await createTrack('Backref Member Sync Track'); await postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], }, @@ -469,7 +470,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { ); const trackId = await createTrack('Backref Dynamic Ignore Track'); await postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], }, diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index 8aca6aae..f95977a1 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -217,7 +217,7 @@ describe('Release Tracks Bundle Export API', function () { .expect(200); // Members - await postAction(`/api/release-tracks/${trackId}/contents`, { + await postAction(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [ { obj_ref: memberObject.stix.id, obj_modified: memberObject.stix.modified }, { obj_ref: linkedMemberObject.stix.id, obj_modified: linkedMemberObject.stix.modified }, diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index f1c42d55..1f313696 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -101,7 +101,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { async function setMembers(trackId, technique) { return postObject( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }, diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 72f862dc..5a1c7843 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -363,7 +363,7 @@ describe('Release-track release planning and commit API', function () { it('records immutable component versions when previewing and releasing a virtual draft', async function () { const member = (await post('/api/techniques', buildTechnique('Provenance Member'), 201)).body; const component = await createTrack('Provenance Component'); - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], }); const firstComponentRelease = await post( @@ -401,7 +401,7 @@ describe('Release-track release planning and commit API', function () { // Advance the component after materialization. Virtual release provenance // must remain tied to the frozen component resolution, not current state. - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], }); const secondComponentRelease = await post( @@ -696,7 +696,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Virtual Materialization Member'), 201) ).body; const component = await createTrack('Virtual Materialization Component'); - await post(`/api/release-tracks/${component.id}/contents`, { + await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], }); await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}); @@ -769,9 +769,13 @@ describe('Release-track release planning and commit API', function () { ], }; - await post(`/api/release-tracks/${virtual.id}/contents`, contents, 400); await post( - `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}/contents`, + `/api/release-tracks/${virtual.id}/contents?confirm_track_id=${virtual.id}`, + contents, + 400, + ); + await post( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}/contents?confirm_track_id=${virtual.id}`, contents, 400, ); @@ -788,7 +792,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Release Conflict B', revisionA), 201) ).body; const track = await createTrack('Release Conflict'); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], }); await post(`/api/release-tracks/${track.id}/candidates`, { diff --git a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js index 8cbbeb03..5096b577 100644 --- a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js +++ b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js @@ -125,7 +125,7 @@ describe('Release-track cross-tier revision uniqueness', function () { } async function setMembers(trackId, objects) { - return post(`/api/release-tracks/${trackId}/contents`, { + return post(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: objects.map((object) => ({ obj_ref: object.stix.id, obj_modified: object.stix.modified, diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 45691556..cf46f412 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -109,6 +109,7 @@ describe('Release Tracks API', function () { await request(app) .post(`/api/release-tracks/${trackId}/contents`) + .query({ confirm_track_id: trackId }) .send({ x_mitre_contents: [ { diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index f152f89b..a89e59ca 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -152,7 +152,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { async function setMembers(trackId, objects) { return post( - `/api/release-tracks/${trackId}/contents`, + `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { x_mitre_contents: objects.map((object) => ({ obj_ref: object.stix.id, diff --git a/app/tests/api/release-tracks/tagged-content-immutability.spec.js b/app/tests/api/release-tracks/tagged-content-immutability.spec.js index e0c03567..d72c675a 100644 --- a/app/tests/api/release-tracks/tagged-content-immutability.spec.js +++ b/app/tests/api/release-tracks/tagged-content-immutability.spec.js @@ -71,7 +71,7 @@ describe('Release-track authoritative tagged-content immutability', function () { name: 'Historical Immutability', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], }); await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '1.0' }); @@ -79,7 +79,7 @@ describe('Release-track authoritative tagged-content immutability', function () // A newer draft removes the member, so latest-snapshot reconciliation // deliberately removes the object's denormalized backref. The historical // tagged snapshot remains the immutable authority. - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [{ obj_ref: replacement.stix.id, obj_modified: replacement.stix.modified }], }); const current = ( diff --git a/app/tests/api/release-tracks/virtual-deduplication.spec.js b/app/tests/api/release-tracks/virtual-deduplication.spec.js index 8950a190..b35b9d27 100644 --- a/app/tests/api/release-tracks/virtual-deduplication.spec.js +++ b/app/tests/api/release-tracks/virtual-deduplication.spec.js @@ -89,7 +89,7 @@ describe('Virtual release-track deduplication API', function () { async function createReleasedComponent(name, members) { const track = await post('/api/release-tracks/new', { name, type: 'standard' }); await post( - `/api/release-tracks/${track.id}/contents`, + `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: members.map((member) => ({ obj_ref: member.stix.id, diff --git a/app/tests/api/release-tracks/virtual-determinism.spec.js b/app/tests/api/release-tracks/virtual-determinism.spec.js index f4f70380..a58d028b 100644 --- a/app/tests/api/release-tracks/virtual-determinism.spec.js +++ b/app/tests/api/release-tracks/virtual-determinism.spec.js @@ -82,7 +82,7 @@ describe('Virtual release-track deterministic membership API', function () { type: 'standard', }); const contents = await post( - `/api/release-tracks/${component.id}/contents`, + `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [ { diff --git a/app/tests/api/release-tracks/virtual-domain-filters.spec.js b/app/tests/api/release-tracks/virtual-domain-filters.spec.js index f158ee29..ae38757d 100644 --- a/app/tests/api/release-tracks/virtual-domain-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-domain-filters.spec.js @@ -114,7 +114,7 @@ describe('Virtual Release Track Domain Filters API', function () { type: 'standard', }); await post( - `/api/release-tracks/${component.id}/contents`, + `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { x_mitre_contents: [enterprise, ics, shared, noDomain, enterpriseMatrix].map((object) => ({ obj_ref: object.stix.id, diff --git a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js index fb18ac67..c580c436 100644 --- a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js @@ -203,7 +203,7 @@ describe('Virtual release-track object-type filters API', function () { const matrix = await post('/api/matrices', buildMatrix('Excluded Type Member')); await post( - `/api/release-tracks/${componentTrack.id}/contents`, + `/api/release-tracks/${componentTrack.id}/contents?confirm_track_id=${componentTrack.id}`, { x_mitre_contents: [mitigation, matrix].map((object) => ({ obj_ref: object.stix.id, diff --git a/app/tests/api/release-tracks/virtual-quarantine.spec.js b/app/tests/api/release-tracks/virtual-quarantine.spec.js index 34f91d15..d2e0a24d 100644 --- a/app/tests/api/release-tracks/virtual-quarantine.spec.js +++ b/app/tests/api/release-tracks/virtual-quarantine.spec.js @@ -73,7 +73,7 @@ describe('Virtual release-track quarantine API', function () { async function createReleasedComponent(name, member) { const track = await createTrack(name); - await post(`/api/release-tracks/${track.id}/contents`, { + await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { x_mitre_contents: [ { obj_ref: member.stix.id, diff --git a/app/tests/middleware/error-handler.spec.js b/app/tests/middleware/error-handler.spec.js index 05549787..cd139b5b 100644 --- a/app/tests/middleware/error-handler.spec.js +++ b/app/tests/middleware/error-handler.spec.js @@ -13,6 +13,7 @@ const { InvalidPostOperationError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, + ReleaseTrackAuditError, } = require('../../exceptions'); describe('error-handler middleware', function () { @@ -200,4 +201,28 @@ describe('error-handler middleware', function () { ).toBe(true); expect(next.called).toBe(false); }); + + it('should return durable audit identifiers when finalization fails', function () { + const err = new ReleaseTrackAuditError('release-track--track', 'audit-id', { + details: 'Inspect the operation.', + }); + const res = { + status: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }; + const next = sinon.stub(); + + errorHandler.serviceExceptions(err, {}, res, next); + + expect(res.status.calledOnceWithExactly(500)).toBe(true); + expect( + res.send.calledOnceWithExactly({ + message: 'Release-track audit recording could not be finalized', + details: 'Inspect the operation.', + track_id: 'release-track--track', + audit_event_id: 'audit-id', + }), + ).toBe(true); + expect(next.called).toBe(false); + }); }); diff --git a/docs/README.md b/docs/README.md index 65a8e53f..b4e1ee03 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,7 @@ Architecture, patterns, and implementation details for contributors. - [Error Handling](developer/release-tracks/error-handling.md): Error handling patterns - [Implementation Notes](developer/release-tracks/implementation-notes.md): Implementation notes and decisions - [Releases By Object](developer/release-tracks/releases-by-object.md): Registry catalogue, fan-out query, and indexing design +- [Authorization](developer/release-tracks/authorization.md): Role matrix, destructive confirmation, and audit contract ## Admin Documentation @@ -59,6 +60,7 @@ Configuration, deployment, and identity provider setup. - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records - [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures +- [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator member replacements and track deletions ### Authentication diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md new file mode 100644 index 00000000..548e7a68 --- /dev/null +++ b/docs/admin/release-track-audit.md @@ -0,0 +1,50 @@ +# Release-Track Destructive Audit Events + +Workbench stores administrator-initiated member replacement and full-track +deletion attempts in `releaseTrackAuditEvents`. + +Each record contains: + +- `event_id`, `action`, and `track_id` +- the authenticated `actor` +- the exact `confirmation` supplied by the caller +- a bounded request/result summary +- `pending`, `completed`, or `failed` status +- start/finish timestamps and failure detail + +Inspect recent events: + +```javascript +db.releaseTrackAuditEvents.find().sort({ started_at: -1 }).limit(50).pretty(); +``` + +Inspect destructive actions for one track: + +```javascript +db.releaseTrackAuditEvents + .find({ + track_id: 'release-track--...', + }) + .sort({ started_at: -1 }) + .pretty(); +``` + +Inspect incomplete or failed attempts: + +```javascript +db.releaseTrackAuditEvents + .find({ + status: { $in: ['pending', 'failed'] }, + }) + .sort({ started_at: 1 }) + .pretty(); +``` + +A `pending` event can mean the process stopped after the audit insert or the +operation completed but the final audit update failed. Inspect the target +track before retrying. A failed member replacement may also have persisted a +new snapshot if backref reconciliation subsequently failed; correlate its +timestamp with `releaseTrackReconciliations`. + +These records have no automatic TTL. Establish retention and archive policy +according to local audit requirements. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 0bfed042..c46b2791 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -145,6 +145,33 @@ Done when: ## P0 — Align the Angular connector with the current routes +### [ ] Add administrator confirmation for destructive release-track actions + +Full track deletion and both direct standard-track member replacement routes +are administrator-only. They now require the query parameter +`confirm_track_id` to exactly equal the `:id` path parameter: + +```text +DELETE /api/release-tracks/:id?confirm_track_id=:id +POST /api/release-tracks/:id/contents?confirm_track_id=:id +POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id +``` + +Do not expose these actions to editors or team leads. Before sending a request, +show the track name and ID, explain that direct replacement bypasses the normal +candidate/staged workflow or that deletion removes all history, and require an +explicit confirmation interaction. A missing or stale ID returns `400`; a +non-administrator returns `401`. + +Done when: + +- Route guards and action visibility match the documented authorization + matrix. +- The connector sends the selected track's exact ID as `confirm_track_id`. +- Dialogs cannot reuse confirmation state after the selected track changes. +- Tests cover administrator success plus editor, missing-confirmation, and + mismatched-confirmation rejection. + ### [ ] Use only the explicit snapshot-retrieval endpoints The release-track resource path no longer doubles as an implicit request for diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 0c0995e4..efb91503 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -94,7 +94,31 @@ Verification result (2026-07-30): minutes to roughly one minute; the remaining unrelated transport flake is tracked separately from this completed integrity change. -- [ ] P0.4 — Correct destructive authorization and add durable audit records. +### P0.4 — Correct destructive authorization and add durable audit records + +- [x] Require administrator authorization for full track deletion and both + direct member-replacement routes. +- [x] Require an exact `confirm_track_id` precondition on each destructive + request so stale or accidental UI actions fail before persistence. +- [x] Persist a durable, actor-attributed audit event before each operation + and record completion or failure without hiding partial persistence. +- [x] Add an authorization matrix and operator-facing audit documentation. +- [x] Update OpenAPI, frontend tasks, and Bruno requests for the confirmation + contract. +- [x] Add admin/editor, missing/mismatched confirmation, success/failure + audit, lint, OpenAPI, focused, and complete-suite verification. + +Verification result (2026-07-30): + +- Lint and OpenAPI validation pass. +- The focused authorization/audit and middleware group passes (11). The + complete release-track and virtual-scheduler group passes all 148 relevant + cases; one roaming setup 404 passed immediately in isolation (8). +- The required clean full suite passes: OpenAPI 2, config 21, API 960, + middleware 29, and scheduler 10. +- The `internalattack` focused release-track suite passes (33), its complete + suite passes (246), and changed-file Ruff checks pass. + - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. - [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and operator intervention. diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md new file mode 100644 index 00000000..11fec021 --- /dev/null +++ b/docs/developer/release-tracks/authorization.md @@ -0,0 +1,41 @@ +# Release-Track Authorization + +Release-track access follows the existing Workbench roles. Read operations are +available to visitors and higher. Normal draft workflow operations require an +editor, team lead, or administrator. Operations that can replace authoritative +membership or destroy history require an administrator. + +## Authorization matrix + +| Capability | Visitor | Editor / team lead | Administrator | +| --------------------------------------------------------------------- | ------: | -----------------: | ------------: | +| List tracks, snapshots, candidates, and staged objects | Yes | Yes | Yes | +| Preview releases and export snapshots | Yes | Yes | Yes | +| Create tracks and drafts; manage candidates/staged/config/composition | No | Yes | Yes | +| Tag a standard or virtual snapshot | No | Yes | Yes | +| Delete an untagged individual snapshot | No | Yes | Yes | +| Replace standard-track members directly | No | No | Yes | +| Delete an entire track and all snapshot history | No | No | Yes | + +The two direct replacement routes and full-track deletion also require +`confirm_track_id` to equal the `:id` path parameter. Authorization runs before +the controller, and confirmation runs before request-body validation or +persistence. + +## Audited destructive actions + +The following actions create a `releaseTrackAuditEvents` record before their +business operation begins: + +- `replace_members_latest` +- `replace_members_historical` +- `delete_track` + +Each event records the authenticated actor, confirmation value, target track, +request summary, timestamps, and a `pending`, `completed`, or `failed` status. +An audit insert failure prevents the destructive operation. If the operation +persists but final audit-state recording fails, the API returns a structured +`500` containing the audit event ID instead of reporting unconditional +success. + +See the [operator audit guide](../../admin/release-track-audit.md). diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index b7dfb798..f722075a 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -47,10 +47,10 @@ POST /api/release-tracks/new POST /api/release-tracks/new-from-bundle POST /api/release-tracks/import POST /api/release-tracks/:id/meta -POST /api/release-tracks/:id/contents +POST /api/release-tracks/:id/contents?confirm_track_id=:id POST /api/release-tracks/:id/snapshots/latest/release POST /api/release-tracks/:id/clone -DELETE /api/release-tracks/:id +DELETE /api/release-tracks/:id?confirm_track_id=:id ``` ### Snapshot Operations @@ -470,6 +470,11 @@ POST /api/release-tracks/:id/contents Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** The main workflow for enrolling new member objects into `x_mitre_contents` is through the candidate-staging promotion cycle described in [versioning.md](./versioning.md). +This operation requires the administrator role. The +`confirm_track_id` query parameter must exactly equal the `:id` path +parameter. Every accepted attempt is recorded in the durable release-track +destructive audit trail. + This operation is available only for standard tracks. Virtual membership is computed from component releases and can only be updated by materializing a virtual draft with `POST /api/release-tracks/:id/virtual/snapshots/create`. @@ -568,12 +573,13 @@ POST /api/release-tracks/:id/clone ### Delete Release Track ``` -DELETE /api/release-tracks/:id +DELETE /api/release-tracks/:id?confirm_track_id=:id ``` -**Query Parameters:** - -- `versions` - `latest` (delete only latest, default: all) +This irreversible operation requires the administrator role and removes the +track's complete snapshot history. `confirm_track_id` must exactly equal the +`:id` path parameter. Every accepted attempt is recorded in the durable +release-track destructive audit trail. --- @@ -628,14 +634,16 @@ Creates new snapshot with updated metadata. ### Update Contents (Specific Snapshot) ``` -POST /api/release-tracks/:id/snapshots/:modified/contents +POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id ``` Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** **Request Body:** Same as [Update Contents](#update-contents) for latest snapshot. -Like the latest form, this operation is restricted to standard tracks. +Like the latest form, this operation is restricted to standard tracks, +requires the administrator role and exact track-ID confirmation, and creates +a durable audit event. ### Release/Tag Specific Snapshot @@ -764,7 +772,7 @@ contains the same exact revision in `members` and `candidates`, the transition repairs the duplicate and retains the `members` occurrence. A dynamic candidate remains `"latest"` if it is promoted to staged. -``` +```` POST /api/release-tracks/:id/candidates/review ```/ @@ -776,7 +784,7 @@ POST /api/release-tracks/:id/candidates/review "to": "awaiting-review", "object_refs": [{ "id": "attack-pattern--uuid", "modified": "2024-01-15T10:00:00Z" }] } -``` +```` --- From 8730e8820dd25c095a7759d7bd669cf9b1da2d9e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:13:12 -0400 Subject: [PATCH 36/55] chore(release-tracks): remove pre-release version migration Retain database-enforced release version uniqueness while treating existing beta track collections as disposable development state. --- ...lease-version-uniqueness-migration.spec.js | 74 -------- docs/developer/TODO.md | 37 ++-- .../release-tracks/implementation-notes.md | 14 +- docs/user/release-tracks/versioning.md | 6 - ...nforce-release-track-version-uniqueness.js | 173 ------------------ 5 files changed, 32 insertions(+), 272 deletions(-) delete mode 100644 app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js delete mode 100644 migrations/20260730040000-enforce-release-track-version-uniqueness.js diff --git a/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js b/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js deleted file mode 100644 index 895080d1..00000000 --- a/app/tests/api/release-tracks/release-version-uniqueness-migration.spec.js +++ /dev/null @@ -1,74 +0,0 @@ -'use strict'; - -const { expect } = require('expect'); -const mongoose = require('mongoose'); - -const config = require('../../../config/config'); -const database = require('../../../lib/database-in-memory'); -const databaseConfiguration = require('../../../lib/database-configuration'); -const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); -const migration = require('../../../../migrations/20260730040000-enforce-release-track-version-uniqueness'); - -const UNIQUE_INDEX = 'unique_tagged_version'; -const LEGACY_INDEX = 'id_1_version_1'; - -describe('Release-track tagged-version uniqueness migration', function () { - before(async function () { - await database.initializeConnection(); - await databaseConfiguration.checkSystemConfiguration(); - config.validateRequests.withAttackDataModel = true; - }); - - after(async function () { - await database.closeConnection(); - }); - - it('fails closed on legacy duplicates before replacing indexes and is rerunnable after repair', async function () { - const track = await releaseTracksService.createTrack({ - name: 'Legacy Duplicate Release Versions', - type: 'standard', - }); - const released = await releaseTracksService.releaseLatest(track.id, { - version: '1.0', - userAccountId: 'migration-test', - }); - const collection = mongoose.connection.db.collection(track.id); - - await collection.dropIndex(UNIQUE_INDEX); - await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX }); - - const duplicate = { ...released }; - delete duplicate._id; - duplicate.modified = new Date(new Date(released.modified).getTime() + 1000); - await collection.insertOne(duplicate); - await mongoose.connection.db - .collection('releaseTrackRegistry') - .deleteOne({ track_id: track.id }); - - await expect(migration.up(mongoose.connection.db)).rejects.toMatchObject({ - message: expect.stringContaining(`${track.id} version 1.0 (2 snapshots)`), - duplicates: [ - expect.objectContaining({ - track_id: track.id, - version: '1.0', - }), - ], - }); - - let indexes = await collection.indexes(); - expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(true); - expect(indexes.some((index) => index.name === UNIQUE_INDEX)).toBe(false); - - await collection.deleteOne({ modified: duplicate.modified }); - await migration.up(mongoose.connection.db); - await migration.up(mongoose.connection.db); - - indexes = await collection.indexes(); - expect(indexes.some((index) => index.name === LEGACY_INDEX)).toBe(false); - expect(indexes.find((index) => index.name === UNIQUE_INDEX)).toMatchObject({ - key: { id: 1, version: 1 }, - unique: true, - partialFilterExpression: { version: { $type: 'string' } }, - }); - }); -}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index efb91503..04d21c2a 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -15,19 +15,34 @@ can be reviewed or reverted item by item. identifies the track and requested version. - [x] Add a regression that releases two distinct drafts concurrently with the same version and proves exactly one succeeds. -- [x] Add a rerunnable migration that fails closed on pre-existing duplicates - before replacing the legacy non-unique index. -- [x] Update release-version documentation and run focused, migration, - middleware, lint, and complete-suite verification. - -Verification result (2026-07-30): - -- The deterministic concurrent-release, migration, middleware, and isolated - roaming-failure group passes (39). +- [x] Adopt the pre-release reset policy for collections created with the + former non-unique index. No shared deployment retains beta release-track + data, so this change deliberately does not establish a permanent + migration contract for local development state. +- [x] Update release-version documentation and run focused, middleware, and + lint verification. +- [ ] Obtain one clean aggregate `npm test` run for the migration cleanup. Three + attempts exposed the repository's roaming cross-spec isolation failure; + every affected spec passed immediately in isolation. + +Original implementation verification (2026-07-30): + +- The deterministic concurrent-release, middleware, and isolated + roaming-failure group passes. - The required clean full suite passes: OpenAPI 2, config 21, API 947, middleware 25, and scheduler 10. -- The migration preflights the union of registry IDs and canonical orphan - release-track collection names before making any index changes. +- New dynamic track collections create the unique partial index before their + initial snapshot is persisted. Existing personal development tracks created + under the former beta schema are reset or recreated. + +Pre-release migration cleanup verification (2026-07-30): + +- Release planning and concurrent-version coverage passes all 20 cases. +- Error middleware passes all 9 cases; lint and diff checks pass. +- Aggregate attempts failed in different, unrelated modules: user accounts, + groups, releases-by-object, and tagged-content immutability. Those modules + pass in isolation (9, 23, 8, and 1 cases respectively), confirming no + reproducible migration-cleanup regression. ### P0.2 — Make primary release membership fail closed diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 739b71f3..ec2db9dd 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -19,14 +19,12 @@ whose `version` is a string. Drafts therefore remain unlimited at `version: null`, while the database—not an application-level preflight—decides which concurrent release may claim a version. -Migration `20260730040000-enforce-release-track-version-uniqueness` scans the -union of registered tracks and canonical `release-track--` collection -names before changing any indexes. Including orphan collections matters -because track creation predates transaction-backed registry coordination. If any -`(track_id, version)` has multiple tagged snapshots, it reports all offending -snapshot timestamps and performs no index changes. After operators repair the -data, rerunning the migration replaces the legacy non-unique index -idempotently. +Release tracks are still pre-release, and no shared deployment retains track +data written under the former non-unique index. Existing personal development +tracks are therefore reset or recreated instead of establishing a permanent +upgrade contract for beta data. Once release tracks are formally released, +future index or persistence changes must include an appropriate migration for +supported deployments. ## Validation Rules diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 3290e359..44e6962f 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -196,12 +196,6 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema succeeds and the other receives `409 Conflict` with the conflicting `track_id` and `version`. -Deployments upgrading from an earlier release run a database migration before -serving traffic. The migration checks every release-track collection for -pre-existing duplicate tagged versions and stops without changing indexes if -it finds any. Operators must resolve every reported track/version pair and -rerun the migration; the server does not guess which tagged snapshot to keep. - ### First Tagged Release For release tracks with no prior tagged releases: diff --git a/migrations/20260730040000-enforce-release-track-version-uniqueness.js b/migrations/20260730040000-enforce-release-track-version-uniqueness.js deleted file mode 100644 index 5aed38e6..00000000 --- a/migrations/20260730040000-enforce-release-track-version-uniqueness.js +++ /dev/null @@ -1,173 +0,0 @@ -'use strict'; - -/** - * Replace the legacy non-unique (id, version) index in every dynamic release - * track collection with a unique partial index over tagged snapshots. - * - * The migration preflights every collection before changing any indexes. If a - * deployment already contains duplicate tagged versions, migration stops and - * reports every offending track/version so an operator can repair the data - * deliberately. - */ - -const INDEX_NAME = 'unique_tagged_version'; -const LEGACY_INDEX_NAME = 'id_1_version_1'; -const CONCURRENCY = 8; -const TRACK_COLLECTION_PATTERN = - /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -async function mapWithConcurrency(items, mapper) { - let nextIndex = 0; - - async function worker() { - while (nextIndex < items.length) { - const index = nextIndex++; - await mapper(items[index]); - } - } - - await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, () => worker())); -} - -async function collectionExists(db, trackId) { - return db.listCollections({ name: trackId }, { nameOnly: true }).hasNext(); -} - -async function findTrackCollectionIds(db) { - const [registeredTracks, collections] = await Promise.all([ - db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), - db.listCollections({}, { nameOnly: true }).toArray(), - ]); - - return Array.from( - new Set([ - ...registeredTracks.map((track) => track.track_id), - ...collections - .map((collection) => collection.name) - .filter((name) => TRACK_COLLECTION_PATTERN.test(name)), - ]), - ).sort(); -} - -async function findDuplicateVersions(db, trackIds) { - const duplicates = []; - - await mapWithConcurrency(trackIds, async (trackId) => { - if (!(await collectionExists(db, trackId))) return; - - const matches = await db - .collection(trackId) - .aggregate([ - { $match: { version: { $type: 'string' } } }, - { - $group: { - _id: { id: '$id', version: '$version' }, - count: { $sum: 1 }, - snapshots: { $push: '$modified' }, - }, - }, - { $match: { count: { $gt: 1 } } }, - { $sort: { '_id.version': 1 } }, - ]) - .toArray(); - - for (const match of matches) { - duplicates.push({ - track_id: trackId, - version: match._id.version, - snapshots: match.snapshots, - }); - } - }); - - return duplicates.sort( - (left, right) => - left.track_id.localeCompare(right.track_id) || left.version.localeCompare(right.version), - ); -} - -function isDesiredIndex(index) { - return ( - index?.name === INDEX_NAME && - index.unique === true && - index.key?.id === 1 && - index.key?.version === 1 && - index.partialFilterExpression?.version?.$type === 'string' - ); -} - -async function installUniqueIndex(db, trackId) { - if (!(await collectionExists(db, trackId))) return; - - const collection = db.collection(trackId); - const indexes = await collection.indexes(); - const desired = indexes.find((index) => index.name === INDEX_NAME); - if (isDesiredIndex(desired)) { - if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { - await collection.dropIndex(LEGACY_INDEX_NAME); - } - return; - } - - if (desired) await collection.dropIndex(INDEX_NAME); - if (indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { - await collection.dropIndex(LEGACY_INDEX_NAME); - } - - await collection.createIndex( - { id: 1, version: 1 }, - { - name: INDEX_NAME, - unique: true, - partialFilterExpression: { version: { $type: 'string' } }, - }, - ); -} - -module.exports = { - async up(db) { - const trackIds = await findTrackCollectionIds(db); - const duplicates = await findDuplicateVersions(db, trackIds); - - if (duplicates.length > 0) { - const summary = duplicates - .map( - (duplicate) => - `${duplicate.track_id} version ${duplicate.version} ` + - `(${duplicate.snapshots.length} snapshots)`, - ) - .join('; '); - const error = new Error( - `Duplicate tagged release versions detected; repair them before retrying migration: ${summary}`, - ); - error.duplicates = duplicates; - throw error; - } - - await mapWithConcurrency(trackIds, (trackId) => installUniqueIndex(db, trackId)); - }, - - async down(db) { - const trackIds = await findTrackCollectionIds(db); - - await mapWithConcurrency(trackIds, async (trackId) => { - if (!(await collectionExists(db, trackId))) return; - - const collection = db.collection(trackId); - const indexes = await collection.indexes(); - if (indexes.some((index) => index.name === INDEX_NAME)) { - await collection.dropIndex(INDEX_NAME); - } - if (!indexes.some((index) => index.name === LEGACY_INDEX_NAME)) { - await collection.createIndex({ id: 1, version: 1 }, { name: LEGACY_INDEX_NAME }); - } - }); - }, - - _private: { - findTrackCollectionIds, - findDuplicateVersions, - installUniqueIndex, - isDesiredIndex, - }, -}; From 86d11ed661c590e9edc1d465af826328732d3406 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:56:52 -0400 Subject: [PATCH 37/55] feat(release-tracks): make snapshot bundles deterministic Persist exact primary, relationship, and secondary revisions in snapshot graph manifests. Pin relationship endpoints, protect referenced revisions, and add migration tooling for existing data. Make persisted snapshot history immutable by removing direct contents and historical metadata mutation routes. Limit snapshot deletion to the latest untagged draft and align the API contract, tests, and documentation. --- .../definitions/components/release-tracks.yml | 6 + app/api/definitions/openapi.yml | 9 - app/api/definitions/paths/analytics-paths.yml | 6 +- app/api/definitions/paths/assets-paths.yml | 4 +- app/api/definitions/paths/campaigns-paths.yml | 6 +- .../definitions/paths/collections-paths.yml | 4 + .../paths/data-components-paths.yml | 6 +- .../definitions/paths/data-sources-paths.yml | 4 +- .../paths/detection-strategies-paths.yml | 6 +- app/api/definitions/paths/groups-paths.yml | 6 +- .../definitions/paths/identities-paths.yml | 4 +- app/api/definitions/paths/matrices-paths.yml | 6 +- .../definitions/paths/mitigations-paths.yml | 6 +- app/api/definitions/paths/notes-paths.yml | 6 +- .../definitions/paths/relationships-paths.yml | 6 +- .../paths/release-tracks-paths.yml | 132 +--- app/api/definitions/paths/software-paths.yml | 6 +- app/api/definitions/paths/tactics-paths.yml | 6 +- .../definitions/paths/techniques-paths.yml | 6 +- app/controllers/collections-controller.js | 4 +- app/controllers/release-tracks-controller.js | 85 -- app/exceptions/index.js | 24 + app/lib/error-handler.js | 4 + app/lib/linkById.js | 17 + app/lib/release-tracks/backref-reconciler.js | 2 +- .../release-tracks/release-track-schemas.js | 13 - app/models/relationship-model.js | 20 + .../release-track-audit-event-model.js | 2 +- .../release-track-graph-manifest-model.js | 85 ++ .../release-track-snapshot-schema.js | 1 + app/routes/release-tracks-routes.js | 31 +- app/services/meta-classes/base.service.js | 50 ++ app/services/release-tracks/export-service.js | 157 +--- .../release-tracks/graph-manifest-service.js | 542 +++++++++++++ .../primary-revision-service.js | 1 + .../release-tracks/release-tracks-service.js | 50 +- .../release-tracks/snapshot-service.js | 167 ++-- .../release-tracks/versioning-service.js | 41 +- app/services/stix/bundle-graph-resolver.js | 469 +++++++++++ app/services/stix/collections-service.js | 42 + app/services/stix/relationships-service.js | 161 ++++ app/services/stix/stix-bundles-service.js | 394 +--------- .../api/attack-objects/attack-objects.spec.js | 6 +- .../relationship-endpoint-pins.spec.js | 170 ++++ .../relationships-pagination.spec.js | 33 + .../api/relationships/relationships.spec.js | 104 +++ .../destructive-authorization.spec.js | 109 +-- .../deterministic-graph-migration.spec.js | 198 +++++ .../release-tracks/ephemeral-bundle.spec.js | 68 +- .../primary-revision-integrity.spec.js | 34 +- .../reconciliation-durability.spec.js | 5 +- .../release-track-test-helpers.js | 49 ++ .../release-tracks-backrefs.spec.js | 35 +- .../release-tracks-bundle.spec.js | 152 +++- .../release-tracks-change-capture.spec.js | 9 +- .../release-tracks-release.spec.js | 32 +- .../release-tracks-tier-invariant.spec.js | 8 +- .../api/release-tracks/release-tracks.spec.js | 17 +- .../release-tracks/releases-by-object.spec.js | 55 +- .../snapshot-immutability.spec.js | 104 +++ .../tagged-content-immutability.spec.js | 23 +- .../virtual-deduplication.spec.js | 13 +- .../virtual-determinism.spec.js | 17 +- .../virtual-domain-filters.spec.js | 19 +- .../virtual-object-type-filters.spec.js | 13 +- .../release-tracks/virtual-quarantine.spec.js | 11 +- app/tests/api/reports/reports.spec.js | 15 + docs/README.md | 3 +- docs/admin/release-track-audit.md | 10 +- docs/admin/release-track-graph-migration.md | 53 ++ docs/developer/FRONTEND_TODO.md | 161 +++- docs/developer/TODO.md | 743 +++++++++++------- .../developer/release-tracks/authorization.md | 22 +- .../release-tracks/backref-reconciliation.md | 8 +- .../developer/release-tracks/bundle-export.md | 121 +-- docs/developer/release-tracks/entities.md | 10 +- .../release-tracks/error-handling.md | 30 +- .../release-tracks/implementation-notes.md | 14 +- .../release-tracks/member-sync-strategies.md | 4 + docs/user/release-tracks/api-reference.md | 95 +-- docs/user/release-tracks/object-backrefs.md | 15 +- docs/user/release-tracks/summary.md | 20 +- docs/user/release-tracks/virtual-tracks.md | 13 +- docs/user/release-tracks/workflow-examples.md | 97 ++- ...-backfill-deterministic-snapshot-graphs.js | 276 +++++++ package.json | 1 + ...viewDeterministicSnapshotGraphMigration.js | 22 + 87 files changed, 3745 insertions(+), 1869 deletions(-) create mode 100644 app/models/release-tracks/release-track-graph-manifest-model.js create mode 100644 app/services/release-tracks/graph-manifest-service.js create mode 100644 app/services/stix/bundle-graph-resolver.js create mode 100644 app/tests/api/relationships/relationship-endpoint-pins.spec.js create mode 100644 app/tests/api/release-tracks/deterministic-graph-migration.spec.js create mode 100644 app/tests/api/release-tracks/release-track-test-helpers.js create mode 100644 app/tests/api/release-tracks/snapshot-immutability.spec.js create mode 100644 docs/admin/release-track-graph-migration.md create mode 100644 migrations/20260730180000-backfill-deterministic-snapshot-graphs.js create mode 100644 scripts/previewDeterministicSnapshotGraphMigration.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index d1549685..d973f9d4 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -23,6 +23,12 @@ components: nullable: true description: 'Semantic version (e.g., "1.0", "2.1") if tagged, null for draft snapshots' example: '1.0' + graph_manifest_id: + type: string + readOnly: true + description: | + Server-controlled identifier for the frozen bundle graph associated + with this snapshot. Clients should treat this value as opaque. name: type: string description: 'Human-readable track name' diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index dec2b7a3..07a7c20f 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -352,9 +352,6 @@ paths: /api/release-tracks/{id}/meta: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1meta' - /api/release-tracks/{id}/contents: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1contents' - /api/release-tracks/{id}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1clone' @@ -409,12 +406,6 @@ paths: /api/release-tracks/{id}/snapshots/{modified}: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}' - /api/release-tracks/{id}/snapshots/{modified}/meta: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1meta' - - /api/release-tracks/{id}/snapshots/{modified}/contents: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1contents' - /api/release-tracks/{id}/snapshots/{modified}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1clone' diff --git a/app/api/definitions/paths/analytics-paths.yml b/app/api/definitions/paths/analytics-paths.yml index 16d84795..dbab5651 100644 --- a/app/api/definitions/paths/analytics-paths.yml +++ b/app/api/definitions/paths/analytics-paths.yml @@ -206,7 +206,7 @@ paths: '404': description: 'A analytic with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/analytics/{stixId}/modified/{modified}: get: @@ -278,7 +278,7 @@ paths: '404': description: 'A analytic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a analytic' operationId: 'analytic-delete' @@ -306,4 +306,4 @@ paths: '404': description: 'A analytic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/assets-paths.yml b/app/api/definitions/paths/assets-paths.yml index 594eac6c..4f76dde8 100644 --- a/app/api/definitions/paths/assets-paths.yml +++ b/app/api/definitions/paths/assets-paths.yml @@ -270,7 +270,7 @@ paths: '404': description: 'An asset with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete an asset' operationId: 'asset-delete' @@ -298,7 +298,7 @@ paths: '404': description: 'An asset with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/assets/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/campaigns-paths.yml b/app/api/definitions/paths/campaigns-paths.yml index 37ec83bb..e02dff77 100644 --- a/app/api/definitions/paths/campaigns-paths.yml +++ b/app/api/definitions/paths/campaigns-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A campaign with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/campaigns/{stixId}/modified/{modified}: get: @@ -250,7 +250,7 @@ paths: '404': description: 'A campaign with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a campaign' operationId: 'campaign-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A campaign with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/campaigns/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/collections-paths.yml b/app/api/definitions/paths/collections-paths.yml index f2bcb30f..b7e8b832 100644 --- a/app/api/definitions/paths/collections-paths.yml +++ b/app/api/definitions/paths/collections-paths.yml @@ -199,6 +199,8 @@ paths: description: 'The collections were successfully deleted.' '404': description: 'A collection with the requested STIX id was not found.' + '409': + description: 'The collection revision or a requested cascade-delete target is pinned by release-track membership or a snapshot graph manifest and cannot be deleted.' /api/collections/{stixId}/modified/{modified}: get: @@ -277,3 +279,5 @@ paths: description: 'The collection was successfully deleted.' '404': description: 'A collection with the requested STIX id was not found.' + '409': + description: 'The collection revision or a requested cascade-delete target is pinned by release-track membership or a snapshot graph manifest and cannot be deleted.' diff --git a/app/api/definitions/paths/data-components-paths.yml b/app/api/definitions/paths/data-components-paths.yml index 1e260c73..b110bd3f 100644 --- a/app/api/definitions/paths/data-components-paths.yml +++ b/app/api/definitions/paths/data-components-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A data component with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/data-components/{stixId}/channels: get: @@ -316,7 +316,7 @@ paths: '404': description: 'A data component with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a data component' operationId: 'data-component-delete' @@ -344,7 +344,7 @@ paths: '404': description: 'A data component with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/data-components/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/data-sources-paths.yml b/app/api/definitions/paths/data-sources-paths.yml index c301a6af..bc33482c 100644 --- a/app/api/definitions/paths/data-sources-paths.yml +++ b/app/api/definitions/paths/data-sources-paths.yml @@ -286,7 +286,7 @@ paths: '404': description: 'A data source with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a data source' operationId: 'data-source-delete' @@ -314,7 +314,7 @@ paths: '404': description: 'A data source with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/data-sources/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/detection-strategies-paths.yml b/app/api/definitions/paths/detection-strategies-paths.yml index a038c768..d83dc126 100644 --- a/app/api/definitions/paths/detection-strategies-paths.yml +++ b/app/api/definitions/paths/detection-strategies-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A detection strategy with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/detection-strategies/{stixId}/modified/{modified}: get: @@ -262,7 +262,7 @@ paths: '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a detection strategy' operationId: 'detection-strategy-delete' @@ -290,4 +290,4 @@ paths: '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/groups-paths.yml b/app/api/definitions/paths/groups-paths.yml index cbe8dd3b..c6e39178 100644 --- a/app/api/definitions/paths/groups-paths.yml +++ b/app/api/definitions/paths/groups-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A group with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/groups/{stixId}/modified/{modified}: get: @@ -250,7 +250,7 @@ paths: '404': description: 'A group with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a group' operationId: 'group-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A group with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/groups/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/identities-paths.yml b/app/api/definitions/paths/identities-paths.yml index 1c104789..10e83651 100644 --- a/app/api/definitions/paths/identities-paths.yml +++ b/app/api/definitions/paths/identities-paths.yml @@ -227,7 +227,7 @@ paths: '404': description: 'An identity with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a identity' operationId: 'identity-delete' @@ -255,4 +255,4 @@ paths: '404': description: 'An identity with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/matrices-paths.yml b/app/api/definitions/paths/matrices-paths.yml index a49d0123..45cc2118 100644 --- a/app/api/definitions/paths/matrices-paths.yml +++ b/app/api/definitions/paths/matrices-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A matrix with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}: get: @@ -250,7 +250,7 @@ paths: '404': description: 'A matrix with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a matrix' operationId: 'matrix-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A matrix with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/mitigations-paths.yml b/app/api/definitions/paths/mitigations-paths.yml index f1253584..2e3bde43 100644 --- a/app/api/definitions/paths/mitigations-paths.yml +++ b/app/api/definitions/paths/mitigations-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A mitigation with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/mitigations/{stixId}/modified/{modified}: get: @@ -262,7 +262,7 @@ paths: '404': description: 'A mitigation with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a mitigation' operationId: 'mitigation-delete' @@ -290,7 +290,7 @@ paths: '404': description: 'A mitigation with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/mitigations/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/notes-paths.yml b/app/api/definitions/paths/notes-paths.yml index f83752ef..eebad19a 100644 --- a/app/api/definitions/paths/notes-paths.yml +++ b/app/api/definitions/paths/notes-paths.yml @@ -176,7 +176,7 @@ paths: '404': description: 'A note with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/notes/{stixId}/modified/{modified}: get: @@ -247,7 +247,7 @@ paths: '404': description: 'A note with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a note' operationId: 'note-delete-version' @@ -275,4 +275,4 @@ paths: '404': description: 'A note with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/relationships-paths.yml b/app/api/definitions/paths/relationships-paths.yml index 0de8060f..69fd5c03 100644 --- a/app/api/definitions/paths/relationships-paths.yml +++ b/app/api/definitions/paths/relationships-paths.yml @@ -248,7 +248,7 @@ paths: '404': description: 'A relationship with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/relationships/{stixId}/modified/{modified}: get: @@ -320,7 +320,7 @@ paths: '404': description: 'A relationship with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a relationship' operationId: 'relationship-delete' @@ -348,4 +348,4 @@ paths: '404': description: 'A relationship with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 94917925..4a001ef4 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -346,54 +346,6 @@ paths: schema: $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' - /api/release-tracks/{id}/contents: - post: - summary: 'Update member contents on the latest snapshot' - operationId: 'release-tracks-update-contents-latest' - description: | - Replace the members tier of a standard track with new contents - (x_mitre_contents format). Virtual membership can only be produced by - POST /api/release-tracks/{id}/virtual/snapshots/create. - obj_modified may be an exact ISO timestamp or the request-time - shorthand "latest"; the server resolves "latest" to the actual latest - stix.modified timestamp before persistence. Snapshot members always - store exact revision pins. - Exact revisions already present in another tier are retained only in members; - different revisions of the same object remain valid across tiers. - Creates a new snapshot clone. - This administrator-only operation requires confirm_track_id to exactly - match the target track and writes a durable audit event. - Request body validated via Zod in controller. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: confirm_track_id - in: query - required: true - description: 'Must exactly equal the id path parameter' - schema: - type: string - responses: - '200': - description: 'Contents updated successfully' - '400': - description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' - '401': - description: 'Administrator role required' - '500': - description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' - content: - application/json: - schema: - oneOf: - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' - /api/release-tracks/{id}/clone: post: summary: 'Clone the latest snapshot into a new release track' @@ -1233,8 +1185,10 @@ paths: summary: 'Delete a specific snapshot' operationId: 'release-tracks-snapshot-delete' description: | - Delete a snapshot by its modified timestamp. - Cannot delete tagged snapshots. + Delete the latest untagged draft snapshot by its modified timestamp. + Tagged snapshots and historical drafts are immutable and cannot be + deleted. Deleting the latest draft reverts the track to its immediately + preceding snapshot. tags: - 'Release Tracks' parameters: @@ -1252,86 +1206,10 @@ paths: '204': description: 'Snapshot deleted successfully' '409': - description: 'Cannot delete tagged snapshot' + description: 'Cannot delete a tagged snapshot or a historical draft' '404': description: 'Snapshot not found' - /api/release-tracks/{id}/snapshots/{modified}/meta: - post: - summary: 'Update metadata on a specific snapshot' - operationId: 'release-tracks-update-meta-by-modified' - description: | - Update metadata on a historical snapshot. - Request body validated via Zod in controller. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: modified - in: path - required: true - schema: - type: string - responses: - '200': - description: 'Metadata updated successfully' - - /api/release-tracks/{id}/snapshots/{modified}/contents: - post: - summary: 'Update contents on a specific snapshot' - operationId: 'release-tracks-update-contents-by-modified' - description: | - Update member contents on a historical standard-track snapshot. - Virtual membership can only be produced by - POST /api/release-tracks/{id}/virtual/snapshots/create. - obj_modified may be an exact ISO timestamp or the request-time - shorthand "latest"; the server resolves "latest" to the actual latest - stix.modified timestamp before persistence. Snapshot members always - store exact revision pins. - Exact revisions already present in another tier are retained only in members; - different revisions of the same object remain valid across tiers. - This administrator-only operation requires confirm_track_id to exactly - match the target track and writes a durable audit event. - Request body validated via Zod in controller. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: modified - in: path - required: true - schema: - type: string - - name: confirm_track_id - in: query - required: true - description: 'Must exactly equal the id path parameter' - schema: - type: string - responses: - '200': - description: 'Contents updated successfully' - '400': - description: 'Track is virtual, the contents request is invalid, or a requested revision does not exist' - '401': - description: 'Administrator role required' - '500': - description: 'The replacement may be persisted, but its durable audit or membership reconciliation failed' - content: - application/json: - schema: - oneOf: - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-audit-error' - - $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' - /api/release-tracks/{id}/snapshots/{modified}/clone: post: summary: 'Clone a specific snapshot into a new release track' diff --git a/app/api/definitions/paths/software-paths.yml b/app/api/definitions/paths/software-paths.yml index 33831f85..2d90bc18 100644 --- a/app/api/definitions/paths/software-paths.yml +++ b/app/api/definitions/paths/software-paths.yml @@ -201,7 +201,7 @@ paths: '404': description: 'A tactic with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/software/{stixId}/modified/{modified}: get: @@ -272,7 +272,7 @@ paths: '404': description: 'A software object with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a software object' operationId: 'software-delete' @@ -300,7 +300,7 @@ paths: '404': description: 'A software object with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/software/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/tactics-paths.yml b/app/api/definitions/paths/tactics-paths.yml index 6caadde9..e007ca7d 100644 --- a/app/api/definitions/paths/tactics-paths.yml +++ b/app/api/definitions/paths/tactics-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A tactic with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}: get: @@ -262,7 +262,7 @@ paths: '404': description: 'A tactic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a tactic' operationId: 'tactic-delete' @@ -290,7 +290,7 @@ paths: '404': description: 'A tactic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/techniques-paths.yml b/app/api/definitions/paths/techniques-paths.yml index 050535e6..773af422 100644 --- a/app/api/definitions/paths/techniques-paths.yml +++ b/app/api/definitions/paths/techniques-paths.yml @@ -214,7 +214,7 @@ paths: '404': description: 'A technique with the requested STIX id was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}: get: @@ -286,7 +286,7 @@ paths: '404': description: 'A technique with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' delete: summary: 'Delete a technique' operationId: 'technique-delete' @@ -314,7 +314,7 @@ paths: '404': description: 'A technique with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned in the members tier of a release track (released content) and cannot be modified or deleted in place. Create a new revision instead.' + description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}/tactics: get: diff --git a/app/controllers/collections-controller.js b/app/controllers/collections-controller.js index 255a2a49..701ca9d9 100644 --- a/app/controllers/collections-controller.js +++ b/app/controllers/collections-controller.js @@ -191,7 +191,7 @@ exports.create = async function (req, res) { } }; -exports.delete = async function (req, res) { +exports.delete = async function (req, res, next) { try { const removedCollections = await collectionsService.delete( req.params.stixId, @@ -205,7 +205,7 @@ exports.delete = async function (req, res) { } } catch (error) { logger.error('Delete collections failed. ' + error); - return res.status(500).send('Unable to delete collections. Server error.'); + return next(error); } }; diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index fb61574b..4b1f9fec 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -38,7 +38,6 @@ const { createTrackBodySchema, createFromBundleBodySchema, updateMetadataBodySchema, - updateContentsBodySchema, releaseBodySchema, releaseVersionSelectionSchema, cloneBodySchema, @@ -461,34 +460,6 @@ exports.updateMetadataByLatest = async function updateMetadataByLatest(req, res, } }; -/** POST /api/release-tracks/:id/contents */ -exports.updateContentsByLatest = async function updateContentsByLatest(req, res, next) { - try { - requireDestructiveConfirmation(req); - const bodyResult = updateContentsBodySchema.safeParse(req.body); - if (!bodyResult.success) { - return next( - new BadRequestError({ - message: 'Invalid contents update', - details: bodyResult.error.errors, - }), - ); - } - - const result = await releaseTracksService.updateContents( - req.params.id, - bodyResult.data, - destructiveActor(req), - req.query.confirm_track_id, - ); - logger.debug(`Success: Updated contents for track ${req.params.id}`); - return res.status(200).send(result); - } catch (err) { - logger.error('Failed to update track contents: ' + err); - return next(err); - } -}; - /** POST /api/release-tracks/:id/snapshots/latest/release */ exports.releaseLatest = async function releaseLatest(req, res, next) { try { @@ -585,62 +556,6 @@ exports.retrieveSnapshotByModified = async function retrieveSnapshotByModified(r } }; -/** POST /api/release-tracks/:id/snapshots/:modified/meta */ -exports.updateMetadataByModified = async function updateMetadataByModified(req, res, next) { - try { - const bodyResult = updateMetadataBodySchema.safeParse(req.body); - if (!bodyResult.success) { - return next( - new BadRequestError({ - message: 'Invalid metadata update', - details: bodyResult.error.errors, - }), - ); - } - - const result = await releaseTracksService.updateMetadataByModified( - req.params.id, - req.params.modified, - bodyResult.data, - req.user?.userAccountId, - ); - logger.debug(`Success: Updated metadata for snapshot ${req.params.modified}`); - return res.status(200).send(result); - } catch (err) { - logger.error('Failed to update snapshot metadata: ' + err); - return next(err); - } -}; - -/** POST /api/release-tracks/:id/snapshots/:modified/contents */ -exports.updateContentsByModified = async function updateContentsByModified(req, res, next) { - try { - requireDestructiveConfirmation(req); - const bodyResult = updateContentsBodySchema.safeParse(req.body); - if (!bodyResult.success) { - return next( - new BadRequestError({ - message: 'Invalid contents update', - details: bodyResult.error.errors, - }), - ); - } - - const result = await releaseTracksService.updateContentsByModified( - req.params.id, - req.params.modified, - bodyResult.data, - destructiveActor(req), - req.query.confirm_track_id, - ); - logger.debug(`Success: Updated contents for snapshot ${req.params.modified}`); - return res.status(200).send(result); - } catch (err) { - logger.error('Failed to update snapshot contents: ' + err); - return next(err); - } -}; - /** POST /api/release-tracks/:id/snapshots/:modified/release */ exports.releaseByModified = async function releaseByModified(req, res, next) { try { diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 8be2df56..88747877 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -351,6 +351,18 @@ class TaggedSnapshotDeletionError extends CustomError { } } +class HistoricalSnapshotDeletionError extends CustomError { + constructor(snapshotModified, latestSnapshotModified, options = {}) { + super('Only the latest untagged snapshot can be deleted', { + ...options, + snapshot_modified: new Date(snapshotModified).toISOString(), + latest_snapshot_modified: latestSnapshotModified + ? new Date(latestSnapshotModified).toISOString() + : null, + }); + } +} + class MemberPinnedRevisionError extends CustomError { constructor(options) { super( @@ -362,6 +374,16 @@ class MemberPinnedRevisionError extends CustomError { } } +class SnapshotGraphPinnedRevisionError extends CustomError { + constructor(options) { + super( + 'This revision is frozen in a release-track snapshot graph and cannot be modified or ' + + 'deleted in place. Create a new revision instead.', + options, + ); + } +} + class InvalidVersionError extends CustomError { constructor(message, options) { super(message || 'Invalid version', options); @@ -437,6 +459,7 @@ module.exports = { DuplicateReleaseVersionError, InvalidObjectRevisionError, TaggedSnapshotDeletionError, + HistoricalSnapshotDeletionError, InvalidVersionError, //** Release track errors */ @@ -449,6 +472,7 @@ module.exports = { VirtualSnapshotNotMaterializedError, TrackNotFoundError, MemberPinnedRevisionError, + SnapshotGraphPinnedRevisionError, //** Database-related errors */ DuplicateIdError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index c187dc47..7ad83715 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -39,6 +39,7 @@ const { DuplicateReleaseVersionError, InvalidObjectRevisionError, TaggedSnapshotDeletionError, + HistoricalSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, ReleaseContentIntegrityError, @@ -49,6 +50,7 @@ const { VirtualSnapshotNotMaterializedError, TrackNotFoundError, MemberPinnedRevisionError, + SnapshotGraphPinnedRevisionError, ObjectHasValidationIssuesError, } = require('../exceptions'); @@ -140,10 +142,12 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof AlreadyReleasedError || err instanceof DuplicateReleaseVersionError || err instanceof TaggedSnapshotDeletionError || + err instanceof HistoricalSnapshotDeletionError || err instanceof ReleaseConflictError || err instanceof ReleaseContentIntegrityError || err instanceof VirtualSnapshotNotMaterializedError || err instanceof MemberPinnedRevisionError || + err instanceof SnapshotGraphPinnedRevisionError || err instanceof ObjectHasValidationIssuesError || err instanceof ActiveOrganizationIdentityDeleteError ) { diff --git a/app/lib/linkById.js b/app/lib/linkById.js index c37707ec..c46188f6 100644 --- a/app/lib/linkById.js +++ b/app/lib/linkById.js @@ -23,6 +23,23 @@ function attackReference(externalReferences) { } const linkByIdRegex = /\(LinkById: ([A-Z]+[0-9]+(\.[0-9]+)?)\)/g; + +function extractLinkByIds(stixObject) { + const values = [ + stixObject?.description, + stixObject?.type === 'attack-pattern' ? stixObject.x_mitre_detection : undefined, + ...(stixObject?.external_references || []).map((reference) => reference.description), + ]; + const attackIds = new Set(); + for (const value of values) { + for (const match of value?.matchAll(linkByIdRegex) || []) { + attackIds.add(match[1]); + } + } + return [...attackIds]; +} +exports.extractLinkByIds = extractLinkByIds; + async function convertLinkById(text, getAttackObject) { if (text) { let convertedText = ''; diff --git a/app/lib/release-tracks/backref-reconciler.js b/app/lib/release-tracks/backref-reconciler.js index 0a539cc2..b83ddf08 100644 --- a/app/lib/release-tracks/backref-reconciler.js +++ b/app/lib/release-tracks/backref-reconciler.js @@ -21,7 +21,7 @@ // (latest) snapshot, compute the desired set of backrefs and diff it against // the documents that currently carry an entry for that track. This single // code path covers every membership mutation (add/remove/review/promote/ -// demote/release/member-sync/clone/bundle-import/updateContents) as well as +// demote/release/member-sync/clone/bundle-import) as well as // snapshot deletion (membership reverts to the new latest snapshot) and // track deletion (snapshot = null removes all entries). // diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 8f0c6d85..aaf9d8df 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -398,18 +398,6 @@ const updateMetadataBodySchema = z.object({ object_marking_refs: z.array(stixIdentifierSchema).optional(), }); -/** POST /release-tracks/:id/contents */ -const updateContentsBodySchema = z.object({ - x_mitre_contents: z - .array( - z.object({ - obj_ref: stixIdentifierSchema, - obj_modified: z.iso.datetime().or(z.literal('latest')), - }), - ) - .min(1), -}); - const releaseVersionSelectionSchema = z .object({ increment: releaseIncrementSchema.optional(), @@ -567,7 +555,6 @@ module.exports = { createTrackBodySchema, createFromBundleBodySchema, updateMetadataBodySchema, - updateContentsBodySchema, releaseBodySchema, cloneBodySchema, addCandidatesBodySchema, diff --git a/app/models/relationship-model.js b/app/models/relationship-model.js index 223aa4d6..2a794940 100644 --- a/app/models/relationship-model.js +++ b/app/models/relationship-model.js @@ -23,10 +23,22 @@ const relationshipProperties = { x_mitre_log_source_channel: String, }; +const exactObjectRevision = { + object_ref: { type: String, required: true }, + object_modified: { type: Date, required: true }, +}; +const exactObjectRevisionSchema = new mongoose.Schema(exactObjectRevision, { + _id: false, +}); + // Create the definition const relationshipDefinition = { workspace: { ...workspaceDefinitions.common, + relationship_endpoints: { + source: { type: exactObjectRevisionSchema, required: true }, + target: { type: exactObjectRevisionSchema, required: true }, + }, }, stix: { ...stixCoreDefinitions.commonRequiredSDO, @@ -43,6 +55,14 @@ relationshipSchema.index({ 'stix.id': 1, 'stix.modified': -1 }, { unique: true } // Multikey index supporting reverse lookups from release tracks // (release-track backref reconciliation queries by workspace.release_tracks.id) relationshipSchema.index({ 'workspace.release_tracks.id': 1 }, { sparse: true }); +relationshipSchema.index({ + 'workspace.relationship_endpoints.source.object_ref': 1, + 'workspace.relationship_endpoints.source.object_modified': 1, +}); +relationshipSchema.index({ + 'workspace.relationship_endpoints.target.object_ref': 1, + 'workspace.relationship_endpoints.target.object_modified': 1, +}); // Create the model const RelationshipModel = mongoose.model(ModelName.Relationship, relationshipSchema); diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js index ded1fa76..2218f47f 100644 --- a/app/models/release-tracks/release-track-audit-event-model.js +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -9,7 +9,7 @@ const releaseTrackAuditEventSchema = new mongoose.Schema( action: { type: String, required: true, - enum: ['replace_members_latest', 'replace_members_historical', 'delete_track'], + enum: ['delete_track'], }, track_id: { type: String, required: true, validate: validateTrackId }, status: { diff --git a/app/models/release-tracks/release-track-graph-manifest-model.js b/app/models/release-tracks/release-track-graph-manifest-model.js new file mode 100644 index 00000000..07f049d1 --- /dev/null +++ b/app/models/release-tracks/release-track-graph-manifest-model.js @@ -0,0 +1,85 @@ +'use strict'; + +const mongoose = require('mongoose'); + +const exactRevisionSchema = new mongoose.Schema( + { + object_ref: { type: String, required: true }, + object_modified: { type: Date, required: true }, + }, + { _id: false }, +); + +const manifestSchema = new mongoose.Schema( + { + manifest_id: { type: String, required: true, unique: true }, + track_id: { type: String, required: true }, + snapshot_modified: { type: Date, required: true }, + state: { + type: String, + enum: ['pending', 'active'], + required: true, + default: 'pending', + }, + schema_version: { type: Number, required: true, default: 1 }, + resolver_version: { type: String, required: true }, + baseline_reconstruction: { type: Boolean, required: true, default: false }, + created_at: { type: Date, required: true, default: Date.now }, + }, + { collection: 'releaseTrackGraphManifests' }, +); + +manifestSchema.index( + { track_id: 1, snapshot_modified: 1, state: 1 }, + { name: 'manifest_by_snapshot' }, +); + +const entrySchema = new mongoose.Schema( + { + manifest_id: { type: String, required: true }, + track_id: { type: String, required: true }, + snapshot_modified: { type: Date, required: true }, + revision_key: { type: String, required: true }, + kind: { + type: String, + enum: ['root', 'relationship', 'secondary', 'supporting', 'link_target'], + required: true, + }, + tier: { + type: String, + enum: ['members', 'staged', 'candidates', 'quarantine'], + }, + object_status: { type: String }, + object_ref: { type: String, required: true }, + object_modified: { type: Date }, + source: { type: exactRevisionSchema }, + target: { type: exactRevisionSchema }, + discovered_from: { type: [exactRevisionSchema], default: undefined }, + // Relationship payloads are frozen so description-only corrections do + // not change older bundles. Marking definitions are not STIX-versioned, + // so their complete payload is frozen for the same replay guarantee. + frozen_stix: { type: mongoose.Schema.Types.Mixed }, + }, + { collection: 'releaseTrackGraphManifestEntries' }, +); + +entrySchema.index( + { manifest_id: 1, revision_key: 1, kind: 1, tier: 1 }, + { unique: true, name: 'unique_manifest_entry' }, +); +entrySchema.index( + { object_ref: 1, object_modified: 1, manifest_id: 1 }, + { name: 'manifest_revision_protection' }, +); +entrySchema.index({ manifest_id: 1, kind: 1, tier: 1 }); + +const ReleaseTrackGraphManifest = mongoose.model('ReleaseTrackGraphManifest', manifestSchema); +const ReleaseTrackGraphManifestEntry = mongoose.model( + 'ReleaseTrackGraphManifestEntry', + entrySchema, +); + +module.exports = { + ReleaseTrackGraphManifest, + ReleaseTrackGraphManifestEntry, +}; diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 814a7190..ee25cfcc 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -364,6 +364,7 @@ const releaseTrackSnapshotDefinition = { default: null, validate: validateVersion, }, + graph_manifest_id: { type: String }, // Release track metadata name: { diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 40838dc7..8204fca0 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -76,19 +76,6 @@ router releaseTracksController.updateMetadataByLatest, ); -/** - * !!IMPORTANT - * The following endpoint is considered dangerous. It is intended for retroactive hotfixes only. Thus, only admins may use it. - * The main workflow for enrolling new member objects into members is through the candidate-staging promotion cycle. - */ -router - .route('/release-tracks/:id/contents') - .post( - authn.authenticate, - authz.requireRole(authz.admin), - releaseTracksController.updateContentsByLatest, - ); - router .route('/release-tracks/:id/clone') .post( @@ -248,25 +235,9 @@ router ); // ============================================================================= -// Snapshot-specific operations (parameterised by :modified) +// Snapshot-specific read, release, clone, and deletion operations // ============================================================================= -router - .route('/release-tracks/:id/snapshots/:modified/meta') - .post( - authn.authenticate, - authz.requireRole(authz.editorOrHigher), - releaseTracksController.updateMetadataByModified, - ); - -router - .route('/release-tracks/:id/snapshots/:modified/contents') - .post( - authn.authenticate, - authz.requireRole(authz.admin), - releaseTracksController.updateContentsByModified, - ); - router .route('/release-tracks/:id/snapshots/:modified/release/preview') .get( diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 30a9b0c6..4bf2512d 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -23,6 +23,7 @@ const { AlreadyRevokedError, SelfRevocationError, MemberPinnedRevisionError, + SnapshotGraphPinnedRevisionError, } = require('../../exceptions'); const { getSchema } = require('../../lib/validation-schemas'); const { deepFreezeStix } = require('../../lib/import-safety'); @@ -374,9 +375,12 @@ class BaseService extends ServiceWithHooks { // Strip workspace.release_tracks — server-controlled; maintained by // release-track backref reconciliation, and pinned to specific revisions, // so a copy from a prior GET must not ride along onto a new version. + // Strip workspace.relationship_endpoints — relationship services resolve + // these exact endpoint pins from authoritative object revisions. if (data.workspace) { delete data.workspace.validation; delete data.workspace.release_tracks; + delete data.workspace.relationship_endpoints; } if (!options.preserveAttackId) { @@ -767,6 +771,48 @@ class BaseService extends ServiceWithHooks { } } + /** + * Protect every exact revision captured by an active or in-progress graph + * manifest. Relationship payloads are frozen in the manifest, so a + * non-topology PUT remains safe; relationship endpoint changes are rejected + * separately by RelationshipsService. Hard deletion is always rejected. + */ + static async assertNotGraphPinned(document, operation) { + const graphManifestService = require('../release-tracks/graph-manifest-service'); + const pins = await graphManifestService.findPinsForRevision( + document.stix.id, + document.stix.modified, + ); + if ( + pins.length === 0 || + (operation === 'updated' && pins.every((pin) => pin.kind === 'relationship')) + ) { + return; + } + + throw new SnapshotGraphPinnedRevisionError({ + details: + `Revision ${document.stix.id} (modified ` + + `${new Date(document.stix.modified).toISOString()}) is frozen in ` + + `${pins.length} release-track snapshot graph manifest(s) and cannot be ${operation} ` + + 'in place.', + snapshot_graph_pins: pins, + }); + } + + static async assertNoGraphPinnedVersions(stixId, operation) { + const graphManifestService = require('../release-tracks/graph-manifest-service'); + const pins = await graphManifestService.findPinsForObject(stixId); + if (pins.length === 0) return; + + throw new SnapshotGraphPinnedRevisionError({ + details: + `Object ${stixId} has revision(s) frozen in ${pins.length} release-track snapshot ` + + `graph manifest(s) and cannot be ${operation}.`, + snapshot_graph_pins: pins, + }); + } + /** * Refresh workspace.release_tracks on a response object after domain * events have run. The created/updated event is awaited, and its listeners @@ -876,6 +922,7 @@ class BaseService extends ServiceWithHooks { if (data.workspace) { delete data.workspace.validation; delete data.workspace.release_tracks; + delete data.workspace.relationship_endpoints; } // Extract ATT&CK ID from external_references and propagate to workspace.attack_id @@ -985,6 +1032,7 @@ class BaseService extends ServiceWithHooks { // Members-pinned revisions are released content — immutable in place. await BaseService.assertNotMemberPinned(document, 'updated'); + await BaseService.assertNotGraphPinned(document, 'updated'); // TODO: diff analysis — detect field-level changes vs document // TODO: if no changes detected, short-circuit (no-op) @@ -1108,6 +1156,7 @@ class BaseService extends ServiceWithHooks { return null; } await BaseService.assertNotMemberPinned(existing, 'deleted'); + await BaseService.assertNotGraphPinned(existing, 'deleted'); const document = await this.repository.findOneAndDelete(stixId, stixModified); @@ -1415,6 +1464,7 @@ class BaseService extends ServiceWithHooks { // Deleting all versions must not destroy a members-pinned revision const memberPinned = await this.repository.retrieveMemberPinnedVersionsLean(stixId); await BaseService.assertNoMemberPinnedVersions(stixId, memberPinned, 'deleted'); + await BaseService.assertNoGraphPinnedVersions(stixId, 'deleted'); const result = await this.repository.deleteMany(stixId); if (result.deletedCount > 0) { diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index fb469a4b..06911881 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -18,24 +18,16 @@ // ============================================================================= const config = require('../../config/config'); -const types = require('../../lib/types'); const logger = require('../../lib/logger'); const linkById = require('../../lib/linkById'); -const EventBus = require('../../lib/event-bus'); -const Events = require('../../lib/event-constants'); -const { selectRelationshipsForBundle } = require('../../lib/stix-bundle-relationships'); -const revisionReference = require('../../lib/release-tracks/revision-reference'); const primaryRevisionService = require('./primary-revision-service'); +const graphManifestService = require('./graph-manifest-service'); const { bundleTransformSchema, workbenchTransformSchema, filesystemStoreTransformSchema, } = require('../../lib/release-tracks/export-schemas'); -function getRepositoryMap() { - return primaryRevisionService.getRepositoryMap(); -} - // ============================================================================= // Hydration // ============================================================================= @@ -57,145 +49,34 @@ exports.hydrateMembers = async function hydrateMembers(entries) { // Bundle assembly helpers // ============================================================================= -/** - * Select the tier entries that belong in a bundle export. - * - * Members are always included. Staged and candidate entries are included only - * when named in `include`. When `state` is provided it further narrows the - * staged/candidate entries to those whose workflow status matches — except - * entries marked 'reviewed', which are always included irrespective of - * `state` (reviewed content is release-ready by definition, mirroring how all - * members are inherently reviewed). - * - * @param {Object} snapshot - The raw snapshot document - * @param {Object} options - { include?: Array<'staged'|'candidates'>, state?: Array } - * @returns {Array<{object_ref: string, object_modified: string|Date}>} Deduplicated entries - */ -function collectBundleEntries(snapshot, options) { - const include = options.include || []; - const state = options.state; - - const filterByState = (entries) => { - if (!state) return entries; - return entries.filter( - (entry) => entry.object_status === 'reviewed' || state.includes(entry.object_status), - ); - }; - - const entries = [...(snapshot.members || [])]; - if (include.includes('staged')) { - entries.push(...filterByState(snapshot.staged || [])); - } - if (include.includes('candidates')) { - entries.push(...filterByState(snapshot.candidates || [])); - } - - // Deduplicate by object_ref + object_modified - const seen = new Set(); - const deduped = []; - for (const entry of entries) { - const key = `${entry.object_ref}::` + revisionReference.modifiedKey(entry.object_modified); - if (seen.has(key)) continue; - seen.add(key); - deduped.push(entry); - } - - return deduped; -} - -/** - * Fetch identities and marking definitions referenced by the hydrated - * documents (via created_by_ref / object_marking_refs) that are not already - * part of the export. Emitted bundles must be self-contained, so referenced - * supporting objects are appended even though they are not tier entries. - * - * @param {Array} documents - Hydrated lean documents ({ stix, ... }) - * @returns {Promise>} Supporting lean documents - */ -async function fetchSupportingObjects(documents) { - const repoMap = getRepositoryMap(); - const existingIds = new Set(documents.map((doc) => doc.stix.id)); - - const identityIds = new Set(); - const markingIds = new Set(); - for (const doc of documents) { - if (doc.stix.created_by_ref && !existingIds.has(doc.stix.created_by_ref)) { - identityIds.add(doc.stix.created_by_ref); - } - for (const ref of doc.stix.object_marking_refs || []) { - if (!existingIds.has(ref)) markingIds.add(ref); - } - } - - const supportingObjects = []; - const fetchLatest = async (repo, stixId, description) => { - try { - const doc = await repo.retrieveLatestByStixIdLean(stixId); - if (doc) supportingObjects.push(doc); - else logger.warn(`ExportService: Referenced ${description} not found: ${stixId}`); - } catch (err) { - logger.warn(`ExportService: Could not fetch ${description} "${stixId}": ${err.message}`); - } - }; - - await Promise.all([ - ...[...identityIds].map((id) => fetchLatest(repoMap[types.Identity], id, 'identity')), - ...[...markingIds].map((id) => - fetchLatest(repoMap[types.MarkingDefinition], id, 'marking definition'), - ), - ]); - - return supportingObjects; -} - -/** - * Fetch the latest publishable relationships connecting selected bundle - * objects. Relationship revisions remain indirect export-time content rather - * than snapshot members. - * - * @param {Array} documents - Hydrated selected object documents - * @returns {Promise>} - */ -async function fetchRelationships(documents) { - const selectedIds = new Set(documents.map((document) => document.stix.id)); - if (selectedIds.size === 0) return []; - - const results = await EventBus.emit(Events.BUNDLE_RELATIONSHIPS_REQUESTED, { - objectRefs: [...selectedIds], - }); - const relationships = results?.[0]; - if (!relationships) { - throw new Error('Unable to retrieve relationships for release-track bundle export'); - } - - return selectRelationshipsForBundle(relationships, selectedIds).filter( - (relationship) => !selectedIds.has(relationship.stix.id), - ); -} - /** * Convert LinkById tags (e.g. "(LinkById: T1234)") in descriptions to - * markdown citations, preferring objects already in the export before - * falling back to a database lookup. Mirrors the legacy stix-bundles-service - * behavior so bundles emitted from release tracks match published output. + * markdown citations using only object revisions frozen in the manifest. * * @param {Array} documents - Hydrated lean documents ({ stix, ... }) */ -async function convertLinkByIdTags(documents) { +async function convertLinkByIdTags(documents, linkTargetDocuments) { const byAttackId = new Map(); - for (const doc of documents) { + for (const doc of [...documents, ...linkTargetDocuments]) { const attackId = linkById.getAttackId(doc.stix); if (attackId) byAttackId.set(attackId, doc); } - const getAttackObject = async (attackId) => - byAttackId.get(attackId) || (await linkById.getAttackObjectFromDatabase(attackId)); + const getAttackObject = async (attackId) => byAttackId.get(attackId); for (const doc of documents) { await linkById.convertLinkByIdTags(doc.stix, getAttackObject); } } +function selectedDraftTierIsDynamic(snapshot, options) { + return (options.include || []).some( + (tier) => + ['staged', 'candidates'].includes(tier) && + (snapshot[tier] || []).some((entry) => entry.object_modified === 'latest'), + ); +} + // ============================================================================= // Format helpers (delegating to Zod transform schemas) // ============================================================================= @@ -266,12 +147,12 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd */ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options = {}) { if (format === 'bundle') { - const entries = collectBundleEntries(snapshot, options); - const hydratedObjects = await exports.hydrateMembers(entries); - const relationships = await fetchRelationships(hydratedObjects); - const supportingObjects = await fetchSupportingObjects([...hydratedObjects, ...relationships]); - const allObjects = [...hydratedObjects, ...relationships, ...supportingObjects]; - await convertLinkByIdTags(allObjects); + const graph = + options.captureGraph || selectedDraftTierIsDynamic(snapshot, options) + ? await graphManifestService.replayPlannedSnapshot(snapshot, options) + : await graphManifestService.replay(snapshot, options); + const allObjects = graph.documents; + await convertLinkByIdTags(allObjects, graph.linkTargetDocuments); return exports.formatAsBundle(snapshot, allObjects, { stixVersion: options.stixVersion, diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js new file mode 100644 index 00000000..1ed69ade --- /dev/null +++ b/app/services/release-tracks/graph-manifest-service.js @@ -0,0 +1,542 @@ +'use strict'; + +const { v4: uuidv4 } = require('uuid'); +const linkById = require('../../lib/linkById'); +const bundleRelationships = require('../../lib/stix-bundle-relationships'); +const attackObjectsRepository = require('../../repository/attack-objects-repository'); +const relationshipsRepository = require('../../repository/relationships-repository'); +const detectionStrategiesRepository = require('../../repository/detection-strategies-repository'); +const BundleGraphResolver = require('../stix/bundle-graph-resolver'); +const { + ReleaseTrackGraphManifest, + ReleaseTrackGraphManifestEntry, +} = require('../../models/release-tracks/release-track-graph-manifest-model'); +const { ReleaseContentIntegrityError } = require('../../exceptions'); +const primaryRevisionService = require('./primary-revision-service'); + +const RESOLVER_VERSION = 'bounded-attack-graph-v1'; +const TIERS = ['members', 'staged', 'candidates', 'quarantine']; +const MUTATION_PROTECTED_ENTRY_FILTER = { + $or: [ + { kind: { $ne: 'root' } }, + { kind: 'root', tier: { $in: ['members', 'quarantine'] } }, + { kind: 'root', 'discovered_from.0': { $exists: true } }, + ], +}; + +function revisionKey(objectRef, objectModified) { + return `${objectRef}::${new Date(objectModified).getTime()}`; +} + +function endpointFor(relationship, side) { + const endpoint = relationship.workspace?.relationship_endpoints?.[side]; + const objectRef = relationship.stix[`${side}_ref`]; + if (!endpoint || endpoint.object_ref !== objectRef || !endpoint.object_modified) { + return null; + } + return { + object_ref: endpoint.object_ref, + object_modified: endpoint.object_modified, + }; +} + +async function buildManifestEntries(snapshot) { + const rootRequests = []; + for (const tier of TIERS) { + for (const entry of snapshot[tier] || []) { + rootRequests.push({ ...entry, tier }); + } + } + + const hydratedRoots = await primaryRevisionService.assertStoredEntries(rootRequests); + const rootMetadata = new Map( + hydratedRoots.entries.map((entry) => [ + revisionKey(entry.object_ref, entry.object_modified), + entry, + ]), + ); + + const rootObjectRefs = new Set(hydratedRoots.entries.map((entry) => entry.object_ref)); + + const relationships = await relationshipsRepository.retrieveAllForBundle({ + includeRevoked: false, + includeDeprecated: false, + }); + const missing = []; + const pinnedRelationships = []; + for (const relationship of relationships) { + const source = endpointFor(relationship, 'source'); + const target = endpointFor(relationship, 'target'); + if (!source || !target) { + // Legacy relationships outside this snapshot's bounded graph cannot + // affect its replay. Fail closed only when an unpinned relationship + // touches a primary member by STIX ID. + if ( + rootObjectRefs.has(relationship.stix.source_ref) || + rootObjectRefs.has(relationship.stix.target_ref) + ) { + missing.push({ + object_ref: relationship.stix.id, + object_modified: new Date(relationship.stix.modified).toISOString(), + dependency: 'relationship_endpoints', + }); + } + continue; + } + pinnedRelationships.push({ relationship, source, target }); + } + if (missing.length > 0) { + throw new ReleaseContentIntegrityError(missing, { + details: 'Snapshot graph capture found relationships without exact endpoint pins.', + }); + } + + const graphResolver = new BundleGraphResolver({ + attackObjectsRepository, + detectionStrategiesRepository, + repositoryMap: primaryRevisionService.getRepositoryMap(), + policy: { + isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, + relationshipIsActive: bundleRelationships.relationshipIsActive, + secondaryObjectIsValid: (document) => Boolean(document), + }, + options: { + inferDomains: false, + includeRevoked: true, + includeDeprecated: true, + includeMissingAttackId: true, + }, + relationships: pinnedRelationships.map((candidate) => candidate.relationship), + onMissingDependency(reference) { + missing.push({ + ...reference, + object_modified: new Date(reference.object_modified).toISOString(), + }); + }, + }); + const resolvedGraph = await graphResolver.resolve(hydratedRoots.documents); + if (missing.length > 0) { + const uniqueMissing = [ + ...new Map( + missing.map((reference) => [ + `${reference.object_ref}::${reference.object_modified}`, + reference, + ]), + ).values(), + ]; + throw new ReleaseContentIntegrityError(uniqueMissing, { + details: 'Snapshot graph capture could not hydrate every exact dependency.', + }); + } + const selectedDocuments = new Map( + resolvedGraph.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + const selectedRelationships = resolvedGraph.relationships.map((relationship) => ({ + relationship, + source: endpointFor(relationship, 'source'), + target: endpointFor(relationship, 'target'), + })); + const relationshipDocuments = selectedRelationships.map((candidate) => candidate.relationship); + const discoverySources = resolvedGraph.dependencies; + const supportingDocuments = await graphResolver.loadSupportingDocuments(resolvedGraph.objects); + + const linkTargets = new Map(); + for (const document of [...selectedDocuments.values(), ...relationshipDocuments]) { + for (const attackId of linkById.extractLinkByIds(document.stix)) { + if (!linkTargets.has(attackId)) { + const target = await linkById.getAttackObjectFromDatabase(attackId); + if (target) { + linkTargets.set(attackId, target); + } + } + } + } + + const entries = []; + for (const [key, document] of selectedDocuments) { + const root = rootMetadata.get(key); + entries.push({ + revision_key: key, + kind: root ? 'root' : 'secondary', + tier: root?.tier, + object_status: root?.object_status, + object_ref: document.stix.id, + object_modified: document.stix.modified, + discovered_from: discoverySources.get(key) || [], + }); + } + for (const candidate of selectedRelationships) { + entries.push({ + revision_key: revisionKey( + candidate.relationship.stix.id, + candidate.relationship.stix.modified, + ), + kind: 'relationship', + object_ref: candidate.relationship.stix.id, + object_modified: candidate.relationship.stix.modified, + source: candidate.source, + target: candidate.target, + frozen_stix: candidate.relationship.stix, + }); + } + for (const document of supportingDocuments) { + const isVersioned = Boolean(document.stix.modified); + entries.push({ + revision_key: isVersioned + ? revisionKey(document.stix.id, document.stix.modified) + : `${document.stix.id}::unversioned`, + kind: 'supporting', + object_ref: document.stix.id, + object_modified: document.stix.modified, + frozen_stix: isVersioned ? undefined : document.stix, + }); + } + for (const document of linkTargets.values()) { + entries.push({ + revision_key: revisionKey(document.stix.id, document.stix.modified), + kind: 'link_target', + object_ref: document.stix.id, + object_modified: document.stix.modified, + }); + } + + return entries; +} + +async function prepare(snapshot, options = {}) { + const manifestId = `release-track-graph-manifest--${uuidv4()}`; + const entries = await buildManifestEntries(snapshot); + const common = { + manifest_id: manifestId, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + }; + + await ReleaseTrackGraphManifest.create({ + ...common, + state: 'pending', + resolver_version: RESOLVER_VERSION, + baseline_reconstruction: options.baselineReconstruction === true, + }); + try { + if (entries.length > 0) { + await ReleaseTrackGraphManifestEntry.insertMany( + entries.map((entry) => ({ ...common, ...entry })), + ); + } + } catch (err) { + await discard(manifestId); + throw err; + } + return manifestId; +} + +async function activate(manifestId) { + await ReleaseTrackGraphManifest.updateOne( + { manifest_id: manifestId, state: 'pending' }, + { $set: { state: 'active' } }, + ).exec(); +} + +async function discard(manifestId) { + await Promise.all([ + ReleaseTrackGraphManifestEntry.deleteMany({ manifest_id: manifestId }).exec(), + ReleaseTrackGraphManifest.deleteOne({ manifest_id: manifestId }).exec(), + ]); +} + +async function discardSnapshot(trackId, snapshotModified) { + const manifests = await ReleaseTrackGraphManifest.find({ + track_id: trackId, + snapshot_modified: snapshotModified, + }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec(); + const manifestIds = manifests.map((manifest) => manifest.manifest_id); + if (manifestIds.length === 0) return; + + await Promise.all([ + ReleaseTrackGraphManifestEntry.deleteMany({ + manifest_id: { $in: manifestIds }, + }).exec(), + ReleaseTrackGraphManifest.deleteMany({ + manifest_id: { $in: manifestIds }, + }).exec(), + ]); +} + +async function discardTrack(trackId) { + const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec(); + const manifestIds = manifests.map((manifest) => manifest.manifest_id); + + await Promise.all([ + manifestIds.length > 0 + ? ReleaseTrackGraphManifestEntry.deleteMany({ + manifest_id: { $in: manifestIds }, + }).exec() + : Promise.resolve(), + ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }).exec(), + ]); +} + +function rootIsSelected(entry, options) { + if (entry.tier === 'members') return true; + if (!['staged', 'candidates'].includes(entry.tier)) return false; + if (!(options.include || []).includes(entry.tier)) return false; + if (!options.state) return true; + return entry.object_status === 'reviewed' || options.state.includes(entry.object_status); +} + +async function replayEntries(entries, manifest, options) { + const versionedEntries = entries.filter( + (entry) => entry.object_modified && entry.kind !== 'relationship', + ); + const hydrated = await primaryRevisionService.assertStoredEntries( + versionedEntries.map((entry) => ({ + object_ref: entry.object_ref, + object_modified: entry.object_modified, + })), + ); + const documentsByRevision = new Map( + hydrated.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + for (const entry of entries) { + if (entry.kind === 'relationship' && entry.frozen_stix) { + documentsByRevision.set(entry.revision_key, { + stix: entry.frozen_stix, + }); + } + } + + const selectedRevisionKeys = new Set( + entries + .filter((entry) => entry.kind === 'root' && rootIsSelected(entry, options)) + .map((entry) => entry.revision_key), + ); + + // Special embedded-reference dependencies can be chained (for example, a + // detection strategy discovered through an analytic that was itself a + // relationship secondary). Replay only follows edges frozen in the + // manifest; it never asks the live database to expand the graph. + let added; + do { + added = false; + for (const entry of entries) { + if ( + !['root', 'secondary'].includes(entry.kind) || + selectedRevisionKeys.has(entry.revision_key) + ) { + continue; + } + if ( + (entry.discovered_from || []).some((source) => + selectedRevisionKeys.has(revisionKey(source.object_ref, source.object_modified)), + ) + ) { + selectedRevisionKeys.add(entry.revision_key); + added = true; + } + } + } while (added); + + const selectedRelationships = entries.filter( + (entry) => + entry.kind === 'relationship' && + selectedRevisionKeys.has( + revisionKey(entry.source.object_ref, entry.source.object_modified), + ) && + selectedRevisionKeys.has(revisionKey(entry.target.object_ref, entry.target.object_modified)), + ); + for (const entry of selectedRelationships) { + selectedRevisionKeys.add(entry.revision_key); + } + + const selectedDocuments = [...selectedRevisionKeys] + .map((key) => documentsByRevision.get(key)) + .filter(Boolean); + const supportingRefs = new Set(); + for (const document of selectedDocuments) { + if (document.stix.created_by_ref) { + supportingRefs.add(document.stix.created_by_ref); + } + for (const objectRef of document.stix.object_marking_refs || []) { + supportingRefs.add(objectRef); + } + } + + const supportingDocuments = entries + .filter((entry) => entry.kind === 'supporting' && supportingRefs.has(entry.object_ref)) + .map((entry) => + entry.object_modified + ? documentsByRevision.get(entry.revision_key) + : { stix: entry.frozen_stix }, + ) + .filter(Boolean); + const linkTargetDocuments = entries + .filter((entry) => entry.kind === 'link_target') + .map((entry) => documentsByRevision.get(entry.revision_key)) + .filter(Boolean); + + const emittedByRevision = new Map(); + for (const document of [...selectedDocuments, ...supportingDocuments]) { + const key = document.stix.modified + ? revisionKey(document.stix.id, document.stix.modified) + : `${document.stix.id}::unversioned`; + emittedByRevision.set(key, document); + } + + return { + documents: [...emittedByRevision.values()], + linkTargetDocuments, + manifest, + }; +} + +async function replay(snapshot, options = {}) { + if (!snapshot.graph_manifest_id) { + throw new ReleaseContentIntegrityError( + [ + { + track_id: snapshot.id, + snapshot_modified: new Date(snapshot.modified).toISOString(), + dependency: 'graph_manifest', + }, + ], + { details: 'Snapshot does not reference a deterministic graph manifest.' }, + ); + } + + const manifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: snapshot.graph_manifest_id, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + state: { $in: ['pending', 'active'] }, + }) + .lean() + .exec(); + if (!manifest) { + throw new ReleaseContentIntegrityError( + [{ manifest_id: snapshot.graph_manifest_id, dependency: 'graph_manifest' }], + { details: 'Snapshot graph manifest is missing.' }, + ); + } + + // A snapshot link is the durable commit record. If the process stopped + // after linking a complete pending manifest but before activation, replay + // remains deterministic and repairs the visibility marker opportunistically. + if (manifest.state === 'pending') { + await activate(manifest.manifest_id); + manifest.state = 'active'; + } + + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: manifest.manifest_id, + }) + .lean() + .exec(); + return replayEntries(entries, manifest, options); +} + +async function replayPlannedSnapshot(snapshot, options = {}) { + const entries = await buildManifestEntries(snapshot); + return replayEntries( + entries, + { + manifest_id: null, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + state: 'preview', + resolver_version: RESOLVER_VERSION, + }, + options, + ); +} + +async function findPinsForRevision(objectRef, objectModified) { + const entries = await ReleaseTrackGraphManifestEntry.find({ + object_ref: objectRef, + object_modified: objectModified, + ...MUTATION_PROTECTED_ENTRY_FILTER, + }) + .select({ + manifest_id: 1, + track_id: 1, + snapshot_modified: 1, + kind: 1, + tier: 1, + _id: 0, + }) + .lean() + .exec(); + if (entries.length === 0) return []; + + const protectedManifestIds = new Set( + ( + await ReleaseTrackGraphManifest.find({ + manifest_id: { $in: entries.map((entry) => entry.manifest_id) }, + state: { $in: ['pending', 'active'] }, + }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec() + ).map((manifest) => manifest.manifest_id), + ); + return entries.filter((entry) => protectedManifestIds.has(entry.manifest_id)); +} + +async function findPinsForObject(objectRef) { + const entries = await ReleaseTrackGraphManifestEntry.find({ + object_ref: objectRef, + object_modified: { $ne: null }, + ...MUTATION_PROTECTED_ENTRY_FILTER, + }) + .select({ + manifest_id: 1, + track_id: 1, + snapshot_modified: 1, + object_modified: 1, + kind: 1, + tier: 1, + _id: 0, + }) + .lean() + .exec(); + if (entries.length === 0) return []; + + const protectedManifestIds = new Set( + ( + await ReleaseTrackGraphManifest.find({ + manifest_id: { $in: entries.map((entry) => entry.manifest_id) }, + state: { $in: ['pending', 'active'] }, + }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec() + ).map((manifest) => manifest.manifest_id), + ); + return entries.filter((entry) => protectedManifestIds.has(entry.manifest_id)); +} + +module.exports = { + prepare, + activate, + discard, + discardSnapshot, + discardTrack, + replay, + replayPlannedSnapshot, + findPinsForRevision, + findPinsForObject, + buildManifestEntries, + RESOLVER_VERSION, +}; diff --git a/app/services/release-tracks/primary-revision-service.js b/app/services/release-tracks/primary-revision-service.js index 9b12b7ea..e8863355 100644 --- a/app/services/release-tracks/primary-revision-service.js +++ b/app/services/release-tracks/primary-revision-service.js @@ -19,6 +19,7 @@ function getRepositoryMap() { [types.Tactic]: require('../../repository/tactics-repository'), [types.Group]: require('../../repository/groups-repository'), [types.Campaign]: require('../../repository/campaigns-repository'), + [types.Collection]: require('../../repository/collections-repository'), [types.Mitigation]: require('../../repository/mitigations-repository'), [types.Matrix]: require('../../repository/matrix-repository'), [types.Relationship]: require('../../repository/relationships-repository'), diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 7a2ebab0..234f52ca 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -6,7 +6,7 @@ // Orchestrator that delegates to domain-specific sub-services. This is the // single entry point consumed by the controller layer. // -// Phase 1: Track management, snapshot CRUD, config → snapshot-service +// Phase 1: Track management, snapshot lifecycle, config → snapshot-service // Phase 2: Candidates, staged, object versions → standard-track-service // Phase 3: Auto-promotion, workflow → workflow-service // Phase 4: Release planning and versioning → versioning-service @@ -289,49 +289,6 @@ exports.updateMetadata = function updateMetadata(trackId, updates, userId) { return snapshotService.updateMetadata(trackId, updates, userId); }; -exports.updateMetadataByModified = function updateMetadataByModified( - trackId, - modified, - updates, - userId, -) { - return snapshotService.updateMetadataByModified(trackId, modified, updates, userId); -}; - -exports.updateContents = function updateContents(trackId, contents, actor, confirmation) { - return destructiveAuditService.execute( - { - action: 'replace_members_latest', - trackId, - ...destructiveIdentity(trackId, actor, confirmation), - request: { members_count: contents.x_mitre_contents.length }, - }, - () => snapshotService.updateContents(trackId, contents, actor?.user_account_id), - ); -}; - -exports.updateContentsByModified = function updateContentsByModified( - trackId, - modified, - contents, - actor, - confirmation, -) { - return destructiveAuditService.execute( - { - action: 'replace_members_historical', - trackId, - ...destructiveIdentity(trackId, actor, confirmation), - request: { - source_snapshot_modified: modified, - members_count: contents.x_mitre_contents.length, - }, - }, - () => - snapshotService.updateContentsByModified(trackId, modified, contents, actor?.user_account_id), - ); -}; - exports.cloneTrack = function cloneTrack(trackId, options) { return snapshotService.cloneTrack(trackId, options); }; @@ -425,7 +382,10 @@ async function renderReleasePlan(plan, options) { if (format === 'summary') return plan.summary; if (plan.blockingError) throw plan.blockingError; if (format === 'bundle') { - return exportService.exportSnapshot(plan.plannedSnapshot, format, options); + return exportService.exportSnapshot(plan.plannedSnapshot, format, { + ...options, + captureGraph: true, + }); } return formatWorkbenchSnapshot(plan.plannedSnapshot, options); } diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 45ee635a..ddf8e334 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -4,7 +4,7 @@ // Snapshot Service // // Core snapshot lifecycle operations: track creation, retrieval, cloning, -// metadata/contents updates, configuration, and deletion. +// metadata updates, configuration, and deletion. // // This is the foundational sub-service consumed by the facade and by other // sub-services (standard-track, versioning, virtual-track) that need to @@ -21,11 +21,12 @@ const versionUtils = require('../../lib/release-tracks/version-utils'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const primaryRevisionService = require('./primary-revision-service'); const reconciliationService = require('./reconciliation-service'); +const graphManifestService = require('./graph-manifest-service'); const { TrackNotFoundError, NotFoundError, TaggedSnapshotDeletionError, - BadRequestError, + HistoricalSnapshotDeletionError, } = require('../../exceptions'); // ============================================================================= @@ -53,35 +54,6 @@ function normalizeTierSummary(summary) { }; } -function assertStandardTrack(snapshot) { - if (snapshot.type !== 'standard') { - throw new BadRequestError({ - message: 'Direct contents updates are only available for standard release tracks', - details: - 'Virtual members are computed from component tracks; create a virtual snapshot to update them', - }); - } -} - -/** - * Convert contents request entries into exact revision pins. - * - * `latest` is request-time shorthand only. It must never be persisted because - * snapshot membership is defined by an immutable `(object_ref, - * object_modified)` pair. - * - * @param {Array<{obj_ref: string, obj_modified: string}>} contents - * @returns {Promise>} - */ -async function resolveContentsMembers(contents) { - const requested = contents.map((entry) => ({ - object_ref: entry.obj_ref, - object_modified: - entry.obj_modified === 'latest' ? entry.obj_modified : new Date(entry.obj_modified), - })); - return (await primaryRevisionService.assertRequestEntries(requested)).entries; -} - /** * Recompute and persist denormalized registry counters from actual snapshot data. * @@ -132,6 +104,38 @@ async function emitContentsChanged(trackId, snapshot) { } exports.emitContentsChanged = emitContentsChanged; +/** + * Persist a snapshot and its graph manifest as one logical operation. + * + * Mongo transactions are not available across the dynamically named snapshot + * collections in every supported deployment. A pending manifest plus + * compensation keeps partially completed writes invisible to replay and + * protection queries. + */ +async function saveSnapshotWithManifest(trackId, snapshotData) { + const manifestId = await graphManifestService.prepare(snapshotData); + snapshotData.graph_manifest_id = manifestId; + + try { + const saved = await dynamicRepo.saveSnapshot(trackId, snapshotData); + try { + await graphManifestService.activate(manifestId); + } catch (err) { + // The saved snapshot is already linked to a complete pending manifest. + // Replay can safely activate it later, so do not turn a committed + // snapshot into an ambiguous client-visible failure. + logger.warn( + `SnapshotService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, + ); + } + return saved; + } catch (err) { + await graphManifestService.discard(manifestId); + throw err; + } +} +exports.saveSnapshotWithManifest = saveSnapshotWithManifest; + // ============================================================================= // Track management // ============================================================================= @@ -192,7 +196,7 @@ exports.createTrack = async function createTrack(data) { // Create collection + indexes, then persist the initial snapshot await modelFactory.ensureIndexes(trackId); - const snapshot = await dynamicRepo.saveSnapshot(trackId, initialSnapshot); + const snapshot = await saveSnapshotWithManifest(trackId, initialSnapshot); // Register in the central registry await registryRepo.create({ @@ -317,6 +321,7 @@ exports.getSnapshotByModified = async function getSnapshotByModified(trackId, mo */ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, overrides) { const clone = deepClone(sourceSnapshot); + delete clone.graph_manifest_id; clone.modified = new Date(); clone.version = null; // clones are always drafts delete clone.scheduled_materialization; @@ -331,7 +336,7 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov } const normalized = tierRevisionInvariant.normalizeSnapshot(clone); - const saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); + const saved = await saveSnapshotWithManifest(trackId, normalized.snapshot); await syncRegistryCounters(trackId); // The clone (modified = now) is the track's new latest snapshot @@ -384,6 +389,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { const now = new Date(); const clone = deepClone(sourceSnapshot); + delete clone.graph_manifest_id; clone.id = newTrackId; clone.modified = now; clone.version = null; @@ -399,7 +405,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { ); await modelFactory.ensureIndexes(newTrackId); - const saved = await dynamicRepo.saveSnapshot(newTrackId, normalized.snapshot); + const saved = await saveSnapshotWithManifest(newTrackId, normalized.snapshot); await registryRepo.create({ track_id: newTrackId, @@ -459,82 +465,6 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId return exports.cloneSnapshot(trackId, source, overrides); }; -/** - * Update metadata on a specific snapshot (creates a new snapshot clone). - * - * @param {string} trackId - * @param {string|Date} modified - * @param {Object} updates - { name?, description?, object_marking_refs? } - * @param {string} [_userId] - * @returns {Promise} The new snapshot - */ -exports.updateMetadataByModified = async function updateMetadataByModified( - trackId, - modified, - updates, - // eslint-disable-next-line no-unused-vars - _userId, -) { - const source = await exports.getSnapshotByModified(trackId, modified); - const overrides = {}; - if (updates.name !== undefined) overrides.name = updates.name; - if (updates.description !== undefined) overrides.description = updates.description; - if (updates.object_marking_refs !== undefined) - overrides.object_marking_refs = updates.object_marking_refs; - - const registryUpdates = {}; - if (updates.name !== undefined) registryUpdates.name = updates.name; - if (updates.description !== undefined) registryUpdates.description = updates.description; - if (Object.keys(registryUpdates).length > 0) { - registryUpdates.updated_at = new Date(); - await registryRepo.updateByTrackId(trackId, registryUpdates); - } - - return exports.cloneSnapshot(trackId, source, overrides); -}; - -// ============================================================================= -// Contents updates -// ============================================================================= - -/** - * Replace member contents on the latest snapshot (creates a new snapshot clone). - * - * @param {string} trackId - * @param {Object} contents - { x_mitre_contents: [{ obj_ref, obj_modified }] } - * @param {string} [_userId] - * @returns {Promise} The new snapshot - */ -// eslint-disable-next-line no-unused-vars -exports.updateContents = async function updateContents(trackId, contents, _userId) { - const source = await exports.getLatestSnapshot(trackId); - assertStandardTrack(source); - const members = await resolveContentsMembers(contents.x_mitre_contents); - return exports.cloneSnapshot(trackId, source, { members }); -}; - -/** - * Replace member contents on a specific snapshot (creates a new snapshot clone). - * - * @param {string} trackId - * @param {string|Date} modified - * @param {Object} contents - { x_mitre_contents: [{ obj_ref, obj_modified }] } - * @param {string} [_userId] - * @returns {Promise} The new snapshot - */ -exports.updateContentsByModified = async function updateContentsByModified( - trackId, - modified, - contents, - // eslint-disable-next-line no-unused-vars - _userId, -) { - const source = await exports.getSnapshotByModified(trackId, modified); - assertStandardTrack(source); - const members = await resolveContentsMembers(contents.x_mitre_contents); - return exports.cloneSnapshot(trackId, source, { members }); -}; - // ============================================================================= // Configuration // ============================================================================= @@ -608,10 +538,14 @@ exports.updateConfig = async function updateConfig(trackId, config, _userId) { exports.deleteTrack = async function deleteTrack(trackId) { const registry = await registryRepo.findByTrackId(trackId); if (!registry) { + // A previous delete may have removed the registry only after dropping the + // dynamic snapshot collection but stopped before manifest cleanup. + await graphManifestService.discardTrack(trackId); throw new TrackNotFoundError(trackId); } await dynamicRepo.dropCollection(trackId); + await graphManifestService.discardTrack(trackId); await registryRepo.deleteByTrackId(trackId); // Remove all backrefs to the deleted track @@ -630,6 +564,9 @@ exports.deleteTrack = async function deleteTrack(trackId) { exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { const snapshot = await dynamicRepo.getSnapshotByModified(trackId, modified); if (!snapshot) { + // Make a retry after an interrupted delete clean any orphaned manifests + // even though the snapshot document is already gone. + await graphManifestService.discardSnapshot(trackId, modified); throw new NotFoundError({ details: `Snapshot with modified '${modified}' not found for track '${trackId}'`, }); @@ -639,13 +576,19 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { throw new TaggedSnapshotDeletionError(snapshot.version); } + const latest = await dynamicRepo.getLatestSnapshot(trackId); + if (!latest || new Date(latest.modified).getTime() !== new Date(snapshot.modified).getTime()) { + throw new HistoricalSnapshotDeletionError(snapshot.modified, latest?.modified); + } + await dynamicRepo.deleteSnapshot(trackId, modified); + await graphManifestService.discardSnapshot(trackId, snapshot.modified); await syncRegistryCounters(trackId); // Deleting the latest snapshot reverts membership to the previous snapshot // (or clears it if no snapshots remain) - const latest = await dynamicRepo.getLatestSnapshot(trackId); - await emitContentsChanged(trackId, latest); + const revertedLatest = await dynamicRepo.getLatestSnapshot(trackId); + await emitContentsChanged(trackId, revertedLatest); logger.verbose(`SnapshotService: Deleted snapshot '${modified}' from track "${trackId}"`); }; diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 6f902d2b..702511fc 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -12,6 +12,7 @@ const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-in const revisionReference = require('../../lib/release-tracks/revision-reference'); const releaseHistoryService = require('./release-history-service'); const primaryRevisionService = require('./primary-revision-service'); +const graphManifestService = require('./graph-manifest-service'); const logger = require('../../lib/logger'); const { AlreadyReleasedError, @@ -280,17 +281,47 @@ async function planLoadedSnapshot(trackId, snapshot, options) { async function commitPlan(plan) { if (plan.blockingError) throw plan.blockingError; - const tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { - version: plan.version, - versionHistoryEntry: plan.versionHistoryEntry, - additionalOps: Object.keys(plan.additionalOps).length > 0 ? plan.additionalOps : undefined, - }); + const manifestId = await graphManifestService.prepare(plan.plannedSnapshot); + + let tagged; + try { + tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { + version: plan.version, + versionHistoryEntry: plan.versionHistoryEntry, + additionalOps: { + ...plan.additionalOps, + graph_manifest_id: manifestId, + }, + }); + } catch (err) { + await graphManifestService.discard(manifestId); + throw err; + } if (!tagged) { + await graphManifestService.discard(manifestId); await releaseHistoryService.reconcileTaggedReleases(plan.trackId); throw new AlreadyReleasedError('(concurrent release)'); } + // Link the complete pending manifest before activation. The snapshot link + // is the durable commit record, and replay can recover a linked pending + // manifest if the process stops in this narrow window. + try { + await graphManifestService.activate(manifestId); + } catch (err) { + logger.warn( + `VersioningService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, + ); + } + + if ( + plan.sourceSnapshot.graph_manifest_id && + plan.sourceSnapshot.graph_manifest_id !== manifestId + ) { + await graphManifestService.discard(plan.sourceSnapshot.graph_manifest_id); + } + await releaseHistoryService.reconcileTaggedReleases(plan.trackId); const latest = await dynamicRepo.getLatestSnapshot(plan.trackId); await snapshotService.emitContentsChanged(plan.trackId, latest); diff --git a/app/services/stix/bundle-graph-resolver.js b/app/services/stix/bundle-graph-resolver.js new file mode 100644 index 00000000..9374bafb --- /dev/null +++ b/app/services/stix/bundle-graph-resolver.js @@ -0,0 +1,469 @@ +'use strict'; + +const _ = require('lodash'); +const linkById = require('../../lib/linkById'); +const logger = require('../../lib/logger'); + +/** + * Resolves the bounded ATT&CK object graph used by bundle exports. + * + * A resolver instance belongs to exactly one export request. Keeping its + * caches, relationship set, and inferred-domain state request-local prevents + * overlapping exports from observing or overwriting each other's state. + * + * This resolver deliberately preserves the legacy one-hop relationship + * expansion and named ATT&CK special cases. It is not a general transitive + * graph walker. + */ +class BundleGraphResolver { + /** + * @param {Object} dependencies + * @param {Object} dependencies.attackObjectsRepository + * @param {Object} dependencies.detectionStrategiesRepository + * @param {Object} dependencies.policy + * @param {Function} dependencies.policy.isDeprecatedPattern + * @param {Function} dependencies.policy.relationshipIsActive + * @param {Function} dependencies.policy.secondaryObjectIsValid + * @param {Object} dependencies.options + * @param {Array} dependencies.relationships + */ + constructor({ + attackObjectsRepository, + detectionStrategiesRepository, + repositoryMap, + policy, + options, + relationships, + onMissingDependency, + }) { + this.attackObjectsRepository = attackObjectsRepository; + this.detectionStrategiesRepository = detectionStrategiesRepository; + this.repositoryMap = repositoryMap; + this.exactEndpoints = Boolean(repositoryMap); + this.policy = policy; + this.options = options; + this.relationships = _.cloneDeep(relationships); + this.onMissingDependency = onMissingDependency; + + this.attackObjectCache = new Map(); + this.attackObjectByAttackIdCache = new Map(); + this.domainCache = new Map(); + this.dependencies = new Map(); + } + + revisionKey(objectRef, objectModified) { + return `${objectRef}::${new Date(objectModified).getTime()}`; + } + + documentKey(document) { + return this.exactEndpoints + ? this.revisionKey(document.stix.id, document.stix.modified) + : document.stix.id; + } + + endpointKey(relationship, side) { + const objectRef = relationship.stix[`${side}_ref`]; + if (!this.exactEndpoints) return objectRef; + const endpoint = relationship.workspace?.relationship_endpoints?.[side]; + if (endpoint?.object_ref !== objectRef || !endpoint.object_modified) { + return `${objectRef}::unpinned`; + } + return this.revisionKey(endpoint.object_ref, endpoint.object_modified); + } + + hasEndpoint(objectsMap, relationship, side) { + return objectsMap.has(this.endpointKey(relationship, side)); + } + + endpointDocument(objectsMap, relationship, side) { + return objectsMap.get(this.endpointKey(relationship, side)); + } + + rememberDependency(document, sourceDocument) { + if (!document || !sourceDocument) return; + const key = this.documentKey(document); + const sources = this.dependencies.get(key) || new Map(); + sources.set(this.documentKey(sourceDocument), { + object_ref: sourceDocument.stix.id, + object_modified: sourceDocument.stix.modified, + }); + this.dependencies.set(key, sources); + } + + /** + * Resolve the object and relationship graph for the supplied primary roots. + * + * @param {Array} primaryObjects Workbench-shaped primary documents + * @returns {Promise<{ + * objects: Array, + * documents: Array, + * relationships: Array, + * attackObjectByAttackIdCache: Map + * }>} + */ + async resolve(primaryObjects) { + const objects = []; + const objectsMap = new Map(); + + for (const primaryObject of _.cloneDeep(primaryObjects)) { + this.addAttackObject(primaryObject, objects, objectsMap); + } + + const primaryObjectRelationships = this.relationships.filter( + (relationship) => + this.hasEndpoint(objectsMap, relationship, 'source') || + this.hasEndpoint(objectsMap, relationship, 'target'), + ); + + await this.addSecondaryObjects(primaryObjectRelationships, objectsMap, objects); + await this.processSecondaryRelationships(objects, objectsMap); + + const selectedRelationships = []; + for (const relationship of this.relationships) { + if (this.relationshipCanBeEmitted(relationship, objectsMap)) { + objects.push(relationship.stix); + selectedRelationships.push(relationship); + } + } + + return { + objects, + documents: [...objectsMap.values()], + relationships: selectedRelationships, + attackObjectByAttackIdCache: this.attackObjectByAttackIdCache, + dependencies: new Map( + [...this.dependencies].map(([key, sources]) => [key, [...sources.values()]]), + ), + }; + } + + /** + * Load identities and marking definitions referenced by the resolved graph. + * + * @param {Array} stixObjects + * @returns {Promise>} STIX-shaped supporting objects + */ + async loadSupportingObjects(stixObjects) { + return (await this.loadSupportingDocuments(stixObjects)).map((document) => document.stix); + } + + /** + * Load Workbench-shaped supporting documents for manifest capture. + * + * @param {Array} stixObjects + * @returns {Promise>} + */ + async loadSupportingDocuments(stixObjects) { + const identityRefs = new Set(); + const markingRefs = new Set(); + + for (const stixObject of stixObjects) { + if (stixObject.created_by_ref) { + identityRefs.add(stixObject.created_by_ref); + } + for (const markingRef of stixObject.object_marking_refs || []) { + markingRefs.add(markingRef); + } + } + + const supportingDocuments = []; + for (const stixId of identityRefs) { + const identity = await this.getAttackObject(stixId); + if (identity) { + supportingDocuments.push(identity); + } else { + logger.warn(`Referenced identity not found: ${stixId}`); + } + } + + for (const stixId of markingRefs) { + const markingDefinition = await this.getAttackObject(stixId); + if (markingDefinition) { + supportingDocuments.push(markingDefinition); + } + } + + return supportingDocuments; + } + + /** + * Resolve one attack object by STIX ID within this request. + * + * The legacy exporter is intentionally best-effort. Deterministic snapshot + * capture will use a strict adapter that treats missing dependencies as an + * integrity failure. + * + * @param {string} stixId + * @returns {Promise} + */ + async getAttackObject(stixId) { + try { + if (this.attackObjectCache.has(stixId)) { + return this.attackObjectCache.get(stixId); + } + + const attackObject = await this.attackObjectsRepository.retrieveLatestByStixIdLean(stixId); + const requestLocalObject = attackObject ? _.cloneDeep(attackObject) : null; + + if (requestLocalObject) { + this.attackObjectCache.set(stixId, requestLocalObject); + } + return requestLocalObject; + } catch (err) { + logger.error(`Error retrieving attack object ${stixId}:`, err); + return null; + } + } + + async getAttackObjectRevision(objectRef, objectModified) { + if (!objectModified || !this.repositoryMap) { + return this.getAttackObject(objectRef); + } + + const cacheKey = `${objectRef}::${new Date(objectModified).getTime()}`; + if (this.attackObjectCache.has(cacheKey)) { + return this.attackObjectCache.get(cacheKey); + } + + const repository = this.repositoryMap[objectRef.split('--')[0]]; + if (!repository) { + return null; + } + + const attackObject = ( + await repository.findManyByIdAndModified([ + { + object_ref: objectRef, + object_modified: objectModified, + }, + ]) + )[0]; + const requestLocalObject = attackObject ? _.cloneDeep(attackObject) : null; + this.attackObjectCache.set(cacheKey, requestLocalObject); + return requestLocalObject; + } + + async getRelationshipEndpoint(relationship, side) { + const endpoint = relationship.workspace?.relationship_endpoints?.[side]; + const objectRef = relationship.stix[`${side}_ref`]; + const object = await this.getAttackObjectRevision( + objectRef, + endpoint?.object_ref === objectRef ? endpoint.object_modified : undefined, + ); + if (!object && endpoint?.object_ref === objectRef && endpoint.object_modified) { + this.onMissingDependency?.({ + object_ref: objectRef, + object_modified: endpoint.object_modified, + dependency: 'relationship_endpoint', + }); + } + return object; + } + + addAttackObject(attackObject, objects, objectsMap) { + if (!attackObject || objectsMap.has(this.documentKey(attackObject))) { + return; + } + + objects.push(attackObject.stix); + objectsMap.set(this.documentKey(attackObject), attackObject); + const attackId = linkById.getAttackId(attackObject.stix); + if (attackId) { + this.attackObjectByAttackIdCache.set(attackId, attackObject); + } + } + + relationshipCanBeEmitted(relationship, objectsMap) { + return ( + !this.policy.isDeprecatedPattern(relationship.stix) && + this.policy.relationshipIsActive(relationship) && + this.hasEndpoint(objectsMap, relationship, 'source') && + this.hasEndpoint(objectsMap, relationship, 'target') + ); + } + + async processSecondaryObject(secondaryObject) { + if (!this.policy.secondaryObjectIsValid(secondaryObject, this.options)) { + return false; + } + + if ( + this.options.inferDomains !== false && + (secondaryObject.stix.type === 'intrusion-set' || secondaryObject.stix.type === 'campaign') + ) { + if (secondaryObject.stix.x_mitre_domains) { + this.domainCache.set(secondaryObject.stix.id, secondaryObject.stix.x_mitre_domains); + } + secondaryObject.stix.x_mitre_domains = + await this.getDomainsForSecondaryObject(secondaryObject); + } + return true; + } + + async getDomainsForSecondaryObject(attackObject) { + const relationships = this.relationships.filter( + (relationship) => relationship.stix.source_ref === attackObject.stix.id, + ); + + const domains = new Set(); + for (const relationship of relationships) { + const targetObject = await this.getRelationshipEndpoint(relationship, 'target'); + const targetDomains = + this.domainCache.get(targetObject?.stix.id) || targetObject?.stix.x_mitre_domains || []; + for (const domain of targetDomains) { + domains.add(domain); + } + } + return [...domains]; + } + + async addSecondaryObjects(primaryObjectRelationships, objectsMap, objects) { + for (const relationship of primaryObjectRelationships) { + if (relationship.stix.relationship_type === 'detects') { + continue; + } + + let secondarySide; + if (!this.hasEndpoint(objectsMap, relationship, 'source')) { + secondarySide = 'source'; + } else if (!this.hasEndpoint(objectsMap, relationship, 'target')) { + secondarySide = 'target'; + } + + if (!secondarySide) { + continue; + } + + const secondaryObject = await this.getRelationshipEndpoint(relationship, secondarySide); + if (await this.processSecondaryObject(secondaryObject)) { + const primarySide = secondarySide === 'source' ? 'target' : 'source'; + this.rememberDependency( + secondaryObject, + this.endpointDocument(objectsMap, relationship, primarySide), + ); + this.addAttackObject(secondaryObject, objects, objectsMap); + } + } + } + + async processSecondaryRelationships(objects, objectsMap) { + for (const relationship of this.relationships) { + await this.addAttributedGroup(relationship, objects, objectsMap); + await this.addDetectionStrategy(relationship, objects, objectsMap); + await this.addRevokedSecondaryObject(relationship, objects, objectsMap); + } + + const analyticIds = objects + .filter((object) => object.type === 'x-mitre-analytic') + .map((analytic) => analytic.id); + + if (analyticIds.length === 0) { + return; + } + + const detectionStrategyDocs = await this.detectionStrategiesRepository.findByAnalyticRefs( + analyticIds, + this.options, + ); + + for (const sourceDoc of detectionStrategyDocs) { + const detectionStrategyDoc = _.cloneDeep(sourceDoc); + if ( + !objectsMap.has(this.documentKey(detectionStrategyDoc)) && + this.policy.secondaryObjectIsValid(detectionStrategyDoc, this.options) + ) { + for (const analyticId of detectionStrategyDoc.stix.x_mitre_analytic_refs || []) { + for (const candidate of objectsMap.values()) { + if (candidate.stix.id === analyticId) { + this.rememberDependency(detectionStrategyDoc, candidate); + } + } + } + this.rememberAndSetDomains(detectionStrategyDoc, [this.options.domain]); + this.addAttackObject(detectionStrategyDoc, objects, objectsMap); + } + } + } + + async addAttributedGroup(relationship, objects, objectsMap) { + if ( + relationship.stix.relationship_type !== 'attributed-to' || + !this.hasEndpoint(objectsMap, relationship, 'source') || + this.hasEndpoint(objectsMap, relationship, 'target') + ) { + return; + } + + const groupObject = await this.getRelationshipEndpoint(relationship, 'target'); + if ( + groupObject?.stix.type === 'intrusion-set' && + this.policy.secondaryObjectIsValid(groupObject, this.options) + ) { + this.rememberDependency( + groupObject, + this.endpointDocument(objectsMap, relationship, 'source'), + ); + this.rememberAndSetDomains(groupObject, [this.options.domain]); + this.addAttackObject(groupObject, objects, objectsMap); + } + } + + async addDetectionStrategy(relationship, objects, objectsMap) { + if ( + relationship.stix.relationship_type !== 'detects' || + !this.hasEndpoint(objectsMap, relationship, 'target') || + this.hasEndpoint(objectsMap, relationship, 'source') + ) { + return; + } + + const detectionStrategy = await this.getRelationshipEndpoint(relationship, 'source'); + if ( + detectionStrategy?.stix.type === 'x-mitre-detection-strategy' && + this.policy.secondaryObjectIsValid(detectionStrategy, this.options) + ) { + this.rememberDependency( + detectionStrategy, + this.endpointDocument(objectsMap, relationship, 'target'), + ); + this.rememberAndSetDomains(detectionStrategy, [this.options.domain]); + this.addAttackObject(detectionStrategy, objects, objectsMap); + } + } + + async addRevokedSecondaryObject(relationship, objects, objectsMap) { + if ( + relationship.stix.relationship_type !== 'revoked-by' || + this.hasEndpoint(objectsMap, relationship, 'source') || + !this.hasEndpoint(objectsMap, relationship, 'target') + ) { + return; + } + + const revokedObject = await this.getRelationshipEndpoint(relationship, 'source'); + if (!this.policy.secondaryObjectIsValid(revokedObject, this.options)) { + return; + } + + this.rememberDependency( + revokedObject, + this.endpointDocument(objectsMap, relationship, 'target'), + ); + if (revokedObject.stix.type === 'intrusion-set' || revokedObject.stix.type === 'campaign') { + this.rememberAndSetDomains(revokedObject, [this.options.domain]); + } + this.addAttackObject(revokedObject, objects, objectsMap); + } + + rememberAndSetDomains(attackObject, domains) { + if (this.options.inferDomains === false) { + return; + } + if (attackObject.stix.x_mitre_domains) { + this.domainCache.set(attackObject.stix.id, attackObject.stix.x_mitre_domains); + } + attackObject.stix.x_mitre_domains = domains; + } +} + +module.exports = BundleGraphResolver; diff --git a/app/services/stix/collections-service.js b/app/services/stix/collections-service.js index 4551d14f..4b2b63d5 100644 --- a/app/services/stix/collections-service.js +++ b/app/services/stix/collections-service.js @@ -256,6 +256,36 @@ class CollectionsService extends BaseService { } } + async assertCollectionContentsCanBeDeleted(collection, stixId, modified) { + for (const reference of collection.stix.x_mitre_contents || []) { + const referenceObj = await attackObjectsService.retrieveOneByVersionLean( + reference.object_ref, + reference.object_modified, + ); + if (!referenceObj) continue; + + const matchQuery = { + 'stix.id': { $ne: stixId }, + 'stix.x_mitre_contents': { + $elemMatch: { + object_ref: reference.object_ref, + object_modified: reference.object_modified, + }, + }, + }; + if (modified) { + delete matchQuery['stix.id']; + matchQuery.$or = [{ 'stix.id': { $ne: stixId } }, { 'stix.modified': { $ne: modified } }]; + } + + const matches = await this.repository.findWithContents(matchQuery, { lean: true }); + if (matches.length === 0) { + await BaseService.assertNotMemberPinned(referenceObj, 'deleted'); + await BaseService.assertNotGraphPinned(referenceObj, 'deleted'); + } + } + } + async delete(stixId, deleteAllContents = false) { if (!stixId) { throw new MissingParameterError('stixId'); @@ -266,7 +296,15 @@ class CollectionsService extends BaseService { throw new BadlyFormattedParameterError({ parameterName: 'stixId' }); } + for (const collection of collections) { + await BaseService.assertNotMemberPinned(collection, 'deleted'); + await BaseService.assertNotGraphPinned(collection, 'deleted'); + } + if (deleteAllContents) { + for (const collection of collections) { + await this.assertCollectionContentsCanBeDeleted(collection, stixId); + } for (const collection of collections) { await this.deleteAllContentsOfCollection(collection, stixId); } @@ -290,7 +328,11 @@ class CollectionsService extends BaseService { throw new BadlyFormattedParameterError({ parameterName: 'stixId' }); } + await BaseService.assertNotMemberPinned(collection, 'deleted'); + await BaseService.assertNotGraphPinned(collection, 'deleted'); + if (deleteAllContents) { + await this.assertCollectionContentsCanBeDeleted(collection, stixId, modified); await this.deleteAllContentsOfCollection(collection, stixId, modified); } diff --git a/app/services/stix/relationships-service.js b/app/services/stix/relationships-service.js index 8f32da1f..79f9c9e4 100644 --- a/app/services/stix/relationships-service.js +++ b/app/services/stix/relationships-service.js @@ -1,11 +1,14 @@ 'use strict'; +const _ = require('lodash'); const { BaseService } = require('../meta-classes'); const relationshipsRepository = require('../../repository/relationships-repository'); +const attackObjectsRepository = require('../../repository/attack-objects-repository'); const { Relationship: RelationshipType } = require('../../lib/types'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); const logger = require('../../lib/logger'); +const { BadRequestError, InvalidObjectRevisionError } = require('../../exceptions'); // Map STIX types to ATT&CK types const objectTypeMap = new Map([ @@ -23,11 +26,95 @@ const objectTypeMap = new Map([ ]); class RelationshipsService extends BaseService { + /** + * Resolve STIX ID-only relationship endpoints to the exact object revisions + * they mean when this relationship revision is created. + * + * The pins are Workbench metadata rather than custom STIX properties. They + * are therefore validated by Mongoose, remain server-controlled, and are + * naturally omitted from emitted STIX bundles. + * + * @param {Object} data Workbench-shaped relationship document + * @returns {Promise} + */ + static async pinEndpointRevisions(data) { + const [source, target] = await Promise.all([ + attackObjectsRepository.retrieveLatestByStixIdLean(data.stix.source_ref), + attackObjectsRepository.retrieveLatestByStixIdLean(data.stix.target_ref), + ]); + + const missing = []; + if (!source) { + missing.push({ + endpoint: 'source', + object_ref: data.stix.source_ref, + object_modified: 'latest', + }); + } + if (!target) { + missing.push({ + endpoint: 'target', + object_ref: data.stix.target_ref, + object_modified: 'latest', + }); + } + if (missing.length > 0) { + throw new InvalidObjectRevisionError(missing, { + details: + 'Relationship endpoints must resolve to exact object revisions before the ' + + 'relationship can be persisted.', + }); + } + + data.workspace = data.workspace || {}; + data.workspace.relationship_endpoints = { + source: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + target: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + }; + } + + async beforeCreate(data) { + await RelationshipsService.pinEndpointRevisions(data); + } + + async beforeUpdate(_stixId, _stixModified, data, existingDocument) { + for (const field of ['source_ref', 'target_ref', 'relationship_type']) { + if (data.stix[field] !== existingDocument.stix[field]) { + throw new BadRequestError({ + details: + `Relationship ${field} cannot be changed in place because that would alter the ` + + 'meaning of an existing graph revision. Create a new relationship revision instead.', + immutable_property: field, + }); + } + } + + data.workspace = data.workspace || {}; + data.workspace.relationship_endpoints = existingDocument.workspace.relationship_endpoints; + } + /** * Initialize event listeners. * Called once on module load. */ static initializeEventListeners() { + const endpointRevisionEvents = [ + ...new Set([ + ...Object.values(EventConstants).filter((eventName) => eventName.endsWith('::created')), + 'identity::created', + 'note::created', + ]), + ]; + for (const event of endpointRevisionEvents) { + EventBus.on(event, this.handleEndpointRevisionCreated.bind(this)); + } + const revokedEvents = [ EventConstants.ATTACK_PATTERN_REVOKED, EventConstants.TACTIC_REVOKED, @@ -69,6 +156,80 @@ class RelationshipsService extends BaseService { logger.info('RelationshipsService: Event listeners initialized'); } + /** + * Carry active relationship edges forward when one of their exact endpoint + * revisions advances. + * + * The prior SRO revision remains pinned to the prior endpoint revisions. + * A new SRO revision is created for the new endpoint state, preserving STIX + * revision immutability while retaining the current graph. + * + * @param {Object} payload Standard BaseService created-event payload + * @returns {Promise<{created: Array}>} + */ + static async handleEndpointRevisionCreated(payload) { + const document = payload?.document; + if (!document?.stix?.id || !document?.stix?.modified) { + return { created: [] }; + } + + const versions = await attackObjectsRepository.retrieveAllById(document.stix.id); + const createdRevisionIndex = versions.findIndex( + (version) => + new Date(version.stix.modified).getTime() === new Date(document.stix.modified).getTime(), + ); + + // Only the latest revision advances the current graph. Older revisions + // arriving in a bulk import retain their historical position. + if (createdRevisionIndex !== 0 || versions.length < 2) { + return { created: [] }; + } + + const previousRevision = versions[1]; + const relationships = await relationshipsRepository.retrieveAllBySourceOrTarget( + document.stix.id, + ); + const relationshipsToAdvance = relationships.filter((relationship) => { + if (relationship.stix.revoked || relationship.stix.x_mitre_deprecated) { + return false; + } + + const endpoints = relationship.workspace?.relationship_endpoints; + return ['source', 'target'].some( + (side) => + endpoints?.[side]?.object_ref === previousRevision.stix.id && + new Date(endpoints[side].object_modified).getTime() === + new Date(previousRevision.stix.modified).getTime(), + ); + }); + + const created = []; + for (const relationship of relationshipsToAdvance) { + const relationshipData = _.cloneDeep(relationship); + delete relationshipData._id; + delete relationshipData.__v; + delete relationshipData.__t; + if (relationshipData.workspace) { + delete relationshipData.workspace.release_tracks; + delete relationshipData.workspace.relationship_endpoints; + } + + const previousRelationshipModified = new Date(relationship.stix.modified).getTime(); + relationshipData.stix.modified = new Date( + Math.max(Date.now(), previousRelationshipModified + 1), + ).toISOString(); + + created.push( + await module.exports.create(relationshipData, { + userAccountId: payload.options?.userAccountId, + automationContext: payload.options?.automationContext, + }), + ); + } + + return { created }; + } + /** * Return the latest active relationship revisions whose endpoints are both * in the requested bundle object set. diff --git a/app/services/stix/stix-bundles-service.js b/app/services/stix/stix-bundles-service.js index abd0750b..199b7f60 100644 --- a/app/services/stix/stix-bundles-service.js +++ b/app/services/stix/stix-bundles-service.js @@ -5,9 +5,9 @@ const config = require('../../config/config'); const { BaseService } = require('../meta-classes'); const linkById = require('../../lib/linkById'); const bundleRelationships = require('../../lib/stix-bundle-relationships'); -const logger = require('../../lib/logger'); const { requiresAttackId } = require('../../lib/attack-id-generator'); const stixConformance = require('../../lib/stix-conformance'); +const BundleGraphResolver = require('./bundle-graph-resolver'); // Import repositories const analyticsRepository = require('../../repository/analytics-repository'); @@ -243,47 +243,6 @@ class StixBundlesService extends BaseService { } } - /** - * Adds an ATT&CK object to the STIX bundle - * @param {Object} attackObject - The ATT&CK object to add - * @param {Object} bundle - The STIX bundle being built - * @param {Map} objectsMap - Map tracking objects in the bundle - - */ - addAttackObjectToBundle(attackObject, bundle, objectsMap) { - if (!objectsMap.has(attackObject.stix.id)) { - bundle.objects.push(attackObject.stix); - objectsMap.set(attackObject.stix.id, true); - const attackId = linkById.getAttackId(attackObject.stix); - if (attackId) { - this.attackObjectByAttackIdCache.set(attackId, attackObject); - } - } - } - - /** - * Processes a secondary object for inclusion in the bundle. - * Validates the object and updates necessary data structures. - * @param {Object} secondaryObject - The secondary object to process - * @param {Object} options - Bundle generation options - * @returns {Promise} True if object was successfully processed - */ - async processSecondaryObject(secondaryObject, options) { - if (!StixBundlesService.secondaryObjectIsValid(secondaryObject, options)) { - return false; - } - - // Handle domains for groups and campaigns - if (secondaryObject.stix.type === 'intrusion-set' || secondaryObject.stix.type === 'campaign') { - if (secondaryObject.stix.x_mitre_domains) { - this.domainCache.set(secondaryObject.stix.id, secondaryObject.stix.x_mitre_domains); - } - secondaryObject.stix.x_mitre_domains = - await this.getDomainsForSecondaryObject(secondaryObject); - } - return true; - } - /** * Validates if a secondary object meets all inclusion criteria for the bundle. * @param {Object} secondaryObject - The object to validate @@ -309,38 +268,6 @@ class StixBundlesService extends BaseService { ); } - /** - * Determines the domains associated with a secondary object based on its relationships. - * @param {Object} attackObject - The secondary object to process - * @returns {Promise>} Array of domain names - */ - async getDomainsForSecondaryObject(attackObject) { - const relationships = this.allRelationships.filter( - (relationship) => relationship.stix.source_ref == attackObject.stix.id, - ); - - const domainMap = new Map(); - for (const relationship of relationships) { - const targetObject = await this.getAttackObject(relationship.stix.target_ref); - // domainCache is used to accurately reflect the STIX bundle post-refactoring in project Orion. - // The additional domains that would otherwise be added are likely correct, but that will - // be handled in a separate data cleanup effort not coinciding with the imminent v17 ATT&CK release. - if (this.domainCache.has(targetObject?.stix.id)) { - for (const domain of this.domainCache.get(targetObject.stix.id)) { - domainMap.set(domain, true); - } - } else { - if (targetObject?.stix.x_mitre_domains) { - for (const domain of targetObject.stix.x_mitre_domains) { - domainMap.set(domain, true); - } - } - } - } - - return [...domainMap.keys()]; - } - // ============================ // Collection Object Management // ============================ @@ -446,13 +373,6 @@ class StixBundlesService extends BaseService { * @returns {Promise} The generated STIX bundle */ async exportBundle(options) { - // Initialize caches for efficient object lookup - this.attackObjectCache = new Map(); // Maps STIX IDs to attack objects - this.identityCache = new Map(); // Maps identity STIX IDs to identity objects - this.markingDefinitionsCache = new Map(); // Maps marking definition STIX IDs to marking objects - this.attackObjectByAttackIdCache = new Map(); // Maps attack IDs to attack objects - this.domainCache = new Map(); // Stores original x-mitre-domains if we change them at runtime - // Initialize bundle const bundle = { type: 'bundle', @@ -517,32 +437,20 @@ class StixBundlesService extends BaseService { primaryObjects = primaryObjects.filter((o) => StixBundlesService.hasAttackId(o)); } - // Put the primary objects in the bundle - // Also create a map of the objects added to the bundle (use the id as the key, since relationships only reference the id) - const objectsMap = new Map(); - for (const primaryObject of primaryObjects) { - this.addAttackObjectToBundle(primaryObject, bundle, objectsMap); - } - - // Since we're querying all relationships, save them for later to prevent future database queries. - this.allRelationships = await this.repositories.relationship.retrieveAllForBundle(options); - - // Filter relationships that have a source_ref or target_ref that points at a primary object - const primaryObjectRelationships = this.allRelationships.filter( - (relationship) => - objectsMap.has(relationship.stix.source_ref) || - objectsMap.has(relationship.stix.target_ref), - ); - - // Get the secondary objects (additional objects pointed to by a relationship) - await this.addSecondaryObjects(primaryObjectRelationships, objectsMap, bundle, options); - - await this.processSecondaryRelationships(bundle, objectsMap, options); - - // Add all valid relationships to the bundle - for (const relationship of this.allRelationships) { - StixBundlesService.addRelationshipToBundle(relationship, bundle, objectsMap); - } + const relationships = await this.repositories.relationship.retrieveAllForBundle(options); + const graphResolver = new BundleGraphResolver({ + attackObjectsRepository: this.repositories.attackObject, + detectionStrategiesRepository: this.repositories.detectionStrategy, + policy: { + isDeprecatedPattern: StixBundlesService.isDeprecatedPattern, + relationshipIsActive: StixBundlesService.relationshipIsActive, + secondaryObjectIsValid: StixBundlesService.secondaryObjectIsValid, + }, + options, + relationships, + }); + const resolvedGraph = await graphResolver.resolve(primaryObjects); + bundle.objects.push(...resolvedGraph.objects); // Add notes if requested if (options.includeNotes) { @@ -550,10 +458,10 @@ class StixBundlesService extends BaseService { } // Convert LinkById tags to markdown citations - await this.convertLinkByIdTags(bundle.objects, this.attackObjectByAttackIdCache); + await this.convertLinkByIdTags(bundle.objects, resolvedGraph.attackObjectByAttackIdCache); // Process identities and marking definitions - await this.processIdentitiesAndMarkings(bundle); + bundle.objects.push(...(await graphResolver.loadSupportingObjects(bundle.objects))); // Conform to STIX version for (const stixObject of bundle.objects) { @@ -566,274 +474,6 @@ class StixBundlesService extends BaseService { return bundle; } - /** - * Add secondary objects to the bundle - those objects which have a relationship - * to a primary object but did not have the proper domain in the database. - * - * Note: 'detects' relationships are skipped here and handled separately in - * processSecondaryRelationships() to support the new ATT&CK spec where only - * detection strategies (not data components) can detect techniques. - * - * @param {Array} primaryObjectRelationships - The relationships to process - * @param {Map} objectsMap - Map of objects currently in the bundle - * @param {Object} bundle - The STIX bundle being built - * @param {Object} options - Bundle generation options - * @returns {Promise} - */ - async addSecondaryObjects(primaryObjectRelationships, objectsMap, bundle, options) { - for (const relationship of primaryObjectRelationships) { - // Skip 'detects' relationships - they require special handling - // - // CONTEXT: The ATT&CK specification changed how detection works: - // - OLD (pre-v17): Data components could detect techniques via 'detects' relationships - // - NEW (v17+): Only detection strategies can detect techniques via 'detects' relationships - // - // WHY WE SKIP HERE: - // 1. Data components are now PRIMARY objects (retrieved by domain), not secondary - // 2. If we processed 'detects' relationships here, we would incorrectly add data - // components as secondary objects based on deprecated relationships - // 3. Detection strategies ARE secondary objects, but they need special domain - // inference logic (they get the domain of the technique they detect) - // - // WHERE THEY'RE HANDLED: - // 'detects' relationships are processed in processSecondaryRelationships() where: - // - We verify the source is a detection strategy (not a data component) - // - We set the detection strategy's x_mitre_domains to match the target technique - // - Deprecated 'detects' from data components are silently ignored - if (relationship.stix.relationship_type === 'detects') { - continue; - } - - if (!objectsMap.has(relationship.stix.source_ref)) { - const secondaryObject = await this.getAttackObject(relationship.stix.source_ref); - - // Only process if the secondary object meets our inclusion criteria - if (await this.processSecondaryObject(secondaryObject, options)) { - this.addAttackObjectToBundle(secondaryObject, bundle, objectsMap); - } - } else if (!objectsMap.has(relationship.stix.target_ref)) { - const secondaryObject = await this.getAttackObject(relationship.stix.target_ref); - - // Only process if the secondary object meets our inclusion criteria - if (await this.processSecondaryObject(secondaryObject, options)) { - this.addAttackObjectToBundle(secondaryObject, bundle, objectsMap); - } - } - } - } - - /** - * Processes all identities and marking definitions referenced in the bundle. - * This ensures that all necessary context objects are included. - * - * Steps: - * 1. Collect all identity references (created_by_ref) - * 2. Collect all marking definition references (object_marking_refs) - * 3. Retrieve objects from cache or database - * 4. Add valid objects to bundle - * 5. Log warnings for missing references - * - * @param {Object} bundle - The STIX bundle being built - * @returns {Promise} - */ - async processIdentitiesAndMarkings(bundle) { - // Map referenced identities and marking definitions - const identitiesMap = new Map(); - const markingDefinitionsMap = new Map(); - - for (const bundleObject of bundle.objects) { - if (bundleObject.created_by_ref) { - identitiesMap.set(bundleObject.created_by_ref, true); - } - - if (bundleObject.object_marking_refs) { - for (const markingRef of bundleObject.object_marking_refs) { - markingDefinitionsMap.set(markingRef, true); - } - } - } - - // Process identities - for (const stixId of identitiesMap.keys()) { - if (this.identityCache.has(stixId)) { - bundle.objects.push(this.identityCache.get(stixId)); - continue; - } - - const identity = await this.getAttackObject(stixId); - if (identity) { - bundle.objects.push(identity.stix); - this.identityCache.set(stixId, identity.stix); - } else { - logger.warn(`Referenced identity not found: ${stixId}`); - } - } - - // Process marking definitions - for (const stixId of markingDefinitionsMap.keys()) { - if (this.markingDefinitionsCache.has(stixId)) { - bundle.objects.push(this.markingDefinitionsCache.get(stixId)); - continue; - } - - const markingDefinition = await this.getAttackObject(stixId); - if (markingDefinition) { - bundle.objects.push(markingDefinition.stix); - this.markingDefinitionsCache.set(stixId, markingDefinition.stix); - } - } - } - - /** - * Processes relationships between secondary objects and handles special cases that need separate processing: - * - Groups referenced by campaigns through 'attributed-to' relationships - * - Detection strategies that detect techniques in the bundle - * - Detection strategies referenced by analytics in the bundle - * - Secondary objects that were revoked by other secondary objects - * - * @param {Object} bundle - The STIX bundle being built - * @param {Map} objectsMap - Map tracking objects currently in bundle - * @param {Object} options - Bundle generation options - * @param {string} options.domain - The domain being processed - * @returns {Promise} - */ - async processSecondaryRelationships(bundle, objectsMap, options) { - for (const relationship of this.allRelationships) { - // Add groups referenced by campaigns through 'attributed-to' relationships - if ( - relationship.stix.relationship_type === 'attributed-to' && - objectsMap.has(relationship.stix.source_ref) && - !objectsMap.has(relationship.stix.target_ref) - ) { - const groupObject = await this.getAttackObject(relationship.stix.target_ref); - if ( - groupObject.stix.type === 'intrusion-set' && - StixBundlesService.secondaryObjectIsValid(groupObject, options) - ) { - if (groupObject.stix.x_mitre_domains) { - this.domainCache.set(groupObject.stix.id, groupObject.stix.x_mitre_domains); - } - groupObject.stix.x_mitre_domains = [options.domain]; - this.addAttackObjectToBundle(groupObject, bundle, objectsMap); - } - } - - // Add detection strategies that detect techniques in the bundle - if ( - relationship.stix.relationship_type === 'detects' && - objectsMap.has(relationship.stix.target_ref) && - !objectsMap.has(relationship.stix.source_ref) - ) { - const detectionStrategy = await this.getAttackObject(relationship.stix.source_ref); - if ( - detectionStrategy.stix.type === 'x-mitre-detection-strategy' && - StixBundlesService.secondaryObjectIsValid(detectionStrategy, options) - ) { - if (detectionStrategy.stix.x_mitre_domains) { - this.domainCache.set(detectionStrategy.stix.id, detectionStrategy.stix.x_mitre_domains); - } - // Set x_mitre_domains on each exported detection strategy - detectionStrategy.stix.x_mitre_domains = [options.domain]; - this.addAttackObjectToBundle(detectionStrategy, bundle, objectsMap); - } - } - - // Add secondary objects that were revoked by other secondary objects - if ( - relationship.stix.relationship_type === 'revoked-by' && - !objectsMap.has(relationship.stix.source_ref) && - objectsMap.has(relationship.stix.target_ref) - ) { - const revokedObject = await this.getAttackObject(relationship.stix.source_ref); - if (StixBundlesService.secondaryObjectIsValid(revokedObject, options)) { - if ( - revokedObject.stix.type === 'intrusion-set' || - revokedObject.stix.type === 'campaign' - ) { - if (revokedObject.stix.x_mitre_domains) { - this.domainCache.set(revokedObject.stix.id, revokedObject.stix.x_mitre_domains); - } - revokedObject.stix.x_mitre_domains = [options.domain]; - } - this.addAttackObjectToBundle(revokedObject, bundle, objectsMap); - } - } - } - - // Add detection strategies referenced by analytics in the bundle - // This is a key requirement of the new ATT&CK spec: detection strategies should be - // included if they reference an analytic that is in the domain - const analyticsInBundle = bundle.objects.filter((obj) => obj.type === 'x-mitre-analytic'); - - if (analyticsInBundle.length > 0) { - // Collect all analytic IDs in the bundle - const analyticIds = analyticsInBundle.map((analytic) => analytic.id); - - // Single batch query to find all detection strategies that reference any of these analytics - // This replaces the N+1 query pattern that was causing timeouts - const detectionStrategyDocs = await this.repositories.detectionStrategy.findByAnalyticRefs( - analyticIds, - options, - ); - - for (const detectionStrategyDoc of detectionStrategyDocs) { - if ( - !objectsMap.has(detectionStrategyDoc.stix.id) && - StixBundlesService.secondaryObjectIsValid(detectionStrategyDoc, options) - ) { - if (detectionStrategyDoc.stix.x_mitre_domains) { - this.domainCache.set( - detectionStrategyDoc.stix.id, - detectionStrategyDoc.stix.x_mitre_domains, - ); - } - // Set x_mitre_domains on each exported detection strategy - detectionStrategyDoc.stix.x_mitre_domains = [options.domain]; - this.addAttackObjectToBundle(detectionStrategyDoc, bundle, objectsMap); - } - } - } - } - - // ============================ - // Repository Access Methods (+Cache Management) - // ============================ - - /** - * Retrieves an attack object by its STIX ID, using cache when possible. - * Implements a caching strategy to minimize database queries. - * - * Process: - * 1. Check cache using STIX ID - * 2. If not found, query database - * 3. If found in database, cache for future use - * 4. Handle errors gracefully - * - * @param {string} stixId - The STIX ID of the object to retrieve - * @returns {Promise} The attack object or null if not found/error - */ - async getAttackObject(stixId) { - try { - // First check cache - const cacheKey = stixId; - if (this.attackObjectCache.has(cacheKey)) { - return this.attackObjectCache.get(cacheKey); - } - - // Use the existing repository method that exactly matches the original logic - const attackObject = await this.repositories.attackObject.retrieveLatestByStixIdLean(stixId); - - if (attackObject) { - this.attackObjectCache.set(cacheKey, attackObject); - } - - return attackObject; - } catch (err) { - logger.error(`Error retrieving attack object ${stixId}:`, err); - return null; - } - } - /** * Converts LinkById tags to markdown citations * @param {Array} bundleObjects - Objects in the bundle diff --git a/app/tests/api/attack-objects/attack-objects.spec.js b/app/tests/api/attack-objects/attack-objects.spec.js index 287d8bc4..bd4a9184 100644 --- a/app/tests/api/attack-objects/attack-objects.spec.js +++ b/app/tests/api/attack-objects/attack-objects.spec.js @@ -141,8 +141,10 @@ describe('ATT&CK Objects API', function () { const markingDefinitions = attackObjects.filter((x) => x.stix.type === 'marking-definition'); expect(markingDefinitions.length).toBe(5); - // Placeholder identity, 4 TLP marking definitions, 18 collection contents, 2 collection objects - expect(attackObjects.length).toBe(1 + 4 + 18 + 2); + // Placeholder identity, 4 TLP marking definitions, 18 imported collection contents, + // 2 collection objects, and the propagated relationship revision pinned to the + // second bundle's newer target revision. + expect(attackObjects.length).toBe(1 + 4 + 18 + 2 + 1); }); it('GET /api/attack-objects returns zero objects with an ATT&CK ID that does not exist', async function () { diff --git a/app/tests/api/relationships/relationship-endpoint-pins.spec.js b/app/tests/api/relationships/relationship-endpoint-pins.spec.js new file mode 100644 index 00000000..672e98a4 --- /dev/null +++ b/app/tests/api/relationships/relationship-endpoint-pins.spec.js @@ -0,0 +1,170 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const { cloneForCreate } = require('../../shared/clone-for-create'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Relationship endpoint revision pins', function () { + let app; + let passportCookie; + let source; + let target; + let relationship; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, expectedStatus = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return response.body; + } + + before('create endpoint objects and their relationship', async function () { + const sourceTimestamp = new Date().toISOString(); + source = await post('/api/software', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'malware', + spec_version: '2.1', + created: sourceTimestamp, + modified: sourceTimestamp, + name: 'Revision-pinned source', + description: 'Source object for relationship revision pin tests.', + is_family: false, + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + object_marking_refs: [markingDefinitionId], + }, + }); + + const targetTimestamp = new Date().toISOString(); + target = await post('/api/techniques', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + created: targetTimestamp, + modified: targetTimestamp, + name: 'Revision-pinned target', + description: 'Target object for relationship revision pin tests.', + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'execution' }], + x_mitre_is_subtechnique: false, + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + object_marking_refs: [markingDefinitionId], + }, + }); + + const relationshipTimestamp = new Date().toISOString(); + relationship = await post('/api/relationships', { + workspace: { + workflow: { state: 'work-in-progress' }, + relationship_endpoints: { + source: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + target: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + }, + }, + stix: { + type: 'relationship', + spec_version: '2.1', + created: relationshipTimestamp, + modified: relationshipTimestamp, + relationship_type: 'uses', + source_ref: source.stix.id, + target_ref: target.stix.id, + object_marking_refs: [markingDefinitionId], + }, + }); + }); + + it('stores server-resolved exact endpoint revisions outside the STIX payload', function () { + expect(relationship.workspace.relationship_endpoints).toEqual({ + source: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + target: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + }); + expect(relationship.stix.x_mitre_source_ref_modified).toBeUndefined(); + expect(relationship.stix.x_mitre_target_ref_modified).toBeUndefined(); + }); + + it('creates a new SRO revision when an endpoint advances', async function () { + const sourceRevision = cloneForCreate(source); + sourceRevision.stix.modified = new Date( + new Date(source.stix.modified).getTime() + 1000, + ).toISOString(); + sourceRevision.stix.description = 'A newer source revision.'; + + const newSource = await post('/api/software', sourceRevision); + const response = await request(app) + .get(`/api/relationships/${relationship.stix.id}?versions=all`) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + expect(response.body).toHaveLength(2); + const [latestRelationship, originalRelationship] = response.body; + expect(latestRelationship.stix.id).toBe(relationship.stix.id); + expect(latestRelationship.stix.modified).not.toBe(originalRelationship.stix.modified); + expect(latestRelationship.workspace.relationship_endpoints.source).toEqual({ + object_ref: source.stix.id, + object_modified: newSource.stix.modified, + }); + expect(latestRelationship.workspace.relationship_endpoints.target).toEqual({ + object_ref: target.stix.id, + object_modified: target.stix.modified, + }); + expect(originalRelationship.workspace.relationship_endpoints.source).toEqual({ + object_ref: source.stix.id, + object_modified: source.stix.modified, + }); + }); + + it('does not emit internal endpoint pins in STIX bundles', async function () { + const response = await request(app) + .get('/api/release-tracks/ephemeral/enterprise?includeToc=false') + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + const emittedRelationship = response.body.objects.find( + (object) => object.id === relationship.stix.id, + ); + expect(emittedRelationship).toBeDefined(); + expect(emittedRelationship.workspace).toBeUndefined(); + expect(emittedRelationship.x_mitre_source_ref_modified).toBeUndefined(); + expect(emittedRelationship.x_mitre_target_ref_modified).toBeUndefined(); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/relationships/relationships-pagination.spec.js b/app/tests/api/relationships/relationships-pagination.spec.js index de2f5fd9..63f1fd5e 100644 --- a/app/tests/api/relationships/relationships-pagination.spec.js +++ b/app/tests/api/relationships/relationships-pagination.spec.js @@ -1,6 +1,8 @@ const relationshipsService = require('../../../services/stix/relationships-service'); const PaginationTests = require('../../shared/pagination'); const config = require('../../../config/config'); +const Software = require('../../../models/software-model'); +const Technique = require('../../../models/technique-model'); config.validateRequests.withOpenApi = true; @@ -33,8 +35,39 @@ const options = { label: 'Relationships', validateWithAdm: true, }; +let endpointsCreated = false; const relationshipsPaginationService = { async create(data, options) { + if (!endpointsCreated) { + const endpointModified = new Date(); + await Promise.all([ + Software.create({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'malware', + spec_version: '2.1', + id: sourceRef1, + created: endpointModified, + modified: endpointModified, + name: 'Pagination relationship source', + is_family: false, + }, + }), + Technique.create({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + id: targetRef1, + created: endpointModified, + modified: endpointModified, + name: 'Pagination relationship target', + x_mitre_is_subtechnique: false, + }, + }), + ]); + endpointsCreated = true; + } delete data.stix.name; return relationshipsService.create(data, options); }, diff --git a/app/tests/api/relationships/relationships.spec.js b/app/tests/api/relationships/relationships.spec.js index 9e71a4b8..982968e7 100644 --- a/app/tests/api/relationships/relationships.spec.js +++ b/app/tests/api/relationships/relationships.spec.js @@ -7,6 +7,8 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const config = require('../../../config/config'); const login = require('../../shared/login'); const { cloneForCreate } = require('../../shared/clone-for-create'); +const Software = require('../../../models/software-model'); +const Technique = require('../../../models/technique-model'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -27,6 +29,16 @@ const initialObjectData = { workflow: { state: 'work-in-progress', }, + relationship_endpoints: { + source: { + object_ref: 'malware--00000000-0000-4000-8000-000000000000', + object_modified: '2000-01-01T00:00:00.000Z', + }, + target: { + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000000', + object_modified: '2000-01-01T00:00:00.000Z', + }, + }, }, stix: { spec_version: '2.1', @@ -44,6 +56,7 @@ const initialObjectData = { describe('Relationships API', function () { let app; let passportCookie; + let endpointModified; before(async function () { // Establish the database connection @@ -62,6 +75,37 @@ describe('Relationships API', function () { // Log into the app passportCookie = await login.loginAnonymous(app); + + endpointModified = new Date(); + const endpointCreated = new Date(endpointModified); + await Software.create( + [sourceRef1, sourceRef2].map((id, index) => ({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'malware', + spec_version: '2.1', + id, + created: endpointCreated, + modified: endpointModified, + name: `Relationship source ${index + 1}`, + is_family: false, + }, + })), + ); + await Technique.create( + [targetRef1, targetRef2].map((id, index) => ({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + id, + created: endpointCreated, + modified: endpointModified, + name: `Relationship target ${index + 1}`, + x_mitre_is_subtechnique: false, + }, + })), + ); }); it('GET /api/relationships returns an empty array of relationships', async function () { @@ -110,6 +154,47 @@ describe('Relationships API', function () { expect(relationship1a.stix.created).toBeDefined(); expect(relationship1a.stix.modified).toBeDefined(); expect(relationship1a.stix.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion); + expect(relationship1a.workspace.relationship_endpoints).toEqual({ + source: { + object_ref: sourceRef1, + object_modified: endpointModified.toISOString(), + }, + target: { + object_ref: targetRef1, + object_modified: endpointModified.toISOString(), + }, + }); + }); + + it('POST /api/relationships rejects endpoints that cannot be revision-pinned', async function () { + const timestamp = new Date().toISOString(); + const body = { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'relationship', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + relationship_type: 'uses', + source_ref: sourceRef1, + target_ref: targetRef3, + }, + }; + + const res = await request(app) + .post('/api/relationships') + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(400); + + expect(res.body.missing_references).toEqual([ + { + endpoint: 'target', + object_ref: targetRef3, + object_modified: 'latest', + }, + ]); }); it('GET /api/relationships returns the added relationship', async function () { @@ -189,6 +274,25 @@ describe('Relationships API', function () { expect(relationship.stix.modified).toBe(relationship1a.stix.modified); }); + it('PUT /api/relationships rejects in-place endpoint changes', async function () { + const body = structuredClone(relationship1a); + body.stix.source_ref = sourceRef2; + + const res = await request(app) + .put( + '/api/relationships/' + + relationship1a.stix.id + + '/modified/' + + relationship1a.stix.modified, + ) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(400); + + expect(res.body.immutable_property).toBe('source_ref'); + }); + it('POST /api/relationships does not create a relationship with the same id and modified date', async function () { const body = relationship1a; await request(app) diff --git a/app/tests/api/release-tracks/destructive-authorization.spec.js b/app/tests/api/release-tracks/destructive-authorization.spec.js index 8dd05d0e..682fee1c 100644 --- a/app/tests/api/release-tracks/destructive-authorization.spec.js +++ b/app/tests/api/release-tracks/destructive-authorization.spec.js @@ -13,27 +13,6 @@ const ReleaseTrackAuditEvent = require('../../../models/release-tracks/release-t const auditRepository = require('../../../repository/release-tracks/release-track-audit-event.repository'); const systemConfigurationService = require('../../../services/system/system-configuration-service'); -const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; - -function techniquePayload() { - const timestamp = new Date().toISOString(); - return { - workspace: { workflow: { state: 'work-in-progress' } }, - stix: { - created: timestamp, - modified: timestamp, - name: 'Destructive authorization member', - description: 'Member used by destructive authorization tests.', - spec_version: '2.1', - type: 'attack-pattern', - object_marking_refs: [markingDefinitionId], - kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], - x_mitre_is_subtechnique: false, - x_mitre_platforms: ['Windows'], - }, - }; -} - describe('Release-track destructive authorization and audit', function () { let app; let passportCookie; @@ -77,76 +56,34 @@ describe('Release-track destructive authorization and audit', function () { return (await api('post', path, body, status, query)).body; } - it('requires admin role, exact confirmation, and durable outcome records', async function () { + it('requires admin role, exact confirmation, and a durable outcome record', async function () { await setRole('admin'); - const technique = await post('/api/techniques', techniquePayload(), 201); const track = await post( '/api/release-tracks/new', { name: 'Destructive authorization standard', type: 'standard' }, 201, ); - const contents = { - x_mitre_contents: [ - { - obj_ref: technique.stix.id, - obj_modified: technique.stix.modified, - }, - ], - }; await setRole('editor'); - await api('post', `/api/release-tracks/${track.id}/contents`, contents, 401, { - confirm_track_id: track.id, - }); - await api( - 'post', - `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, - contents, - 401, - { confirm_track_id: track.id }, - ); await api('delete', `/api/release-tracks/${track.id}`, undefined, 401, { confirm_track_id: track.id, }); expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); await setRole('admin'); - await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400); - await api('post', `/api/release-tracks/${track.id}/contents`, contents, 400, { + await api('delete', `/api/release-tracks/${track.id}`, undefined, 400); + await api('delete', `/api/release-tracks/${track.id}`, undefined, 400, { confirm_track_id: 'release-track--00000000-0000-4000-8000-000000000099', }); - await api('delete', `/api/release-tracks/${track.id}`, undefined, 400); expect(await ReleaseTrackAuditEvent.countDocuments()).toBe(0); - const latest = await post(`/api/release-tracks/${track.id}/contents`, contents, 200, { - confirm_track_id: track.id, - }); - await post( - `/api/release-tracks/${track.id}/snapshots/${track.modified}/contents`, - contents, - 200, - { confirm_track_id: track.id }, - ); - - const virtual = await post( - '/api/release-tracks/new', - { name: 'Destructive authorization virtual', type: 'virtual' }, - 201, - ); - await api('post', `/api/release-tracks/${virtual.id}/contents`, contents, 400, { - confirm_track_id: virtual.id, - }); - await api('delete', `/api/release-tracks/${track.id}`, undefined, 204, { confirm_track_id: track.id, }); const events = await ReleaseTrackAuditEvent.find().sort({ started_at: 1 }).lean().exec(); - expect(events).toHaveLength(4); + expect(events).toHaveLength(1); expect(events.map((event) => [event.action, event.status])).toEqual([ - ['replace_members_latest', 'completed'], - ['replace_members_historical', 'completed'], - ['replace_members_latest', 'failed'], ['delete_track', 'completed'], ]); expect(events[0]).toMatchObject({ @@ -157,38 +94,20 @@ describe('Release-track destructive authorization and audit', function () { role: 'admin', authentication_strategy: 'anonymId', }, - request: { members_count: 1 }, - result: { - snapshot_modified: new Date(latest.modified), - members_count: 1, - }, + result: { deleted: true }, }); - expect(events[2].track_id).toBe(virtual.id); - expect(events[2].error.message).toContain( - 'Direct contents updates are only available for standard release tracks', - ); - expect(events[3].result).toEqual({ deleted: true }); }); it('reports an audit-finalization failure without hiding the persisted mutation', async function () { await setRole('admin'); - const technique = await post('/api/techniques', techniquePayload(), 201); const track = await post( '/api/release-tracks/new', - { name: 'Audit finalization failure standard', type: 'standard' }, + { name: 'Track deletion audit finalization failure', type: 'standard' }, 201, ); - const contents = { - x_mitre_contents: [ - { - obj_ref: technique.stix.id, - obj_modified: technique.stix.modified, - }, - ], - }; sinon.stub(auditRepository, 'complete').rejects(new Error('injected audit update failure')); - const response = await api('post', `/api/release-tracks/${track.id}/contents`, contents, 500, { + const response = await api('delete', `/api/release-tracks/${track.id}`, undefined, 500, { confirm_track_id: track.id, }); auditRepository.complete.restore(); @@ -199,17 +118,7 @@ describe('Release-track destructive authorization and audit', function () { }); expect(response.body.audit_event_id).toEqual(expect.any(String)); - const latest = await api( - 'get', - `/api/release-tracks/${track.id}/snapshots/latest`, - undefined, - 200, - ); - expect(latest.body.members).toHaveLength(1); - expect(latest.body.members[0]).toMatchObject({ - object_ref: technique.stix.id, - object_modified: technique.stix.modified, - }); + await api('get', `/api/release-tracks/${track.id}/snapshots/latest`, undefined, 404); const pendingEvent = await ReleaseTrackAuditEvent.findOne({ event_id: response.body.audit_event_id, @@ -217,7 +126,7 @@ describe('Release-track destructive authorization and audit', function () { .lean() .exec(); expect(pendingEvent).toMatchObject({ - action: 'replace_members_latest', + action: 'delete_track', track_id: track.id, status: 'pending', }); diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js new file mode 100644 index 00000000..7a149c2e --- /dev/null +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -0,0 +1,198 @@ +'use strict'; + +const mongoose = require('mongoose'); +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const migration = require('../../../../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); +const Relationship = require('../../../models/relationship-model'); +const { + ReleaseTrackGraphManifest, + ReleaseTrackGraphManifestEntry, +} = require('../../../models/release-tracks/release-track-graph-manifest-model'); +const { releaseExactMembers } = require('./release-track-test-helpers'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Deterministic snapshot graph migration', function () { + let app; + let passportCookie; + let technique; + let group; + let relationship; + let trackId; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + before('create and then downgrade representative legacy data', async function () { + const timestamp = new Date().toISOString(); + technique = await post('/api/techniques', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Migration graph technique', + description: 'A primary migration fixture.', + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'execution' }], + x_mitre_is_subtechnique: false, + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + object_marking_refs: [markingDefinitionId], + }, + }); + group = await post('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'intrusion-set', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Migration graph secondary', + description: 'A secondary migration fixture.', + object_marking_refs: [markingDefinitionId], + }, + }); + relationship = await post('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'relationship', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: technique.stix.id, + object_marking_refs: [markingDefinitionId], + }, + }); + const track = await post( + '/api/release-tracks/new', + { name: 'Legacy migration track', type: 'standard' }, + 201, + ); + trackId = track.id; + await releaseExactMembers(app, passportCookie, trackId, [technique]); + + await Relationship.updateOne( + { + 'stix.id': relationship.stix.id, + 'stix.modified': relationship.stix.modified, + }, + { $unset: { 'workspace.relationship_endpoints': '' } }, + ); + await mongoose.connection.db + .collection(trackId) + .updateMany({}, { $unset: { graph_manifest_id: '' } }); + await Promise.all([ + ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }), + ReleaseTrackGraphManifestEntry.deleteMany({ track_id: trackId }), + ]); + }); + + it('supports a non-mutating dry run', async function () { + const report = await migration._private.run(mongoose.connection.db, { + dryRun: true, + }); + + expect(report.dry_run).toBe(true); + expect(report.relationship_pins_written).toBeGreaterThan(0); + expect(report.manifests_created).toBeGreaterThan(0); + const storedRelationship = await Relationship.findOne({ + 'stix.id': relationship.stix.id, + }) + .lean() + .exec(); + expect(storedRelationship.workspace.relationship_endpoints).toBeUndefined(); + expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: trackId })).toBe(0); + }); + + it('pins latest relationships and rerunnably backfills baseline manifests', async function () { + await migration.up(mongoose.connection.db); + + const storedRelationship = await Relationship.findOne({ + 'stix.id': relationship.stix.id, + }) + .lean() + .exec(); + expect(storedRelationship.workspace.relationship_endpoints.source).toEqual({ + object_ref: group.stix.id, + object_modified: new Date(group.stix.modified), + }); + expect(storedRelationship.workspace.relationship_endpoints.target).toEqual({ + object_ref: technique.stix.id, + object_modified: new Date(technique.stix.modified), + }); + + const manifests = await ReleaseTrackGraphManifest.find({ + track_id: trackId, + }) + .lean() + .exec(); + expect(manifests.length).toBeGreaterThan(0); + expect(manifests.every((manifest) => manifest.baseline_reconstruction === true)).toBe(true); + const countAfterFirstRun = manifests.length; + + await migration.up(mongoose.connection.db); + expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: trackId })).toBe( + countAfterFirstRun, + ); + + const response = await request(app) + .get(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + const objectIds = response.body.objects.map((object) => object.id); + expect(objectIds).toContain(technique.stix.id); + expect(objectIds).toContain(group.stix.id); + expect(objectIds).toContain(relationship.stix.id); + }); + + it('replays and activates a complete linked pending manifest after interruption', async function () { + const snapshot = await mongoose.connection.db + .collection(trackId) + .findOne({}, { sort: { modified: -1 } }); + await ReleaseTrackGraphManifest.updateOne( + { manifest_id: snapshot.graph_manifest_id }, + { $set: { state: 'pending' } }, + ).exec(); + + await request(app) + .get(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + const manifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: snapshot.graph_manifest_id, + }) + .lean() + .exec(); + expect(manifest.state).toBe('active'); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/release-tracks/ephemeral-bundle.spec.js b/app/tests/api/release-tracks/ephemeral-bundle.spec.js index d411a18e..3a930581 100644 --- a/app/tests/api/release-tracks/ephemeral-bundle.spec.js +++ b/app/tests/api/release-tracks/ephemeral-bundle.spec.js @@ -49,6 +49,8 @@ describe('Ephemeral Bundle API', function () { let icsTechnique; let group; let relationship; + let icsGroup; + let icsRelationship; before(async function () { await database.initializeConnection(); @@ -72,15 +74,19 @@ describe('Ephemeral Bundle API', function () { return res.body; } - async function getEphemeral(query = '', expectedStatus = 200) { + async function getEphemeralForDomain(domain, query = '', expectedStatus = 200) { const res = await request(app) - .get(`/api/release-tracks/ephemeral/enterprise${query}`) + .get(`/api/release-tracks/ephemeral/${domain}${query}`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(expectedStatus); return res.body; } + async function getEphemeral(query = '', expectedStatus = 200) { + return getEphemeralForDomain('enterprise', query, expectedStatus); + } + function buildTechnique(name, domains, overrides = {}) { const timestamp = new Date().toISOString(); return { @@ -179,6 +185,33 @@ describe('Ephemeral Bundle API', function () { object_marking_refs: [staticMarkingDefinitionId], }, }); + + icsGroup = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Ephemeral ICS Test Group', + spec_version: '2.1', + type: 'intrusion-set', + description: 'Group used to verify request-local graph resolution.', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + + icsRelationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: icsGroup.stix.id, + target_ref: icsTechnique.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); }); it('GET /api/release-tracks/ephemeral/:domain returns a STIX 2.1 bundle with legacy-parity contents', async function () { @@ -231,6 +264,37 @@ describe('Ephemeral Bundle API', function () { expect(toc.object_marking_refs).toContain(staticMarkingDefinitionId); }); + it('isolates graph state across concurrent domain exports', async function () { + const requests = Array.from({ length: 6 }, () => + Promise.all([ + getEphemeralForDomain('enterprise', '?includeToc=false'), + getEphemeralForDomain('ics', '?includeToc=false'), + ]), + ); + + for (const [enterpriseBundle, icsBundle] of await Promise.all(requests)) { + const enterpriseIds = bundleObjectIds(enterpriseBundle); + const icsIds = bundleObjectIds(icsBundle); + + expect(enterpriseIds).toContain(group.stix.id); + expect(enterpriseIds).toContain(relationship.stix.id); + expect(enterpriseIds).not.toContain(icsGroup.stix.id); + expect(enterpriseIds).not.toContain(icsRelationship.stix.id); + + expect(icsIds).toContain(icsGroup.stix.id); + expect(icsIds).toContain(icsRelationship.stix.id); + expect(icsIds).not.toContain(group.stix.id); + expect(icsIds).not.toContain(relationship.stix.id); + + expect( + enterpriseBundle.objects.find((object) => object.id === group.stix.id).x_mitre_domains, + ).toEqual([enterpriseDomain]); + expect( + icsBundle.objects.find((object) => object.id === icsGroup.stix.id).x_mitre_domains, + ).toEqual([icsDomain]); + } + }); + it('includeToc=false omits the TOC object', async function () { const bundle = await getEphemeral('?includeToc=false'); const tocObjects = bundle.objects.filter((o) => o.type === 'x-mitre-collection'); diff --git a/app/tests/api/release-tracks/primary-revision-integrity.spec.js b/app/tests/api/release-tracks/primary-revision-integrity.spec.js index 81c53614..4a043001 100644 --- a/app/tests/api/release-tracks/primary-revision-integrity.spec.js +++ b/app/tests/api/release-tracks/primary-revision-integrity.spec.js @@ -14,6 +14,7 @@ const ReleaseTrackRegistry = require('../../../models/release-tracks/release-tra const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const techniquesRepo = require('../../../repository/techniques-repository'); const { DatabaseError } = require('../../../exceptions'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -138,26 +139,6 @@ describe('Release-track primary revision integrity API', function () { expect(latest.candidates[0].object_modified).toBe(technique.stix.modified); }); - it('rejects direct member replacement atomically when one exact revision is missing', async function () { - const technique = await createTechnique('Reject Missing Direct Member'); - const track = await createTrack('Reject Missing Direct Member Track'); - const missing = missingRevision(); - - const response = await api( - 'post', - `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, - { - x_mitre_contents: [ - { obj_ref: technique.stix.id, obj_modified: technique.stix.modified }, - { obj_ref: missing.object_ref, obj_modified: missing.object_modified }, - ], - }, - 400, - ); - expect(response.body.missing_references).toEqual([missing]); - expect((await dynamicRepo.getAllSnapshots(track.id)).pagination.total).toBe(1); - }); - it('fails preview and release when a staged revision was deleted', async function () { const technique = await createTechnique('Deleted Staged Revision'); const track = await createTrack('Deleted Staged Revision Track'); @@ -199,9 +180,7 @@ describe('Release-track primary revision integrity API', function () { it('rejects cloning and export when a stored primary member is missing', async function () { const technique = await createTechnique('Missing Stored Member'); const track = await createTrack('Missing Stored Member Track'); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [technique]); await deleteTechniqueRevision(technique); const registryCount = await ReleaseTrackRegistry.countDocuments(); @@ -233,9 +212,7 @@ describe('Release-track primary revision integrity API', function () { it('propagates repository hydration failures instead of returning a partial export', async function () { const technique = await createTechnique('Failed Primary Hydration'); const track = await createTrack('Failed Primary Hydration Track'); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [technique]); const hydrationStub = sinon .stub(techniquesRepo, 'findManyByIdAndModified') .rejects(new DatabaseError(new Error('injected hydration failure'))); @@ -255,10 +232,7 @@ describe('Release-track primary revision integrity API', function () { it('aborts virtual materialization when a component member is missing', async function () { const technique = await createTechnique('Missing Virtual Component Member'); const component = await createTrack('Missing Virtual Component'); - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { + await releaseExactMembers(app, passportCookie, component.id, [technique], { version: '1.0', }); const virtual = await createTrack('Missing Virtual Primary', 'virtual', { diff --git a/app/tests/api/release-tracks/reconciliation-durability.spec.js b/app/tests/api/release-tracks/reconciliation-durability.spec.js index 52e0792e..ca9ad18d 100644 --- a/app/tests/api/release-tracks/reconciliation-durability.spec.js +++ b/app/tests/api/release-tracks/reconciliation-durability.spec.js @@ -14,6 +14,7 @@ const attackObjectsRepo = require('../../../repository/attack-objects-repository const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const reconciliationService = require('../../../services/release-tracks/reconciliation-service'); const { DatabaseError } = require('../../../exceptions'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -172,9 +173,7 @@ describe('Release-track durable backref reconciliation', function () { { name: 'Full Scan Repair Track', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [technique]); await Technique.updateOne( { diff --git a/app/tests/api/release-tracks/release-track-test-helpers.js b/app/tests/api/release-tracks/release-track-test-helpers.js new file mode 100644 index 00000000..5f28b73d --- /dev/null +++ b/app/tests/api/release-tracks/release-track-test-helpers.js @@ -0,0 +1,49 @@ +'use strict'; + +const request = require('supertest'); + +function exactObjectRef(object) { + if (object.stix) { + return { id: object.stix.id, modified: object.stix.modified }; + } + if (object.object_ref) { + return { id: object.object_ref, modified: object.object_modified }; + } + return { id: object.id, modified: object.modified }; +} + +function authenticated(requestBuilder, passportCookie) { + return requestBuilder + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); +} + +async function stageExactMembers(app, passportCookie, trackId, objects) { + const refs = objects.map(exactObjectRef); + await authenticated( + request(app).post(`/api/release-tracks/${trackId}/candidates`).send({ object_refs: refs }), + passportCookie, + ).expect(200); + + const response = await authenticated( + request(app) + .post(`/api/release-tracks/${trackId}/candidates/promote`) + .send({ object_refs: refs.map((ref) => ref.id) }), + passportCookie, + ).expect(200); + return response.body; +} + +async function releaseExactMembers(app, passportCookie, trackId, objects, releaseBody = {}) { + await stageExactMembers(app, passportCookie, trackId, objects); + const response = await authenticated( + request(app).post(`/api/release-tracks/${trackId}/snapshots/latest/release`).send(releaseBody), + passportCookie, + ).expect(200); + return response.body; +} + +module.exports = { + releaseExactMembers, + stageExactMembers, +}; diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 20361bf8..0b3b54bd 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -5,6 +5,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -297,30 +298,24 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { }); describe('members and snapshots', function () { - it('setting track contents adds member backrefs and reverts on snapshot delete', async function () { + it('deleting the latest draft reverts its candidate backrefs', async function () { const technique = await postObject('/api/techniques', buildTechnique('Backref Contents')); const trackId = await createTrack('Backref Contents Track'); - const contentsSnapshot = await postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }, - 200, - ); + const candidateSnapshot = await addCandidates(trackId, [technique]); let retrieved = await getTechniqueVersion(technique); expect(entryForTrack(retrieved, trackId)).toEqual({ id: trackId, type: 'standard', - tier: 'members', - status: 'reviewed', + tier: 'candidates', + status: 'work-in-progress', }); - // Deleting the latest snapshot reverts membership to the previous + // Deleting the latest snapshot reverts contents to the previous // (empty) snapshot — the backref disappears await request(app) - .delete(`/api/release-tracks/${trackId}/snapshots/${contentsSnapshot.modified}`) + .delete(`/api/release-tracks/${trackId}/snapshots/${candidateSnapshot.modified}`) .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(204); @@ -364,13 +359,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const revisionA = await postObject('/api/techniques', buildTechnique('Backref Member Sync')); const trackId = await createTrack('Backref Member Sync Track'); - await postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], - }, - 200, - ); + await releaseExactMembers(app, passportCookie, trackId, [revisionA]); // Creating a new revision triggers member sync (default strategy: // track_latest) which auto-enrolls the new revision as a candidate @@ -469,13 +458,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { buildTechnique('Backref Dynamic Ignore'), ); const trackId = await createTrack('Backref Dynamic Ignore Track'); - await postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], - }, - 200, - ); + await releaseExactMembers(app, passportCookie, trackId, [revisionA]); await request(app) .put(`/api/release-tracks/${trackId}/config`) .send({ diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index f95977a1..02017e9c 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -11,8 +11,8 @@ * Covered behavior: * - Default bundle contains members only, plus referenced identities and * marking definitions (self-contained bundle) - * - Active relationships whose endpoints are both selected are added - * dynamically; relationships with an endpoint outside the export are not + * - Active relationships and their bounded secondary objects are frozen in + * a snapshot graph manifest * - `include` adds staged and/or candidate tiers (comma-separated or * repeated, singular or plural tier names) * - `state` narrows the included staged/candidate entries by workflow @@ -31,6 +31,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -53,6 +54,8 @@ describe('Release Tracks Bundle Export API', function () { let relationshipSource; let includedRelationship; let excludedRelationship; + let secondaryGroup; + let secondaryRelationship; let linkedAttackId; let linkedAttackUrl; let candidateWip; @@ -195,6 +198,32 @@ describe('Release Tracks Bundle Export API', function () { object_marking_refs: [staticMarkingDefinitionId], }, }); + secondaryGroup = await postObject('/api/groups', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + name: 'Bundle Secondary Group', + description: 'A relationship-discovered secondary object.', + spec_version: '2.1', + type: 'intrusion-set', + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + secondaryRelationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + description: 'Frozen relationship description.', + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: secondaryGroup.stix.id, + target_ref: memberObject.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); const track = await postAction( '/api/release-tracks/new', @@ -216,14 +245,13 @@ describe('Release Tracks Bundle Export API', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); - // Members - await postAction(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { - x_mitre_contents: [ - { obj_ref: memberObject.stix.id, obj_modified: memberObject.stix.modified }, - { obj_ref: linkedMemberObject.stix.id, obj_modified: linkedMemberObject.stix.modified }, - { obj_ref: relationshipSource.stix.id, obj_modified: relationshipSource.stix.modified }, - ], - }); + // Members enter through the supported candidate → staged → release + // lifecycle. + await releaseExactMembers(app, passportCookie, trackId, [ + memberObject, + linkedMemberObject, + relationshipSource, + ]); // Candidates (all start as work-in-progress) await postAction(`/api/release-tracks/${trackId}/candidates`, { @@ -265,6 +293,8 @@ describe('Release Tracks Bundle Export API', function () { const ids = bundleObjectIds(bundle); expect(ids).toContain(memberObject.stix.id); expect(ids).toContain(linkedMemberObject.stix.id); + expect(ids).toContain(secondaryGroup.stix.id); + expect(ids).toContain(secondaryRelationship.stix.id); // Tier entries not selected via include are excluded expect(ids).not.toContain(candidateWip.stix.id); @@ -314,6 +344,7 @@ describe('Release Tracks Bundle Export API', function () { const ids = bundleObjectIds(bundle); expect(ids).toContain(includedRelationship.stix.id); + expect(ids).toContain(secondaryRelationship.stix.id); expect(ids).not.toContain(excludedRelationship.stix.id); const snapshot = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest`); @@ -322,6 +353,107 @@ describe('Release Tracks Bundle Export API', function () { ); }); + it('replays frozen relationship payloads and protects graph dependencies', async function () { + const relationshipUpdate = JSON.parse(JSON.stringify(secondaryRelationship)); + relationshipUpdate.stix.description = 'A later in-place typo correction.'; + relationshipUpdate.stix.external_references = [ + { + source_name: 'deterministic-bundle-test', + description: 'Regression-test relationship source.', + }, + ]; + + await request(app) + .put( + `/api/relationships/${secondaryRelationship.stix.id}/modified/` + + encodeURIComponent(secondaryRelationship.stix.modified), + ) + .send(relationshipUpdate) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, + ); + const frozenRelationship = bundle.objects.find( + (object) => object.id === secondaryRelationship.stix.id, + ); + expect(frozenRelationship.description).toBe('Frozen relationship description.'); + + await request(app) + .delete( + `/api/relationships/${secondaryRelationship.stix.id}/modified/` + + encodeURIComponent(secondaryRelationship.stix.modified), + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + + const secondaryUpdate = JSON.parse(JSON.stringify(secondaryGroup)); + secondaryUpdate.stix.description = 'Attempted in-place graph drift.'; + await request(app) + .put( + `/api/groups/${secondaryGroup.stix.id}/modified/` + + encodeURIComponent(secondaryGroup.stix.modified), + ) + .send(secondaryUpdate) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + + await request(app) + .delete( + `/api/groups/${secondaryGroup.stix.id}/modified/` + + encodeURIComponent(secondaryGroup.stix.modified), + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + }); + + it('protects graph dependencies from collection cascade deletion', async function () { + const timestamp = new Date().toISOString(); + const collection = await postObject('/api/collections', { + workspace: { + imported: timestamp, + import_categories: {}, + workflow: {}, + }, + stix: { + id: `x-mitre-collection--${trackUuid}`, + type: 'x-mitre-collection', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Graph protection cascade fixture', + description: 'Attempts to cascade-delete a protected secondary object.', + x_mitre_version: '1.0', + x_mitre_contents: [ + { + object_ref: secondaryGroup.stix.id, + object_modified: secondaryGroup.stix.modified, + }, + ], + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + + await request(app) + .delete( + `/api/collections/${collection.stix.id}/modified/` + + `${encodeURIComponent(collection.stix.modified)}?deleteAllContents=true`, + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(409); + + await request(app) + .get( + `/api/groups/${secondaryGroup.stix.id}/modified/` + + encodeURIComponent(secondaryGroup.stix.modified), + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + }); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&includeToc=false omits the TOC', async function () { const bundle = await getBundle( `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index 1f313696..246645e6 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -5,6 +5,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -100,13 +101,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { } async function setMembers(trackId, technique) { - return postObject( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], - }, - 200, - ); + return releaseExactMembers(app, passportCookie, trackId, [technique]); } async function latestSnapshotModified(trackId) { diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 5a1c7843..a2d6686f 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -11,6 +11,7 @@ const login = require('../../shared/login'); const releaseHistoryService = require('../../../services/release-tracks/release-history-service'); const versioningService = require('../../../services/release-tracks/versioning-service'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; const virtualObjectRefs = [ @@ -363,14 +364,10 @@ describe('Release-track release planning and commit API', function () { it('records immutable component versions when previewing and releasing a virtual draft', async function () { const member = (await post('/api/techniques', buildTechnique('Provenance Member'), 201)).body; const component = await createTrack('Provenance Component'); - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], - }); - const firstComponentRelease = await post( - `/api/release-tracks/${component.id}/snapshots/latest/release`, - {}, - ); - expect(firstComponentRelease.body.version).toBe('1.0'); + const firstComponentRelease = await releaseExactMembers(app, passportCookie, component.id, [ + member, + ]); + expect(firstComponentRelease.version).toBe('1.0'); const virtual = ( await post( @@ -401,8 +398,8 @@ describe('Release-track release planning and commit API', function () { // Advance the component after materialization. Virtual release provenance // must remain tied to the frozen component resolution, not current state. - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], + await post(`/api/release-tracks/${component.id}/meta`, { + description: 'Component draft created after virtual materialization', }); const secondComponentRelease = await post( `/api/release-tracks/${component.id}/snapshots/latest/release`, @@ -696,10 +693,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Virtual Materialization Member'), 201) ).body; const component = await createTrack('Virtual Materialization Component'); - await post(`/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, { - x_mitre_contents: [{ obj_ref: member.stix.id, obj_modified: member.stix.modified }], - }); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}); + await releaseExactMembers(app, passportCookie, component.id, [member]); const virtual = ( await post( @@ -758,7 +752,7 @@ describe('Release-track release planning and commit API', function () { expect(preview.body.releasable).toBe(true); }); - it('rejects generic contents replacement for virtual tracks', async function () { + it('does not expose generic contents replacement for virtual tracks', async function () { const virtual = await createTrack('Virtual Contents Guard', 'virtual'); const contents = { x_mitre_contents: [ @@ -772,12 +766,12 @@ describe('Release-track release planning and commit API', function () { await post( `/api/release-tracks/${virtual.id}/contents?confirm_track_id=${virtual.id}`, contents, - 400, + 404, ); await post( `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}/contents?confirm_track_id=${virtual.id}`, contents, - 400, + 404, ); const latest = await get(`/api/release-tracks/${virtual.id}/snapshots/latest`); @@ -792,9 +786,7 @@ describe('Release-track release planning and commit API', function () { await post('/api/techniques', buildTechnique('Release Conflict B', revisionA), 201) ).body; const track = await createTrack('Release Conflict'); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: revisionA.stix.id, obj_modified: revisionA.stix.modified }], - }); + await releaseExactMembers(app, passportCookie, track.id, [revisionA]); await post(`/api/release-tracks/${track.id}/candidates`, { object_refs: [{ id: revisionB.stix.id, modified: 'latest' }], }); diff --git a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js index 5096b577..272ff916 100644 --- a/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js +++ b/app/tests/api/release-tracks/release-tracks-tier-invariant.spec.js @@ -7,6 +7,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; const tiers = ['members', 'staged', 'candidates', 'quarantine']; @@ -125,12 +126,7 @@ describe('Release-track cross-tier revision uniqueness', function () { } async function setMembers(trackId, objects) { - return post(`/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, { - x_mitre_contents: objects.map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }); + return releaseExactMembers(app, passportCookie, trackId, objects); } async function useManualMemberSync(trackId) { diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index cf46f412..33079058 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -7,6 +7,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const AttackObject = require('../../../models/attack-object-model'); const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const logger = require('../../../lib/logger'); logger.level = 'debug'; @@ -107,21 +108,7 @@ describe('Release Tracks API', function () { const trackId = createRes.body.id; - await request(app) - .post(`/api/release-tracks/${trackId}/contents`) - .query({ confirm_track_id: trackId }) - .send({ - x_mitre_contents: [ - { - obj_ref: memberObject.stix.id, - obj_modified: memberObject.stix.modified, - }, - ], - }) - .set('Accept', 'application/json') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) - .expect('Content-Type', /json/); + await releaseExactMembers(app, passportCookie, trackId, [memberObject]); await request(app) .post(`/api/release-tracks/${trackId}/candidates`) diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index a89e59ca..1c24567a 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -8,6 +8,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); const backfillMigration = require('../../../../migrations/20260716000000-backfill-release-track-tagged-releases'); +const { stageExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -67,12 +68,12 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const createdA = await createTrack('Releases By Object A'); trackA = createdA.id; const initialSnapshotModified = createdA.modified; - trackATaggedSnapshot = await setMembers(trackA, [objectRevisionA]); - await releaseLatest(trackA); + await setMembers(trackA, [objectRevisionA]); + trackATaggedSnapshot = await releaseLatest(trackA); - // Remove the requested object from the latest state and tag that state. - // The earlier tagged release must remain discoverable despite its current - // backref disappearing. + // Append another object in a later release. Existing members remain part + // of the immutable lineage because direct member replacement is not + // supported. await setMembers(trackA, [otherObject]); await releaseLatest(trackA); @@ -151,16 +152,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { } async function setMembers(trackId, objects) { - return post( - `/api/release-tracks/${trackId}/contents?confirm_track_id=${trackId}`, - { - x_mitre_contents: objects.map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }, - 200, - ); + return stageExactMembers(app, passportCookie, trackId, objects); } async function releaseLatest(trackId, increment = 'minor') { @@ -177,19 +169,30 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const response = await get(`/api/release-tracks/objects/${objectRevisionA.stix.id}/releases`); expect(response.body.object_ref).toBe(objectRevisionA.stix.id); - expect(response.body.pagination).toEqual({ total: 3, limit: 50, offset: 0 }); - expect(response.body.data).toHaveLength(3); + expect(response.body.pagination).toEqual({ total: 4, limit: 50, offset: 0 }); + expect(response.body.data).toHaveLength(4); - const standardA = response.body.data.find((entry) => entry.track_id === trackA); + const standardA = response.body.data.filter((entry) => entry.track_id === trackA); const standardB = response.body.data.find((entry) => entry.track_id === trackB); const virtual = response.body.data.find((entry) => entry.track_id === virtualTrack); - expect(standardA).toMatchObject({ - track_type: 'standard', - track_name: 'Releases By Object A', - version: '1.0', - object_modified: objectRevisionA.stix.modified, - }); + expect(standardA).toHaveLength(2); + expect(standardA).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + track_type: 'standard', + track_name: 'Releases By Object A', + version: '1.0', + object_modified: objectRevisionA.stix.modified, + }), + expect.objectContaining({ + track_type: 'standard', + track_name: 'Releases By Object A', + version: '1.1', + object_modified: objectRevisionA.stix.modified, + }), + ]), + ); expect(standardB).toMatchObject({ track_type: 'standard', version: '1.0', @@ -238,7 +241,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const standard = await get( `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=standard&order=desc&limit=1&offset=1`, ); - expect(standard.body.pagination).toEqual({ total: 2, limit: 1, offset: 1 }); + expect(standard.body.pagination).toEqual({ total: 3, limit: 1, offset: 1 }); expect(standard.body.data).toHaveLength(1); expect(standard.body.data[0].track_type).toBe('standard'); @@ -293,6 +296,6 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const response = await get( `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=standard`, ); - expect(response.body.pagination.total).toBe(2); + expect(response.body.pagination.total).toBe(3); }); }); diff --git a/app/tests/api/release-tracks/snapshot-immutability.spec.js b/app/tests/api/release-tracks/snapshot-immutability.spec.js new file mode 100644 index 00000000..3bd00ee5 --- /dev/null +++ b/app/tests/api/release-tracks/snapshot-immutability.spec.js @@ -0,0 +1,104 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +describe('Release-track snapshot immutability contract', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + it('does not expose direct snapshot metadata or member-replacement routes', async function () { + const track = await post( + '/api/release-tracks/new', + { name: 'Removed snapshot mutation routes', type: 'standard' }, + 201, + ); + const modified = encodeURIComponent(track.modified); + + await api('post', `/api/release-tracks/${track.id}/contents`, {}, 404); + await api('post', `/api/release-tracks/${track.id}/snapshots/${modified}/meta`, {}, 404); + await api('post', `/api/release-tracks/${track.id}/snapshots/${modified}/contents`, {}, 404); + }); + + it('deletes only the latest untagged draft', async function () { + const initial = await post( + '/api/release-tracks/new', + { name: 'Latest draft deletion boundary', type: 'standard' }, + 201, + ); + const middle = await post(`/api/release-tracks/${initial.id}/meta`, { + description: 'Middle draft', + }); + const latest = await post(`/api/release-tracks/${initial.id}/meta`, { + description: 'Latest draft', + }); + + const historicalDelete = await api( + 'delete', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(initial.modified)}`, + undefined, + 409, + ); + expect(historicalDelete.body).toEqual({ + message: 'Only the latest untagged snapshot can be deleted', + snapshot_modified: initial.modified, + latest_snapshot_modified: latest.modified, + }); + + await api( + 'delete', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(latest.modified)}`, + undefined, + 204, + ); + const reverted = await api( + 'get', + `/api/release-tracks/${initial.id}/snapshots/latest`, + undefined, + 200, + ); + expect(reverted.body.modified).toBe(middle.modified); + + await post(`/api/release-tracks/${initial.id}/snapshots/latest/release`, { + version: '1.0', + }); + const taggedDelete = await api( + 'delete', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(middle.modified)}`, + undefined, + 409, + ); + expect(taggedDelete.text).toContain('Tagged snapshot version 1.0 cannot be deleted'); + }); +}); diff --git a/app/tests/api/release-tracks/tagged-content-immutability.spec.js b/app/tests/api/release-tracks/tagged-content-immutability.spec.js index d72c675a..3118c9ab 100644 --- a/app/tests/api/release-tracks/tagged-content-immutability.spec.js +++ b/app/tests/api/release-tracks/tagged-content-immutability.spec.js @@ -8,6 +8,8 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); +const Technique = require('../../../models/technique-model'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -65,23 +67,24 @@ describe('Release-track authoritative tagged-content immutability', function () it('blocks mutation from historical tagged membership when current backrefs are absent', async function () { const technique = await post('/api/techniques', buildTechnique('Historical Member'), 201); - const replacement = await post('/api/techniques', buildTechnique('Current Draft Member'), 201); const track = await post( '/api/release-tracks/new', { name: 'Historical Immutability', type: 'standard' }, 201, ); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: technique.stix.id, obj_modified: technique.stix.modified }], + await releaseExactMembers(app, passportCookie, track.id, [technique], { + version: '1.0', }); - await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '1.0' }); - // A newer draft removes the member, so latest-snapshot reconciliation - // deliberately removes the object's denormalized backref. The historical - // tagged snapshot remains the immutable authority. - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [{ obj_ref: replacement.stix.id, obj_modified: replacement.stix.modified }], - }); + // Simulate a stale derived backref. The tagged snapshot remains the + // immutable authority even when both denormalized indexes are missing. + await Technique.updateOne( + { + 'stix.id': technique.stix.id, + 'stix.modified': new Date(technique.stix.modified), + }, + { $pull: { 'workspace.release_tracks': { id: track.id } } }, + ); const current = ( await api( 'get', diff --git a/app/tests/api/release-tracks/virtual-deduplication.spec.js b/app/tests/api/release-tracks/virtual-deduplication.spec.js index b35b9d27..64b70c76 100644 --- a/app/tests/api/release-tracks/virtual-deduplication.spec.js +++ b/app/tests/api/release-tracks/virtual-deduplication.spec.js @@ -7,6 +7,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -88,17 +89,7 @@ describe('Virtual release-track deduplication API', function () { async function createReleasedComponent(name, members) { const track = await post('/api/release-tracks/new', { name, type: 'standard' }); - await post( - `/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, - { - x_mitre_contents: members.map((member) => ({ - obj_ref: member.stix.id, - obj_modified: member.stix.modified, - })), - }, - 200, - ); - const release = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}, 200); + const release = await releaseExactMembers(app, passportCookie, track.id, members); return { ...track, release }; } diff --git a/app/tests/api/release-tracks/virtual-determinism.spec.js b/app/tests/api/release-tracks/virtual-determinism.spec.js index a58d028b..d2bfd1a7 100644 --- a/app/tests/api/release-tracks/virtual-determinism.spec.js +++ b/app/tests/api/release-tracks/virtual-determinism.spec.js @@ -9,6 +9,7 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const modelFactory = require('../../../models/release-tracks/model-factory'); const login = require('../../shared/login'); const { cloneForCreate } = require('../../shared/clone-for-create'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -81,19 +82,9 @@ describe('Virtual release-track deterministic membership API', function () { name, type: 'standard', }); - const contents = await post( - `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, - { - x_mitre_contents: [ - { - obj_ref: member.stix.id, - obj_modified: modified, - }, - ], - }, - 200, - ); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}, 200); + const contents = await releaseExactMembers(app, passportCookie, component.id, [ + { id: member.stix.id, modified }, + ]); return { component, contents }; } diff --git a/app/tests/api/release-tracks/virtual-domain-filters.spec.js b/app/tests/api/release-tracks/virtual-domain-filters.spec.js index ae38757d..98e3c984 100644 --- a/app/tests/api/release-tracks/virtual-domain-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-domain-filters.spec.js @@ -6,6 +6,7 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const { cloneForCreate } = require('../../shared/clone-for-create'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -113,17 +114,13 @@ describe('Virtual Release Track Domain Filters API', function () { name: 'Domain Filter Component', type: 'standard', }); - await post( - `/api/release-tracks/${component.id}/contents?confirm_track_id=${component.id}`, - { - x_mitre_contents: [enterprise, ics, shared, noDomain, enterpriseMatrix].map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }, - 200, - ); - await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, {}, 200); + await releaseExactMembers(app, passportCookie, component.id, [ + enterprise, + ics, + shared, + noDomain, + enterpriseMatrix, + ]); // A newer revision has a different domain, but virtual composition must // evaluate the exact revision pinned in the tagged component snapshot. diff --git a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js index c580c436..9353d49e 100644 --- a/app/tests/api/release-tracks/virtual-object-type-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-object-type-filters.spec.js @@ -14,6 +14,7 @@ const { compositionSchema, } = require('../../../models/release-tracks/release-track-snapshot-schema'); const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; const supportedObjectTypes = Object.values(types); @@ -202,17 +203,7 @@ describe('Virtual release-track object-type filters API', function () { const mitigation = await post('/api/mitigations', buildMitigation('Pinned Type Member')); const matrix = await post('/api/matrices', buildMatrix('Excluded Type Member')); - await post( - `/api/release-tracks/${componentTrack.id}/contents?confirm_track_id=${componentTrack.id}`, - { - x_mitre_contents: [mitigation, matrix].map((object) => ({ - obj_ref: object.stix.id, - obj_modified: object.stix.modified, - })), - }, - 200, - ); - await post(`/api/release-tracks/${componentTrack.id}/snapshots/latest/release`, {}, 200); + await releaseExactMembers(app, passportCookie, componentTrack.id, [mitigation, matrix]); const newerMitigationRevision = cloneForCreate(mitigation); newerMitigationRevision.stix.modified = new Date(Date.now() + 1000).toISOString(); diff --git a/app/tests/api/release-tracks/virtual-quarantine.spec.js b/app/tests/api/release-tracks/virtual-quarantine.spec.js index d2e0a24d..f63e6c0c 100644 --- a/app/tests/api/release-tracks/virtual-quarantine.spec.js +++ b/app/tests/api/release-tracks/virtual-quarantine.spec.js @@ -7,6 +7,7 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; @@ -73,15 +74,7 @@ describe('Virtual release-track quarantine API', function () { async function createReleasedComponent(name, member) { const track = await createTrack(name); - await post(`/api/release-tracks/${track.id}/contents?confirm_track_id=${track.id}`, { - x_mitre_contents: [ - { - obj_ref: member.stix.id, - obj_modified: member.stix.modified, - }, - ], - }); - await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}); + await releaseExactMembers(app, passportCookie, track.id, [member]); return track; } diff --git a/app/tests/api/reports/reports.spec.js b/app/tests/api/reports/reports.spec.js index e3e696be..ae4085c4 100644 --- a/app/tests/api/reports/reports.spec.js +++ b/app/tests/api/reports/reports.spec.js @@ -4,6 +4,7 @@ const { expect } = require('expect'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const AttackObject = require('../../../models/attack-object-model'); +const Technique = require('../../../models/technique-model'); const config = require('../../../config/config'); const login = require('../../shared/login'); @@ -70,6 +71,20 @@ describe('Reports API', function () { // Check for a valid database configuration await databaseConfiguration.checkSystemConfiguration(); + const targetTimestamp = new Date(); + await Technique.create({ + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + id: targetRef2, + created: targetTimestamp, + modified: targetTimestamp, + name: 'Report relationship target', + x_mitre_is_subtechnique: false, + }, + }); + // Enable ADM validation; the request payloads in this spec are ADM-compliant config.validateRequests.withAttackDataModel = true; config.validateRequests.withOpenApi = true; diff --git a/docs/README.md b/docs/README.md index b4e1ee03..b4e64a7e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -60,7 +60,8 @@ Configuration, deployment, and identity provider setup. - [Automation Run Audit Trail](admin/automation-runs.md): How to inspect migration and scheduler audit records - [Virtual Track Schedules](admin/virtual-track-schedules.md): UTC execution, restart recovery, retries, and observability - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures -- [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator member replacements and track deletions +- [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator track-deletion attempts +- [Release-Track Deterministic Graph Migration](admin/release-track-graph-migration.md): Preview and operate the relationship-pin and snapshot-manifest backfill ### Authentication diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md index 548e7a68..c9634eb7 100644 --- a/docs/admin/release-track-audit.md +++ b/docs/admin/release-track-audit.md @@ -1,7 +1,7 @@ # Release-Track Destructive Audit Events -Workbench stores administrator-initiated member replacement and full-track -deletion attempts in `releaseTrackAuditEvents`. +Workbench stores administrator-initiated full-track deletion attempts in +`releaseTrackAuditEvents`. Each record contains: @@ -41,10 +41,8 @@ db.releaseTrackAuditEvents ``` A `pending` event can mean the process stopped after the audit insert or the -operation completed but the final audit update failed. Inspect the target -track before retrying. A failed member replacement may also have persisted a -new snapshot if backref reconciliation subsequently failed; correlate its -timestamp with `releaseTrackReconciliations`. +track was deleted but the final audit update failed. Confirm whether the track +still exists before retrying. These records have no automatic TTL. Establish retention and archive policy according to local audit requirements. diff --git a/docs/admin/release-track-graph-migration.md b/docs/admin/release-track-graph-migration.md new file mode 100644 index 00000000..227e562f --- /dev/null +++ b/docs/admin/release-track-graph-migration.md @@ -0,0 +1,53 @@ +# Release-Track Deterministic Graph Migration + +Release-track snapshot bundles depend on exact relationship endpoints and a +frozen snapshot graph manifest. The +`20260730180000-backfill-deterministic-snapshot-graphs` migration establishes +that data for an existing Workbench database. + +## Before deployment + +Run the read-only preview against the target database: + +```bash +DATABASE_URL='mongodb://host/database' \ + npm run preview:deterministic-snapshot-graphs +``` + +The report includes the latest relationship revisions scanned, endpoint pins +that would be written, release-track snapshots found, and baseline manifests +that would be created. No database writes or indexes are created by this +command. + +The preview fails if a latest relationship references a source or target +object that no longer exists. Repair those dangling endpoints before +deployment. Snapshot graph capture fails closed rather than silently producing +an incomplete deterministic baseline. + +## What the migration writes + +- Exact source and target revision metadata is added only to the latest + revision of each relationship in the underlying `relationships` collection. + `view.relationships.latest` may be used for discovery but is never written. +- Each existing release-track snapshot receives a graph manifest containing + its exact primary, relationship, secondary, supporting, and LinkById + dependencies. +- Backfilled manifests are marked `baseline_reconstruction: true`. They + reproduce the graph visible at migration time; the server cannot infer the + historically exact graph of snapshots created before endpoint pins existed. + +The migration is rerunnable. A complete manifest already linked to a snapshot +is reused, and a linked pending manifest left by an interrupted activation is +activated instead of duplicated. + +## Deployment behavior + +With `WB_REST_DATABASE_MIGRATION_ENABLE=true`, the migration runs during +normal server startup. If migrations are managed separately, run the standard +`migrate-mongo` workflow after reviewing the preview and before accepting +release-track traffic. + +After deployment, smoke-test one standard and one virtual snapshot with +`format=bundle`. Editing or hard-deleting a frozen secondary revision should +return `409 Conflict`; creating a new revision remains the supported update +path. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index c46b2791..53920a37 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -21,8 +21,8 @@ Keep these rules in mind while updating the connector: namespace. Snapshot retrieval and release operations are shared. - New virtual-only operations include `/virtual/` in the path. - The current OpenAPI document is authoritative. Some older standard-only - workflow routes, such as `/candidates`, `/staged`, and `/contents`, predate - the namespace convention and do not currently include `/standard/`. + workflow routes, such as `/candidates` and `/staged`, predate the namespace + convention and do not currently include `/standard/`. - A release preview is a read-only `GET`. A release commit is a `POST`. ## P0 — Model draft revision selectors separately from released member pins @@ -93,9 +93,8 @@ interface MissingPrimaryRevisions { ``` - HTTP `400` means the current request selected a revision that does not - exist. Candidate add/version-update and direct standard member replacement - flows should keep the dialog open, identify the missing selections, and let - the operator correct them. + exist. Candidate add/version-update flows should keep the dialog open, + identify the missing selections, and let the operator correct them. - HTTP `409` means an existing draft or snapshot contains a dangling primary reference. Snapshot retrieval, release preview/commit, cloning, virtual materialization/quarantine promotion, and bundle export can return this @@ -109,13 +108,56 @@ Done when: - The release-track connector exposes `missing_references` on `400` and `409` responses instead of flattening the response to a generic message. -- Candidate and direct-content forms keep their input state after a `400` and - highlight the missing revisions. +- Candidate forms keep their input state after a `400` and highlight the + missing revisions. - Snapshot, release, clone, virtual-materialization, and export views present an actionable integrity error for `409`. - Tests cover multiple missing references and prove no partial snapshot or bundle is rendered. +### [ ] Explain snapshot-graph protection conflicts on object edits and deletes + +Release-track snapshots now freeze the exact relationships and secondary +objects needed to reproduce their bundle graph. If an object revision is a +protected dependency of any active or linked-pending snapshot manifest, an +in-place `PUT`, exact-revision `DELETE`, or full-lineage `DELETE` that would +invalidate that graph returns +`409 Conflict`: + +```ts +{ + message: string; + details?: string; + snapshot_graph_pins: Array<{ + track_id: string; + snapshot_modified: string; + kind: 'root' | 'relationship' | 'secondary' | 'supporting' | 'link_target'; + tier?: 'members' | 'staged' | 'candidates' | 'quarantine'; + }>; +} +``` + +This can occur from ordinary object-management screens, not only from the +release-track UI. Present it as a versioning constraint: the operator should +create a new object revision, or remove the draft snapshots that no longer +need the old revision. Do not offer a force-delete path; administrator +authorization does not bypass graph integrity. + +A standalone standard-track candidate or staged root remains editable through +the existing in-place review workflow. It becomes graph-protected only when +the same revision is also needed as a frozen dependency. Description-only +relationship corrections are allowed because older snapshots retain the +relationship payload captured in their manifests; relationship source, +target, and type changes are rejected as graph changes. + +Done when: + +- Shared object edit/delete error handling recognizes + `snapshot_graph_pins`. +- The message identifies the affected release track(s) and recommends a new + revision instead of a blind retry. +- The UI does not expose a force-delete action for graph-protected revisions. + ### [ ] Handle persisted mutations whose membership reconciliation failed A release-track mutation can persist its snapshot before a downstream object @@ -145,23 +187,71 @@ Done when: ## P0 — Align the Angular connector with the current routes -### [ ] Add administrator confirmation for destructive release-track actions +### [x] Remove direct snapshot mutation controls and client methods + +Persisted snapshot history is now immutable. The backend no longer exposes: + +```text +POST /api/release-tracks/:id/contents +POST /api/release-tracks/:id/snapshots/:modified/contents +POST /api/release-tracks/:id/snapshots/:modified/meta +``` -Full track deletion and both direct standard-track member replacement routes -are administrator-only. They now require the query parameter +Remove the corresponding connector methods, payload types, dialogs, buttons, +and tests. Standard-track content should move through candidates, staged, and +release. Virtual content should move through composition materialization and +quarantine resolution. Metadata can be changed only from the latest snapshot +via `POST /api/release-tracks/:id/meta`, which creates a new draft. + +Do not replace removed historical-edit actions with hidden calls or local +state edits. If an operator wants a different result, they should correct the +latest draft, delete it while deletion is still allowed, or create a newer +draft. + +Done when: + +- No Angular code calls or models any of the three removed routes. +- Snapshot history views are read-only except for supported release, clone, + and latest-draft deletion actions. +- Standard and virtual editors direct users to their respective supported + workflows. + +Completed 2026-07-30: the Angular connector methods, payload type, and +regression fixtures were removed. No component or menu called these methods, +so no UI control needed to be migrated. + +### [ ] Offer deletion only for the latest untagged draft + +`DELETE /api/release-tracks/:id/snapshots/:modified` is a narrow “undo latest +draft” operation. The server accepts it only when the selected snapshot is both +untagged and currently latest. Tagged releases and older drafts return `409` +because they are immutable history. + +In snapshot history, show Delete only on the latest item when `version == null`. +After a successful delete, refresh both the latest snapshot and the history; +the preceding snapshot becomes current. If a `409` occurs because another +operation created a newer draft, refresh instead of retrying the stale delete. + +Done when: + +- Tagged and historical rows never offer Delete. +- The confirmation explains that the track will revert to the preceding + snapshot. +- A stale `409` refreshes the view and preserves history. + +### [ ] Add administrator confirmation for full track deletion + +Full track deletion is administrator-only and requires the query parameter `confirm_track_id` to exactly equal the `:id` path parameter: ```text DELETE /api/release-tracks/:id?confirm_track_id=:id -POST /api/release-tracks/:id/contents?confirm_track_id=:id -POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id ``` -Do not expose these actions to editors or team leads. Before sending a request, -show the track name and ID, explain that direct replacement bypasses the normal -candidate/staged workflow or that deletion removes all history, and require an -explicit confirmation interaction. A missing or stale ID returns `400`; a -non-administrator returns `401`. +Do not expose this action to editors or team leads. Before sending the request, +show the track name and ID, explain that deletion removes all history, and +require an explicit confirmation interaction. A missing or stale ID returns +`400`; a non-administrator returns `401`. Done when: @@ -427,21 +517,11 @@ Done when: - A `409` from preview/release explains that materialization is required instead of being swallowed as a null preview. -### [ ] Keep standard-only mutations out of virtual-track controls - -Direct contents replacement is now explicitly rejected for virtual tracks: - -```text -POST /api/release-tracks/:id/contents -POST /api/release-tracks/:id/snapshots/:modified/contents -``` +### [ ] Keep standard workflow controls out of virtual tracks Virtual membership has one authority: composition materialization followed by -optional quarantine resolution. Both contents endpoints return `400 Bad -Request` for a virtual track. - -Hide direct member replacement, candidate, and staged controls when -`type === 'virtual'`. Keep them available for standard tracks on their current +optional quarantine resolution. Hide candidate and staged controls when +`type === 'virtual'`; keep them available for standard tracks on their current routes. Done when: @@ -661,9 +741,9 @@ not mean “resolve every member to its latest object revision.” If the track not acquired another snapshot, repeated `/snapshots/latest` calls identify the same primary revision set. -Bundle downloads remain a documented exception: the backend appends secondary -relationships and supporting objects at request time, so the complete -`format=bundle` graph is not guaranteed to reproduce an earlier download. +Bundle downloads now replay the relationship/secondary graph captured when +the snapshot was created. The generated bundle-envelope ID may change, but +the object graph for a materialized virtual snapshot is stable. Done when: @@ -673,8 +753,8 @@ Done when: `object_modified`. - Tests prove that advancing a component after materialization does not change the displayed virtual member revision. -- User-facing export guidance does not promise byte-identical bundle - regeneration. +- User-facing export guidance distinguishes a stable snapshot object graph + from the intentionally variable bundle-envelope UUID. ## P1 — Submit mode-correct virtual snapshot schedules @@ -880,9 +960,14 @@ Minimum regression coverage: The following changes are useful context but should not create extra connector work: -- Snapshot bundle exports now include valid secondary relationships - dynamically. Existing bundle download code receives a more complete bundle - without changing its request. +- Snapshot bundle exports now include bounded secondary objects and their + relationships from a frozen graph manifest. Existing bundle download code + receives a more complete and reproducible bundle without changing its + request. A standard draft tier explicitly stored as `"latest"` remains + dynamic until release. +- Snapshot responses include an opaque, server-controlled + `graph_manifest_id`. The SPA does not need to send, interpret, or persist + this field; tolerate it in response models and omit it from request bodies. - Release-track object back-references are reconciled when snapshots change. Frontend object refreshes will see the updated membership metadata without a new endpoint. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 04d21c2a..1ad59401 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -7,23 +7,54 @@ This branch implements the prioritized findings in recommendation is kept as a separate conventional commit so the merge request can be reviewed or reverted item by item. +### Current implementation slice — Immutable snapshot history + +- [x] Remove direct latest and historical snapshot member-replacement + endpoints (`POST /:id/contents` and + `POST /:id/snapshots/:modified/contents`) from routes, controllers, + services, validation, and OpenAPI. +- [x] Remove historical metadata rewriting + (`POST /:id/snapshots/:modified/meta`); retain latest metadata updates, + which create a new draft snapshot. +- [x] Permit snapshot deletion only for the latest untagged draft and return a + typed `409 Conflict` for tagged or historical snapshots. +- [x] Replace test setup that depended on direct member replacement with + supported bootstrap/candidate/promotion/release workflows, and add + regressions for the removed endpoints and deletion boundary. +- [x] Update user/developer/admin documentation and the Angular, Python, and + Bruno clients so no downstream surface suggests that persisted snapshot + history can be rewritten. +- [x] Run focused server/client checks, then the complete server `npm test` + suite, and record the results. + +Verification (2026-07-30): + +- Server: focused release-track regression files, OpenAPI validation, lint, + and the complete `npm test` suite passed. +- Angular: connector regression tests, changed-file formatting/lint checks, + and the complete frontend test suite passed. +- `internalattack`: release-track regression tests, changed-file Ruff checks, + and the complete Python test suite passed. +- Bruno: the release-track collection no longer contains the removed mutation + requests, and its modified request files pass scoped whitespace validation. + ### P0.1 — Enforce release version uniqueness - [x] Add a unique partial index for tagged `version` strings in every dynamic - release-track snapshot collection. + release-track snapshot collection. - [x] Convert duplicate-version races into a typed `409 Conflict` that - identifies the track and requested version. + identifies the track and requested version. - [x] Add a regression that releases two distinct drafts concurrently with the - same version and proves exactly one succeeds. + same version and proves exactly one succeeds. - [x] Adopt the pre-release reset policy for collections created with the - former non-unique index. No shared deployment retains beta release-track - data, so this change deliberately does not establish a permanent - migration contract for local development state. + former non-unique index. No shared deployment retains beta release-track + data, so this change deliberately does not establish a permanent + migration contract for local development state. - [x] Update release-version documentation and run focused, middleware, and - lint verification. + lint verification. - [ ] Obtain one clean aggregate `npm test` run for the migration cleanup. Three - attempts exposed the repository's roaming cross-spec isolation failure; - every affected spec passed immediately in isolation. + attempts exposed the repository's roaming cross-spec isolation failure; + every affected spec passed immediately in isolation. Original implementation verification (2026-07-30): @@ -47,22 +78,23 @@ Pre-release migration cleanup verification (2026-07-30): ### P0.2 — Make primary release membership fail closed - [x] Add one shared batch hydrator that resolves dynamic selectors, validates - every exact `(object_ref, object_modified)` pair, and reports all missing - primary revisions without swallowing repository failures. -- [x] Reject nonexistent exact candidate pins, candidate pin updates, direct - member replacement, track cloning, and virtual materialization before - snapshot persistence. + every exact `(object_ref, object_modified)` pair, and reports all missing + primary revisions without swallowing repository failures. +- [x] Reject nonexistent exact candidate pins, candidate pin updates, the + then-supported direct member replacement requests, track cloning, and + virtual materialization before snapshot persistence. Direct replacement + was subsequently removed by the immutable-history slice. - [x] Revalidate existing and promoted members at release-preview and - release-commit boundaries; return a typed `409 Conflict` for corrupt stored - drafts. + release-commit boundaries; return a typed `409 Conflict` for corrupt stored + drafts. - [x] Abort bundle import before creating a track when any authoritative - primary object failed to import or cannot be hydrated. + primary object failed to import or cannot be hydrated. - [x] Abort bundle/workbench export when selected primary revisions cannot be - hydrated; return every missing reference instead of a partial result. + hydrated; return every missing reference instead of a partial result. - [x] Add ingress, partial-import, deleted-staged-revision, virtual - materialization, and incomplete-export regressions. + materialization, and incomplete-export regressions. - [x] Update user/developer documentation and run focused, lint, OpenAPI, and - complete-suite verification. + complete-suite verification. Verification result (2026-07-30): @@ -80,17 +112,17 @@ Verification result (2026-07-30): ### P0.3 — Make tagged-content immutability authoritative and durable - [x] Guard object revision update/delete and delete-all by querying tagged - snapshot membership, even when `workspace.release_tracks` is missing or - stale. + snapshot membership, even when `workspace.release_tracks` is missing or + stale. - [x] Make release-track backref reconciliation failures propagate to the - triggering request so a release is never reported as fully successful when - protection writes failed. + triggering request so a release is never reported as fully successful when + protection writes failed. - [x] Persist every reconciliation attempt and its terminal outcome so - process crashes and partial listener failures remain operator-visible. + process crashes and partial listener failures remain operator-visible. - [x] Provide an idempotent repair command for failed/pending reconciliation - records and a full-scan mode for legacy drift. + records and a full-scan mode for legacy drift. - [x] Add failure-injection, missing-backref, repair, and historical-release - regressions; update user/developer/admin documentation. + regressions; update user/developer/admin documentation. - [x] Run focused, lint, OpenAPI, and complete-suite verification. Verification result (2026-07-30): @@ -111,17 +143,21 @@ Verification result (2026-07-30): ### P0.4 — Correct destructive authorization and add durable audit records +This records the earlier beta contract. The immutable-history slice later +removed both member-replacement routes and their audit action types; durable +auditing now applies only to full-track deletion. + - [x] Require administrator authorization for full track deletion and both - direct member-replacement routes. + direct member-replacement routes. - [x] Require an exact `confirm_track_id` precondition on each destructive - request so stale or accidental UI actions fail before persistence. + request so stale or accidental UI actions fail before persistence. - [x] Persist a durable, actor-attributed audit event before each operation - and record completion or failure without hiding partial persistence. + and record completion or failure without hiding partial persistence. - [x] Add an authorization matrix and operator-facing audit documentation. - [x] Update OpenAPI, frontend tasks, and Bruno requests for the confirmation - contract. + contract. - [x] Add admin/editor, missing/mismatched confirmation, success/failure - audit, lint, OpenAPI, focused, and complete-suite verification. + audit, lint, OpenAPI, focused, and complete-suite verification. Verification result (2026-07-30): @@ -136,27 +172,119 @@ Verification result (2026-07-30): - [ ] P0.5 — Complete the Angular contract migration and end-to-end smoke gate. - [ ] P0.6 — Finish scheduled-materialization fencing, retry bounds, and - operator intervention. + operator intervention. - [ ] P0.7 — Establish and enforce a safe storage operating envelope. - [ ] P0.8 — Harden deployment, database readiness, backup/restore, rollback, - and post-deploy verification. + and post-deploy verification. - [ ] Address P1 recommendations in documented criticality order. +## Current implementation slice — Deterministic snapshot bundle graphs + +This slice replaces export-time relationship and secondary-object discovery +with a frozen graph manifest for every persisted release-track snapshot. The +manifest is authoritative for bundle replay and for protecting every exact +revision on which that bundle depends. + +### Shared graph resolution + +- [x] Extract the bounded ATT&CK graph-selection rules from the mutable + `stix-bundles-service` singleton into a request-local resolver. +- [x] Preserve the existing one-hop and named special-case behavior for + `detects`, `attributed-to`, `revoked-by`, detection strategies, analytics, + and required supporting objects without introducing unrestricted graph + traversal. +- [x] Make the legacy/ephemeral exporter and release-track snapshot capture + use thin adapters around the same resolver. +- [x] Add parity and concurrent-request regression coverage before changing + release-track persistence. + +### Revision-pinned relationships + +- [x] Store server-controlled exact source and target revision pins under + `workspace.relationship_endpoints`; do not add non-ADM fields to emitted + STIX payloads. +- [x] Resolve endpoint revisions when relationships are created, including + bundle-import and automated relationship-creation paths, and fail closed + when an exact endpoint cannot be established. +- [x] Create new SRO revisions when a referenced SDO advances instead of + mutating an existing `(stix.id, stix.modified)` revision. +- [x] Reject in-place source, target, and relationship-type changes. + Description-only corrections remain allowed because manifests freeze the + relationship STIX payload used by existing snapshots. + +### Snapshot graph manifests + +- [x] Persist an exact, tier-aware manifest for every newly created standard + and virtual snapshot. Include primary roots, relationships, secondary + objects, identities, marking definitions, and LinkById render dependencies. +- [x] Store manifest entries in one indexed collection so exact revision + hydration and mutation-protection checks do not scan dynamic snapshot + collections. +- [x] Make snapshot capture fail closed and concurrency-safe. Complete + manifests are linked before activation; linked pending manifests remain + replayable and self-activate after an interrupted write. +- [x] Replace a standard draft's manifest from the resolved release plan in + the same conditional tag update, so staged `"latest"` selectors become + exact released members without creating a second snapshot timestamp. +- [x] Replay `format=bundle` entirely from the frozen manifest, with no live + relationship, secondary, supporting-object, or LinkById discovery. The + deliberate exception is an explicitly included standard draft tier whose + stored selector is `"latest"`; that tier remains dynamic until release. + +### Mutation and deletion protection + +- [x] Centralize exact graph-pin checks in the shared object service layer. +- [x] Return `409 Conflict` when a PUT or exact-revision delete would mutate + an emitted revision referenced by any active or pending manifest. +- [x] Reject lineage deletion when any revision in the lineage is manifest + referenced; administrator authorization must not bypass graph integrity. +- [x] Remove protection entries when their owning snapshot or track is + deleted, with idempotent reconciliation for interrupted cleanup. + +### Migration, documentation, and verification + +- [x] Add a dry-run-capable, idempotent migration that resolves endpoint pins + for the latest revision of each relationship directly from the underlying + collections. Do not write through `view.relationships.latest`. +- [x] Backfill existing snapshots with manifests reconstructed from the graph + visible at migration time and label them as baseline reconstructions rather + than historically exact captures. +- [x] Add regression coverage for newer SDO/SRO revisions, relationship + deprecation, missing dependencies, PUT/delete guards, standard and virtual + snapshots, query-tier filtering, migration reruns, and concurrent capture. +- [x] Update OpenAPI error contracts, user/developer/admin documentation, + frontend guidance, `internalattack`, and Bruno where the observable contract + changes. No request route or parameter changed, so generated clients and + Bruno request definitions require no transport change. +- [x] Run focused specs while iterating, then lint, OpenAPI validation, and the + complete `npm test` suite. +- [x] Propose conventional commits split by independently reviewable + architectural slice; do not commit until requested. + +Verification result (2026-07-30): + +- The focused deterministic-graph group passes all 101 cases; the final + compatibility group for relationship pagination, reports, and collection + imports passes all 27 cases. +- Lint, formatting, OpenAPI validation, and diff checks pass. +- The required clean full suite passes: OpenAPI 2, config 21, API 971, + middleware 29, and scheduler 10. + ## Current implementation slice — Scheduler regression and virtual schedule coverage - [x] Repair the legacy collection-index scheduler spec so it imports the - refactored `sync-collection-indexes-task` module without auto-registering - background jobs during the test. + refactored `sync-collection-indexes-task` module without auto-registering + background jobs during the test. - [x] Add virtual-track coverage proving reconciliation registers scheduled - cron jobs in UTC and removes jobs for tracks that no longer exist. + cron jobs in UTC and removes jobs for tracks that no longer exist. - [x] Add date-schedule boundary coverage for multiple due dates and future - dates. + dates. - [x] Add crash-window recovery coverage for a scheduled virtual snapshot that - was persisted before its occurrence ledger reached `completed`. + was persisted before its occurrence ledger reached `completed`. - [x] Add stale-claim recovery coverage and document any remaining - multi-process lease/fencing limitation. + multi-process lease/fencing limitation. - [x] Run the legacy scheduler spec, the virtual scheduler spec, the aggregate - scheduler suite, lint, and the complete `npm test` suite. + scheduler suite, lint, and the complete `npm test` suite. - [x] Record the coverage conclusion and propose a conventional commit message. Coverage conclusion (2026-07-30): @@ -183,36 +311,36 @@ Verification result (2026-07-30): ### Remaining scheduled-materialization hardening - [ ] Add an owner token (fencing token) to occurrence claims, make terminal - updates conditional on the active token, and renew leases for work that may - exceed the claim duration. Add a true multi-worker regression proving that - an expired worker cannot overwrite the succeeding worker's result. + updates conditional on the active token, and renew leases for work that may + exceed the claim duration. Add a true multi-worker regression proving that + an expired worker cannot overwrite the succeeding worker's result. - [ ] Decide and document an operator policy for permanent failures. If - indefinite one-minute retries are not acceptable, add bounded exponential - backoff plus a terminal/dead-letter state and operator-visible recovery - controls. + indefinite one-minute retries are not acceptable, add bounded exponential + backoff plus a terminal/dead-letter state and operator-visible recovery + controls. ## Current implementation slice — Deterministic standard releases - [x] Preserve `modified: "latest"` and omitted candidate selectors as dynamic - references through the candidate and staged tiers; preserve explicit - timestamps as exact revision pins. + references through the candidate and staged tiers; preserve explicit + timestamps as exact revision pins. - [x] Resolve every dynamic staged reference to the actual latest - `stix.modified` timestamp during standard release planning, before conflict - detection, preview rendering, or commit. + `stix.modified` timestamp during standard release planning, before conflict + detection, preview rendering, or commit. - [x] Ensure tagged members contain exact revisions only and that preview and - commit use the same release-planning rules. + commit use the same release-planning rules. - [x] Make dynamic candidate/staged references safe in tier comparison, - Workbench enrichment, bundle rendering, back-reference reconciliation, and - member-sync paths. + Workbench enrichment, bundle rendering, back-reference reconciliation, and + member-sync paths. - [x] Add regression coverage for dynamic and explicit candidate promotion, - release-time resolution after a newer revision is created, historical - release targeting, conflict handling, and member immutability. + release-time resolution after a newer revision is created, historical + release targeting, conflict handling, and member immutability. - [x] Update OpenAPI, user/developer documentation, frontend guidance, - `internalattack`, and Bruno as required by the corrected contract. + `internalattack`, and Bruno as required by the corrected contract. - [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` - suite. + suite. - [x] Apply logic review, inspect the final diff, and propose conventional - commit messages. + commit messages. Verification result (2026-07-30): @@ -240,78 +368,78 @@ completion backlog. ### P1 — Composition validation and deterministic resolution - [x] Make request validation strict so misspelled keys such as - `filters.domain` return 400 instead of silently disabling filtering. + `filters.domain` return 400 instead of silently disabling filtering. - [x] Validate component selectors according to `resolution_strategy`: - `specific_version` requires `version` and rejects `snapshot`; - `specific_snapshot` requires `snapshot` and rejects `version`; - `latest_tagged` rejects both selector fields. - [x] Make `priority` consistently required in Zod, Mongoose, OpenAPI, docs, - and examples; reject duplicate priorities at the request boundary. + and examples; reject duplicate priorities at the request boundary. - [x] Validate component existence, standard-track type, duplicate track IDs, - and duplicate priorities when a virtual track is initially created, not only - when composition is later updated or materialized. + and duplicate priorities when a virtual track is initially created, not only + when composition is later updated or materialized. - [x] Validate `snapshot_schedule` by mode: - `manual` rejects `cron` and `dates`; - `cron` requires `cron` and rejects `dates`; - `dates` requires at least one date and rejects `cron`. - [x] Constrain or document accepted `filters.object_types` values and add - direct regression coverage for exact-revision filtering. + direct regression coverage for exact-revision filtering. ### P1 — Deduplication correctness - [x] Treat the same exact object revision contributed by multiple components - as one duplicate, not a conflicting revision. + as one duplicate, not a conflicting revision. - [x] Ensure the `quarantine` strategy only quarantines genuinely different - revisions of the same object. + revisions of the same object. - [x] Attribute each surviving revision to one deterministic component so - `objects_contributed` totals cannot exceed `summary.total_objects`. + `objects_contributed` totals cannot exceed `summary.total_objects`. - [x] Add dedicated tests for all four strategies: - `prioritize_latest_object`, `prioritize_latest_snapshot`, - `prioritize_higher_priority`, and `quarantine`. + `prioritize_latest_object`, `prioritize_latest_snapshot`, + `prioritize_higher_priority`, and `quarantine`. ### P1 — Release provenance - [x] Populate virtual release `version_history[].component_versions` from the - materialized snapshot's immutable `composition_resolution`. + materialized snapshot's immutable `composition_resolution`. - [x] Define and test the provenance shape in Mongoose, OpenAPI, and user and - developer documentation. + developer documentation. ### P2 — Scheduled materialization - [x] Connect virtual `snapshot_schedule` metadata to the existing task - scheduler. This is required for virtual-track completion, not an optional - future enhancement. + scheduler. This is required for virtual-track completion, not an optional + future enhancement. - [x] Implement `cron` execution so each matching schedule occurrence - materializes a new virtual draft through the same lifecycle and validation - used by `POST /api/release-tracks/:id/virtual/snapshots/create`. + materializes a new virtual draft through the same lifecycle and validation + used by `POST /api/release-tracks/:id/virtual/snapshots/create`. - [x] Implement `dates` execution so every configured timestamp materializes - exactly one virtual draft, including deterministic handling for restart - recovery, missed timestamps, and duplicate-delivery prevention. + exactly one virtual draft, including deterministic handling for restart + recovery, missed timestamps, and duplicate-delivery prevention. - [x] Preserve `manual` semantics: store no executable schedule and create - drafts only through the explicit virtual snapshot-creation endpoint. + drafts only through the explicit virtual snapshot-creation endpoint. - [x] Define failure behavior when a component has no matching tagged - snapshot, including automation-run audit records and retry policy. + snapshot, including automation-run audit records and retry policy. - [x] Add scheduler integration tests for both `cron` and `dates`, including - successful execution, restart recovery, idempotency, component-resolution - failure, and retry behavior. + successful execution, restart recovery, idempotency, component-resolution + failure, and retry behavior. - [x] Add operational documentation covering scheduler activation, UTC - interpretation, observability, failures, and retries. + interpretation, observability, failures, and retries. ### Current implementation slice — Scheduled virtual materialization - [x] Add a scheduler reconciliation task for persisted virtual-track - `cron` and `dates` schedules while preserving explicit-only `manual` mode. + `cron` and `dates` schedules while preserving explicit-only `manual` mode. - [x] Persist schedule occurrences and claim them atomically so multiple - scheduler instances cannot concurrently process the same occurrence. + scheduler instances cannot concurrently process the same occurrence. - [x] Make snapshot persistence idempotent by recording the scheduled - occurrence on the resulting virtual draft. + occurrence on the resulting virtual draft. - [x] Recover missed `dates` occurrences and failed `cron` or `dates` - occurrences during reconciliation. + occurrences during reconciliation. - [x] Record every materialization attempt in the automation-run audit trail. - [x] Add scheduler integration coverage for success, restart recovery, - duplicate delivery, component failure, and retry. + duplicate delivery, component failure, and retry. - [x] Update OpenAPI, user/developer/operations documentation, frontend - guidance, and Bruno. + guidance, and Bruno. - [x] Run focused scheduler tests, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -346,40 +474,40 @@ Verification result (2026-07-29): ### P2 — Contract decisions - [x] Virtual tracks cannot compose virtual tracks. Components must be - standard tracks; revisit nesting only if a concrete future use case requires - it. + standard tracks; revisit nesting only if a concrete future use case requires + it. - [x] Do not implement the documented native-members/hybrid model. Virtual - tracks are purely compositional; content that is not already represented - belongs in a dedicated standard component track. + tracks are purely compositional; content that is not already represented + belongs in a dedicated standard component track. - [x] Do not implement `resolve=true` or `resolved_content`. Virtual - composition is resolved eagerly into exact object revisions when a draft is - materialized; retrieval must never re-resolve a persisted snapshot. + composition is resolved eagerly into exact object revisions when a draft is + materialized; retrieval must never re-resolve a persisted snapshot. - [x] Do not implement caching or component-release notifications without - measured scale or an approved operator workflow. Persisted snapshots already - avoid composition recomputation, and no notification recipient, channel, or - expected action has been defined. + measured scale or an approved operator workflow. Persisted snapshots already + avoid composition recomputation, and no notification recipient, channel, or + expected action has been defined. ### Current implementation slice — Deterministic virtual membership - [x] Resolve the `latest` request shorthand to the actual latest - `stix.modified` value before standard-track contents are persisted. + `stix.modified` value before standard-track contents are persisted. - [x] Defensively lock any unresolved component member to an exact revision - during virtual materialization, while preserving exact revisions already - frozen into tagged component snapshots. + during virtual materialization, while preserving exact revisions already + frozen into tagged component snapshots. - [x] Add regression coverage proving that component `track_latest` behavior - cannot move a materialized virtual member and repeated snapshot retrieval - returns the same exact revision set. + cannot move a materialized virtual member and repeated snapshot retrieval + returns the same exact revision set. - [x] Remove `resolve=true` and `resolved_content` from the documented - retrieval contract. -- [x] Clearly document that persisted primary member revisions are - deterministic while bundle-time secondary-object and relationship - expansion is not. + retrieval contract. +- [x] Initially document the distinction between deterministic primary + membership and the then-dynamic bundle graph; the later deterministic graph + manifest slice below supersedes that accepted limitation. - [x] Update OpenAPI, frontend guidance, and Bruno where the clarified - contract affects consumers. + contract affects consumers. - [x] Run focused tests, lint, OpenAPI validation, and the complete `npm test` - suite. + suite. - [x] Apply logic review, inspect the final diff, and propose conventional - commit messages. + commit messages. Verification result (2026-07-29): @@ -411,38 +539,38 @@ Verification result (2026-07-29): ```text docs(release-tracks): clarify snapshot determinism - Document exact virtual member pins and the bundle-time secondary-content - consistency boundary in the Bruno collection. + Document exact virtual member pins and the snapshot graph consistency + boundary in the Bruno collection. ``` ### Future architecture — Deterministic bundle graphs -- [ ] Design version-controlled STIX Relationship Objects whose source and - target references identify exact `(object_id, object_modified)` revisions - rather than an entire STIX object provenance chain. -- [ ] Evaluate cloning every affected SRO when a new SDO revision is created, - including atomicity, fan-out, concurrency, migration, and rollback behavior. -- [ ] Measure the resulting database-storage amplification and query/index - costs before approving implementation. -- [ ] Define and persist an export manifest that pins every secondary object, - supporting object, and relationship revision required to reproduce a bundle. -- [ ] Until that architecture is approved and implemented, preserve and - prominently document the accepted constraint that `format=bundle` output is - not graph- or byte-level deterministic. +- [x] Design version-controlled STIX Relationship Objects whose source and + target references identify exact `(object_id, object_modified)` revisions + rather than an entire STIX object provenance chain. +- [x] Evaluate cloning every affected SRO when a new SDO revision is created, + including atomicity, fan-out, concurrency, migration, and rollback behavior. +- [x] Avoid cloning SROs per snapshot by recording exact endpoint metadata on + each SRO revision and storing compact manifest references plus a frozen SRO + payload where description-only PUT compatibility requires it. +- [x] Define and persist an export manifest that pins every secondary object, + supporting object, and relationship revision required to reproduce a bundle. +- [x] Document the resulting guarantee: the emitted object graph is + deterministic, while the generated bundle-envelope UUID is not byte-stable. ### Current implementation slice — Pure standard-track composition - [x] Make standard component tracks a positive service-layer requirement, - preserving rejection during both virtual-track creation and composition - replacement. + preserving rejection during both virtual-track creation and composition + replacement. - [x] Reject unsupported top-level creation properties such as - `native_members` instead of silently stripping them. + `native_members` instead of silently stripping them. - [x] Add regression coverage for virtual-track nesting on both creation and - composition update, and for attempted native-member creation. + composition update, and for attempted native-member creation. - [x] Remove nesting and hybrid/native-member claims from OpenAPI, user and - developer documentation, frontend guidance, and Bruno. + developer documentation, frontend guidance, and Bruno. - [x] Run the focused virtual-composition spec, lint, and complete `npm test` - suite. + suite. - [x] Review the final diff and propose conventional commit messages. Verification result (2026-07-29): @@ -478,46 +606,46 @@ Verification result (2026-07-29): ### Documentation corrections - [ ] Replace `stix.type = "virtual"` with the top-level snapshot - `type: "virtual"`. + `type: "virtual"`. - [ ] Remove the nonexistent snapshot-level `snapshot_id`; retain - `version_history[].snapshot_id`. + `version_history[].snapshot_id`. - [ ] Correct response envelopes and the virtual-create response example. - [ ] Align `composition_resolution` examples with fields actually generated, - or implement the documented `by_type`, `by_tier`, and native statistics. + or implement the documented `by_type`, `by_tier`, and native statistics. - [ ] Align documented error envelopes with centralized error-handler output. - [x] Include required `priority` values in every composition example. - [x] Clearly distinguish configured composition from a materialized draft and - document scheduler activation, timing, recovery, and retry behavior. + document scheduler activation, timing, recovery, and retry behavior. ### Verified complete - [x] Composition changes invalidate inherited materialized contents and - require explicit rematerialization before release. + require explicit rematerialization before release. - [x] Generic contents replacement rejects virtual tracks. - [x] Exact-revision quarantine resolution is available at - `POST /api/release-tracks/:id/virtual/quarantine/promote`. + `POST /api/release-tracks/:id/virtual/quarantine/promote`. - [x] `filters.domains` hydrates and evaluates exact pinned revisions. - [x] Public domain names and STIX `*-attack` names are normalized. - [x] Multiple domain values are supported. - [x] Objects without domain metadata are excluded when a domain filter is set. - [x] Primary Enterprise, ICS, and Mobile matrices use their ATT&CK external ID - as the established domain fallback. + as the established domain fallback. - [x] Virtual tracks resolve only tagged snapshots and consume only component - `members`. + `members`. - [x] Virtual tracks maintain independent draft/release history and use the - shared snapshot retrieval and release endpoints after materialization. + shared snapshot retrieval and release endpoints after materialization. ### Current implementation slice — Strict composition contracts - [x] Add API regression coverage for unknown composition/filter keys on both - virtual-track creation and composition update. + virtual-track creation and composition update. - [x] Require the selector appropriate to each `resolution_strategy` and - reject selectors that do not apply to that strategy. + reject selectors that do not apply to that strategy. - [x] Make the composition, component, filter, and deduplication request - objects strict without changing persisted response shapes. + objects strict without changing persisted response shapes. - [x] Update OpenAPI, user/developer documentation, and Bruno examples. - [x] Run the focused regression spec, then lint and the complete `npm test` - suite. + suite. - [x] Review the final diff and propose a conventional commit message. Verification result (2026-07-29): @@ -542,15 +670,15 @@ Verification result (2026-07-29): ### Current implementation slice — Component identity and priority validation - [x] Add creation and composition-update regression coverage for required - priorities, duplicate priorities, and duplicate component track IDs. + priorities, duplicate priorities, and duplicate component track IDs. - [x] Reject missing component tracks and virtual component tracks before an - initial virtual track is persisted. + initial virtual track is persisted. - [x] Make component priority required and non-negative across Zod, Mongoose, - OpenAPI, user/developer documentation, and Bruno examples. + OpenAPI, user/developer documentation, and Bruno examples. - [x] Keep service-layer component validation as a defense for non-HTTP - callers while moving deterministic duplicates to request validation. + callers while moving deterministic duplicates to request validation. - [x] Run the focused regression specs, then lint and the complete `npm test` - suite. + suite. - [x] Review the final diff and propose a conventional commit message. Verification result (2026-07-29): @@ -585,17 +713,17 @@ Verification result (2026-07-29): ### Current implementation slice — Snapshot schedule contracts - [x] Add creation regressions for valid and invalid `manual`, `cron`, and - `dates` schedule payloads. + `dates` schedule payloads. - [x] Enforce a strict mode-discriminated request contract: - `manual` accepts only `mode`; - `cron` requires `cron` and rejects `dates`; - `dates` requires at least one date and rejects `cron`. - [x] Reject `snapshot_schedule` on standard-track creation instead of silently - dropping it. + dropping it. - [x] Repeat schedule invariants at the service and Mongoose boundaries for - non-HTTP callers. + non-HTTP callers. - [x] Align OpenAPI, user/developer documentation, frontend guidance, the - `internalattack` test fixture, and Bruno. + `internalattack` test fixture, and Bruno. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -638,16 +766,16 @@ Verification result (2026-07-29): ### Current implementation slice — Object-type filter contracts - [x] Define `filters.object_types` against the canonical Workbench STIX type - vocabulary instead of accepting arbitrary strings. + vocabulary instead of accepting arbitrary strings. - [x] Reject empty arrays, duplicate values, malformed values, and unsupported - object types on both virtual-track creation and composition update. + object types on both virtual-track creation and composition update. - [x] Repeat the accepted-value constraint at the Mongoose persistence - boundary. + boundary. - [x] Add direct materialization coverage proving that object-type filtering - preserves the exact revision pinned by the tagged component snapshot rather - than resolving the latest database revision. + preserves the exact revision pinned by the tagged component snapshot rather + than resolving the latest database revision. - [x] Align OpenAPI, user/developer documentation, frontend guidance, and - Bruno; verify whether `internalattack` needs a typed client change. + Bruno; verify whether `internalattack` needs a typed client change. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -686,23 +814,23 @@ Verification result (2026-07-29): ### Current implementation slice — Deterministic virtual deduplication - [x] Add materialization regressions for all four deduplication strategies - using both an exact revision shared by multiple components and genuinely - different revisions of the same STIX object. + using both an exact revision shared by multiple components and genuinely + different revisions of the same STIX object. - [x] Collapse repeated contributions of the same `(object_ref, - object_modified)` revision before applying conflict resolution. +object_modified)` revision before applying conflict resolution. - [x] Count an object contributed by multiple components once in - `duplicates_found`, but include it in `conflicts_resolved` only when multiple - distinct revisions remain after exact-revision collapse. + `duplicates_found`, but include it in `conflicts_resolved` only when multiple + distinct revisions remain after exact-revision collapse. - [x] Choose one deterministic source component for every surviving revision: - use the active strategy's ordering and use component priority as the stable - tie-breaker. + use the active strategy's ordering and use component priority as the stable + tie-breaker. - [x] Quarantine one entry per distinct conflicting revision and leave an - identical revision shared by multiple components in `members`. + identical revision shared by multiple components in `members`. - [x] Derive `objects_contributed` from explicit survivor attribution so its - component total equals `summary.total_objects`. + component total equals `summary.total_objects`. - [x] Align OpenAPI, user/developer documentation, frontend guidance, Bruno, - and `internalattack` if the clarified response semantics require downstream - changes. + and `internalattack` if the clarified response semantics require downstream + changes. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Review the final diff and propose conventional commit messages. @@ -743,20 +871,20 @@ Verification result (2026-07-29): ### Current implementation slice — Virtual release provenance - [x] Add release preview and commit regressions proving that virtual - `version_history[].component_versions` comes from the selected draft's - immutable `composition_resolution`, even if a component is released again - before the virtual draft is tagged. + `version_history[].component_versions` comes from the selected draft's + immutable `composition_resolution`, even if a component is released again + before the virtual draft is tagged. - [x] Define `component_versions` as an optional object keyed by immutable - component track ID with tagged `MAJOR.MINOR` version values. + component track ID with tagged `MAJOR.MINOR` version values. - [x] Populate provenance only for virtual release history entries and leave - standard release history unchanged. + standard release history unchanged. - [x] Enforce the provenance value shape at the Mongoose persistence boundary - and describe it in OpenAPI. + and describe it in OpenAPI. - [x] Align user/developer documentation, frontend guidance, Bruno, and - `internalattack` if the response contract requires downstream changes. + `internalattack` if the response contract requires downstream changes. - [x] Run focused regression specs, lint, and the complete `npm test` suite. - [x] Apply logic and performance review checklists, inspect the final diff, - and propose conventional commit messages. + and propose conventional commit messages. Verification result (2026-07-29): @@ -800,19 +928,19 @@ Verification result (2026-07-29): - [x] Consolidate the virtual-track completion backlog into this section. - [x] Preserve completed implementation evidence in the dated records below. - [x] Move the downstream Angular handoff to - `docs/developer/FRONTEND_TODO.md`. + `docs/developer/FRONTEND_TODO.md`. - [x] Remove the superseded root-level tracker files. ## Document downstream frontend work - [x] Inventory the current release-track API contract and recent endpoint, - terminology, lifecycle, validation, and response-shape changes. + terminology, lifecycle, validation, and response-shape changes. - [x] Inspect the Angular release-track consumers so the handoff identifies - concrete downstream work instead of restating backend implementation notes. + concrete downstream work instead of restating backend implementation notes. - [x] Create `docs/developer/FRONTEND_TODO.md` with task-oriented guidance, - contextual explanations, and acceptance criteria. + contextual explanations, and acceptance criteria. - [x] Cross-check the handoff against OpenAPI, user/developer documentation, - Bruno, and the `internalattack` client. + Bruno, and the `internalattack` client. - [x] Review formatting and the final diff. Verification result (2026-07-29): @@ -833,13 +961,13 @@ Verification result (2026-07-29): ## Implement virtual quarantine resolution - [x] Add end-to-end regression coverage for exact-revision quarantine - promotion, snapshot immutability, back-reference reconciliation, validation, - and virtual-track type enforcement. + promotion, snapshot immutability, back-reference reconciliation, validation, + and virtual-track type enforcement. - [x] Add `POST /api/release-tracks/:id/virtual/quarantine/promote`. - [x] Promote the selected revision to members in a new draft and remove all - quarantined alternatives for the same object. + quarantined alternatives for the same object. - [x] Preserve the immutable composition-resolution record and historical - materialized snapshot. + materialized snapshot. - [x] Update OpenAPI, user/developer documentation, and Bruno. - [x] Run focused regression specs, then lint and the complete `npm test` suite. - [x] Review the final diff and propose a conventional commit message. @@ -868,9 +996,9 @@ Verification result (2026-07-29): ## Harden virtual materialization lifecycle - [x] Record the complete virtual-track audit in the dedicated virtual release - tracks section of this file. + tracks section of this file. - [x] Add regression coverage for stale composition state, unmaterialized - release attempts, and virtual use of standard contents endpoints. + release attempts, and virtual use of standard contents endpoints. - [x] Clear inherited materialized state when virtual composition changes. - [x] Require a materialized virtual draft for release preview and commit. - [x] Restrict generic contents replacement to standard tracks. @@ -896,15 +1024,15 @@ Verification result (2026-07-29): ## Consolidate virtual draft creation and shared release previews - [x] Move virtual-only composition and draft-creation operations under an - explicit `/virtual` capability namespace. + explicit `/virtual` capability namespace. - [x] Remove the standalone virtual snapshot-preview endpoint without an - alias. + alias. - [x] Enhance shared virtual release summaries to compare the persisted draft - with its preceding tagged release without recomputing composition. + with its preceding tagged release without recomputing composition. - [x] Add regression coverage for route removal, type enforcement, latest and - historical virtual previews, and release-preview non-persistence. + historical virtual previews, and release-preview non-persistence. - [x] Update OpenAPI, user/developer documentation, Bruno, and the - `internalattack` Python client. + `internalattack` Python client. - [x] Run focused regression specs, then the complete `npm test` suite. - [x] Review the final diff and propose a conventional commit message. @@ -923,32 +1051,32 @@ Verification result (2026-07-29): ## Bootstrap faster-release core, defense, and virtual tracks - [x] Reconcile the clarified ownership partition with the current release-track - and virtual-composition API. + and virtual-composition API. - [x] Add regression coverage for functional virtual domain filters and - relationship-complete snapshot bundle exports. + relationship-complete snapshot bundle exports. - [x] Implement virtual `filters.domains` using the established ATT&CK domain - inference rules. + inference rules. - [x] Reuse/extract existing bundle relationship logic so snapshot - `format=bundle` exports dynamically include valid secondary relationships. + `format=bundle` exports dynamically include valid secondary relationships. - [x] Inventory and report any additional release-track no-op placeholders. - [x] Update user/developer docs and OpenAPI for the effective contract change; - Bruno has no new or changed request parameter to mirror. + Bruno has no new or changed request parameter to mirror. - [x] Run focused release-track regression specs, then the complete `npm test` - suite. + suite. - [x] Scan all three ATT&CK v19.1 bundles and construct a disjoint exact-revision - partition for Enterprise Core, ICS Core, Mobile Core, and Defense. + partition for Enterprise Core, ICS Core, Mobile Core, and Defense. - [x] Assign the shared identity and marking definitions to Enterprise Core - using the representations supported by release-track snapshots. + using the representations supported by release-track snapshots. - [x] Preflight exact track names and refuse conflicting duplicate tracks. - [x] Create and verify the four v19.1-pinned standard tracks. - [x] Create and verify the three domain-filtered virtual track definitions. - [x] Verify that every in-scope v19.1 object is owned by exactly one standard - track and record intentional relationship/collection exclusions. CTI owns - `course-of-action`; ICS Core owns `x-mitre-asset`. + track and record intentional relationship/collection exclusions. CTI owns + `course-of-action`; ICS Core owns `x-mitre-asset`. - [x] Defer materializing virtual snapshots until the component standard tracks - have tagged releases; no release/tag action was authorized in this bootstrap. + have tagged releases; no release/tag action was authorized in this bootstrap. - [x] Review the final repository diff and propose a conventional commit - message. + message. Operational result (2026-07-28): @@ -982,11 +1110,11 @@ Operational result (2026-07-28): - [x] Scan the ATT&CK v19.1 ICS and Mobile bundles and report every object type. - [x] Preflight the production-mirroring Workbench API and existing tracks. - [x] Create the CTI standard track with the latest intrusion-set, malware, - tool, and campaign revisions as members. + tool, and campaign revisions as members. - [x] Verify the persisted CTI snapshot, object-type coverage, exact latest - revision pins, and counts. + revision pins, and counts. - [x] Record operational results and propose a conventional commit message for - the committable scratchpad update. + the committable scratchpad update. Operational result (2026-07-28): @@ -1002,66 +1130,66 @@ Operational result (2026-07-28): ## Harden release version selection - [x] Reject simultaneous `increment` and `version` selectors inside the - release planner, even when controller validation is bypassed. + release planner, even when controller validation is bypassed. - [x] Add regression coverage for planner-level mutual exclusivity. - [x] Make exact, incremental, default, and ambiguous selection behavior - explicit in OpenAPI, user/developer docs, and Bruno. + explicit in OpenAPI, user/developer docs, and Bruno. - [x] Run the focused release-track spec, lint, and complete `npm test` suite. - The focused release spec passes (10 tests), lint passes, and the complete - backend suite passes (OpenAPI: 2, config: 21, API: 907, middleware: 24). - Targeted frontend Prettier and ESLint pass; TypeScript remains blocked by - the checkout's existing Angular dependency-resolution and unrelated type - errors. + The focused release spec passes (10 tests), lint passes, and the complete + backend suite passes (OpenAPI: 2, config: 21, API: 907, middleware: 24). + Targeted frontend Prettier and ESLint pass; TypeScript remains blocked by + the checkout's existing Angular dependency-resolution and unrelated type + errors. - [x] Review the final diff and propose a conventional commit message. ## Release command and unified previews - [x] Replace bump routes and symbols with explicit release operations for - latest and historical snapshots. + latest and historical snapshots. - [x] Implement one pure release planner shared by summary, workbench, bundle, - and commit paths. + and commit paths. - [x] Remove `dry_run`, rename version `type` to `increment`, and reject - conflicting version-selection inputs. + conflicting version-selection inputs. - [x] Keep release targeting semantics explicit: `latest` resolves at request - time, while `:modified` pins a specific snapshot; no client precondition is - required. + time, while `:modified` pins a specific snapshot; no client precondition is + required. - [x] Add regression coverage for preview parity, non-persistence, conflicts, - formats, validation, historical releases, and removed bump routes. + formats, validation, historical releases, and removed bump routes. - [x] Update OpenAPI, user/developer documentation, Bruno, and frontend - consumers. + consumers. - [x] Run focused tests and frontend checks, then the complete `npm test` - backend suite. - Focused release-track suites pass (49 tests), and the affected backref suite - passes again in isolation (23 tests). The complete backend suite passes on - retry. Targeted frontend formatting and ESLint pass; frontend Vitest and - TypeScript startup remain blocked by the checkout's existing - ESM/dependency-resolution errors. + backend suite. + Focused release-track suites pass (49 tests), and the affected backref suite + passes again in isolation (23 tests). The complete backend suite passes on + retry. Targeted frontend formatting and ESLint pass; frontend Vitest and + TypeScript startup remain blocked by the checkout's existing + ESM/dependency-resolution errors. - [x] Review the final diff and propose a conventional commit message. ## Remove implicit latest-snapshot route - [x] Remove `GET /api/release-tracks/:id` while preserving track deletion. - [x] Make `/snapshots/latest` canonical across OpenAPI, tests, docs, Bruno, - and the frontend consumer. + and the frontend consumer. - [x] Add regression coverage proving the removed method returns 405. - [x] Run focused regression specs followed by the complete `npm test` suite. - The focused suites pass. The aggregate run reached 894 passing with three - unrelated documented roaming failures; all three affected specs pass - together in isolation (51 passing). + The focused suites pass. The aggregate run reached 894 passing with three + unrelated documented roaming failures; all three affected specs pass + together in isolation (51 passing). - [x] Review the final diff and propose a conventional commit message. ## Snapshot history collection endpoint - [x] Add `GET /api/release-tracks/:id/snapshots` with strict tagged filtering - and pagination, plus an explicit `/snapshots/latest` alias. + and pagination, plus an explicit `/snapshots/latest` alias. - [x] Return lightweight, type-oriented summaries: standard snapshots include - member/staged/candidate counts; virtual snapshots include member/quarantine - counts. + member/staged/candidate counts; virtual snapshots include member/quarantine + counts. - [x] Add regression coverage for defaults, filters, pagination, validation, - track types, and not-found behavior. + track types, and not-found behavior. - [x] Update OpenAPI, user/developer documentation, and Bruno requests. - [x] Run the focused regression spec followed by the complete `npm test` - suite. + suite. - [x] Review the final diff and propose a conventional commit message. ## Regression Tests @@ -1069,28 +1197,27 @@ Operational result (2026-07-28): - [ ] Implement regression tests - [x] **Investigate the recurring full-suite flake.** Two root causes found and fixed (2026-07-10) in `app/lib/database-in-memory.js`: - 1. *Port collision*: every spec file stopped and restarted the `mongodb-memory-server` instance, and a fresh mongod would intermittently fail with `Port already in use` — breaking that file's `before` hook (surfacing as `loginAnonymous` 404s) and cascading failures through the file. Fixed by reusing one mongod for all spec files in the process (`closeConnection` drops the database and disconnects but keeps the server running) plus `--exit` on the mocha scripts. - 2. *Vanishing unique indexes*: dropping the database between spec files also drops its indexes, and mongoose's per-model `init()` is memoized per process — so the `stix.id + stix.modified` unique index was intermittently missing for later files, letting duplicate-POST tests (and dependent count tests) fail in roaming pairs. Fixed by explicitly awaiting `createIndexes()` for all registered models after each reconnect. + 1. _Port collision_: every spec file stopped and restarted the `mongodb-memory-server` instance, and a fresh mongod would intermittently fail with `Port already in use` — breaking that file's `before` hook (surfacing as `loginAnonymous` 404s) and cascading failures through the file. Fixed by reusing one mongod for all spec files in the process (`closeConnection` drops the database and disconnects but keeps the server running) plus `--exit` on the mocha scripts. + 2. _Vanishing unique indexes_: dropping the database between spec files also drops its indexes, and mongoose's per-model `init()` is memoized per process — so the `stix.id + stix.modified` unique index was intermittently missing for later files, letting duplicate-POST tests (and dependent count tests) fail in roaming pairs. Fixed by explicitly awaiting `createIndexes()` for all registered models after each reconnect. Residual: rare (≈1 per run under heavy machine load) single-test failures of a different character (a count assertion, a 20s timeout in a pagination GET) still appear occasionally and pass in isolation — likely load-related; keep observing before chasing further. ## Release-track cross-tier revision uniqueness - [x] Read the release-track user and developer documentation and identify the - intended exact-revision invariant. + intended exact-revision invariant. - [x] Trace every standard/virtual tier ingress and transition path. - [x] Add regression coverage proving one `(stix.id, stix.modified)` revision - cannot occupy multiple tiers while different revisions of one ID can. + cannot occupy multiple tiers while different revisions of one ID can. - [x] Enforce the invariant for candidate adds, promotions, demotions, bulk - status transitions, release bumps, member sync, and quarantine workflows. + status transitions, release bumps, member sync, and quarantine workflows. - [x] Update user/developer documentation (and OpenAPI/Bruno only if the API - contract changes). + contract changes). - [x] Run focused specs and the complete `npm test` suite. The task-specific - and constituent suites pass; repeated aggregate runs each encountered one - unrelated roaming API failure that passed immediately in isolation. + and constituent suites pass; repeated aggregate runs each encountered one + unrelated roaming API failure that passed immediately in isolation. - [x] Review the final diff and propose a conventional commit message. - ## Snapshot Output Format **TASK Summary**: Implement support for the `bundle` output format for snapshots @@ -1117,8 +1244,8 @@ The following release-track snapshot retrieval endpoints support `include` and > [!Note] > The ephemeral bundle endpoint (`GET /api/release-tracks/ephemeral/{domain}`) supports `format`, but not tier `include`, because it does not read from a persisted release-track snapshot. Rather, it "blindly" includes all objects in the domain. - **Include Parameter** (controls which tiers are returned): + ``` GET /api/release-tracks/:id/snapshots/latest # Default: all tiers GET /api/release-tracks/:id/snapshots/latest?include=members # Members tier only @@ -1129,6 +1256,7 @@ GET /api/release-tracks/:id/snapshots/latest?include=all # All ti ``` **Format Parameter** (controls output format): + ``` GET /api/release-tracks/:id/snapshots/latest?format=workbench # Workbench snapshot with metadata (default) GET /api/release-tracks/:id/snapshots/latest?format=bundle # Standard STIX 2.1 bundle @@ -1136,6 +1264,7 @@ GET /api/release-tracks/:id/snapshots/latest?format=filesystemstore # Not im ``` **Combined Example:** + ``` GET /api/release-tracks/:id/snapshots/latest?include=all&format=workbench ``` @@ -1168,7 +1297,7 @@ Here is how each of the other query parameters should be handled/mapped to the n For release track retrieval requests that include the `format=bundle` query parameter, the following query parameters must be supported: - `include: ['candidate', 'staged']`: If specified, the value must be equal to an array of at least one value. The parameter acts as a filter, allowing users to specify whether release-track candidates and/or staged objects should be included in the bundle. If the `include` parameter is omitted, only members should be included. -- `state: ['work-in-progress', 'awaiting-review']`: If specified, the value must be equal to an array of at least one value. Notably, objects marked as `"reviewed"` are always included (by nature of all members being included —— all members are inherently "reviewed"), irrespective of this query parameter. The parameter acts as a union filter that logically combines with `include`. In other words, when `include` and `state` are both specified, `include` is applied first, then `state` is applied to the remaining `include`-filtered subset. (i.e., Of the candidates and/or staged objects that are ready to be included in the emitted bundle, only include the ones that are marked as "work-in-progress", "awaiting-review", or either). +- `state: ['work-in-progress', 'awaiting-review']`: If specified, the value must be equal to an array of at least one value. Notably, objects marked as `"reviewed"` are always included (by nature of all members being included —— all members are inherently "reviewed"), irrespective of this query parameter. The parameter acts as a union filter that logically combines with `include`. In other words, when `include` and `state` are both specified, `include` is applied first, then `state` is applied to the remaining `include`-filtered subset. (i.e., Of the candidates and/or staged objects that are ready to be included in the emitted bundle, only include the ones that are marked as "work-in-progress", "awaiting-review", or either). - `stixVersion` should be **preserved**. This parameter allows users to control which STIX version is used in the emitted bundle (`2.0` or `2.1`). It defaults to `2.1`. ### Fixing the /bump/preview endpoint @@ -1213,10 +1342,10 @@ For example: ```yaml workspace: - release_tracks: - - id: 'release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d' - phase: 'candidate' - status: 'work-in-progress' + release_tracks: + - id: 'release-track--3a0e2537-1153-4b16-8ff5-1993f2d9cd7d' + phase: 'candidate' + status: 'work-in-progress' stix: # ... ``` @@ -1228,20 +1357,20 @@ Object CRUD paths can mutate or destroy revisions that release tracks pin, witho - [x] **Reject revision re-keying on PUT.** `updateFull` merged body `stix.id`/`stix.modified` over the stored document, so a PUT could silently re-key a revision and strand any track pins. Now returns 400 when the body identity fields differ from the path parameters. Re-keying must go through POST (a new revision), which member sync captures. Tests: `app/tests/api/base-services/update-identity-guard.spec.js`. -- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the `::updated` → revision-sync path and are marked with the server-assigned **`modified-in-place`** status (content changed with no revision history to diff — reviewers are told *that* something changed, not *what*; the marker is cleared via the review endpoint). Placement is centralized in the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`): tier is decided against `candidacy_threshold`/`auto_promote` (`modified-in-place` ranks with `work-in-progress`), so permissive tracks keep in-place-edited staged entries staged while strict tracks demote them for re-review — and threshold-qualifying placements land directly in `staged` in a single snapshot (no more candidates bounce). Covers in-place deprecation (`x_mitre_deprecated` via PUT). The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Future: an in-document changelog of in-place modifications would let the marker say *what* changed. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. +- [x] **Capture in-place PUTs of pinned revisions.** Implemented 2026-07-13: `BaseService.updateFull` rejects (409, `MemberPinnedRevisionError`) when the revision is pinned in any track's `members` tier — released content is immutable in place; POST a new revision instead. `staged`/`candidates`-pinned revisions ride the `::updated` → revision-sync path and are marked with the server-assigned **`modified-in-place`** status (content changed with no revision history to diff — reviewers are told _that_ something changed, not _what_; the marker is cleared via the review endpoint). Placement is centralized in the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`): tier is decided against `candidacy_threshold`/`auto_promote` (`modified-in-place` ranks with `work-in-progress`), so permissive tracks keep in-place-edited staged entries staged while strict tracks demote them for re-review — and threshold-qualifying placements land directly in `staged` in a single snapshot (no more candidates bounce). Covers in-place deprecation (`x_mitre_deprecated` via PUT). The member-sync misfire (same-key duplicate cross-tier enrollment) is fixed by skipping enrollment of already-pinned revisions and skipping no-op snapshot clones. Future: an in-document changelog of in-place modifications would let the marker say _what_ changed. Tests: `app/tests/api/release-tracks/release-tracks-change-capture.spec.js`. -- [x] **DELETE of tracked objects.** Implemented 2026-07-13 with a simplified decision: DELETE (single version or all versions) is *rejected* (409) when a revision is `members`-pinned, with guidance to retire the object via a new `x_mitre_deprecated` revision instead — members-pinned revisions are immutable and must never be deleted. (The earlier auto-convert-to-deprecation idea was dropped in favor of explicit rejection.) `candidates`/`staged`-pinned deletes remain allowed (the reconciler self-heals the dangling pin). Note: `CollectionsService` overrides `deleteVersionById`, so collections are not covered by the guard. Legacy delete controllers were migrated to the service-exception middleware (`next(err)`) so the 409 maps correctly. +- [x] **DELETE of tracked objects.** Implemented 2026-07-13 with a simplified decision: DELETE (single version or all versions) is _rejected_ (409) when a revision is `members`-pinned, with guidance to retire the object via a new `x_mitre_deprecated` revision instead — members-pinned revisions are immutable and must never be deleted. (The earlier auto-convert-to-deprecation idea was dropped in favor of explicit rejection.) `candidates`/`staged`-pinned deletes remain allowed unless the same revision is also a frozen graph dependency. The deterministic-graph slice extended the guard to secondary/supporting revisions and to `CollectionsService`'s custom exact, lineage, and `deleteAllContents` paths. Legacy delete controllers use the service-exception middleware (`next(err)`) so the 409 maps correctly. -- [x] **Revoke must reach member sync.** Implemented 2026-07-13: member sync subscribes to the 11 per-type `::revoked` events via a payload adapter (`handleStixObjectRevokedEvent`), so the revoked revision (`revoked: true`) is enrolled as a candidate in member tracks and candidate/staged pins move to it — treated exactly like any new revision. The revoke response's primary document carries the resulting backrefs. As decided, member sync is NOT extended to relationships: the revoked-by SRO and deprecation clones are pulled in dynamically at bundle export. +- [x] **Revoke must reach member sync.** Implemented 2026-07-13: member sync subscribes to the 11 per-type `::revoked` events via a payload adapter (`handleStixObjectRevokedEvent`), so the revoked revision (`revoked: true`) is enrolled as a candidate in member tracks and candidate/staged pins move to it — treated exactly like any new revision. The revoke response's primary document carries the resulting backrefs. Member sync is not extended to relationships; the bounded `revoked-by` edge is captured in each new snapshot's graph manifest and replayed from there. - [x] **Technique conversion should reach revision sync.** Implemented 2026-07-13 with the adapter approach (same pattern as `handleStixObjectRevokedEvent`): the `TECHNIQUE_CONVERTED_TO_SUBTECHNIQUE` / `SUBTECHNIQUE_CONVERTED_TO_TECHNIQUE` event payloads now carry the converted revision (`document`) and acting user, and member sync subscribes via `handleStixObjectConvertedEvent`, treating the conversion as a `new-revision` trigger through the workflow gate — candidate/staged pins move to the converted revision, member tracks enroll it as a candidate. The conversion responses refresh `workspace.release_tracks` after event processing (read-your-own-writes). Tests: conversion cases in `release-tracks-change-capture.spec.js` and the updated clone-strip test in `release-tracks-backrefs.spec.js`. ## Get Releases By Object -- [X] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a - caller can retrieve every tagged snapshot whose `members` tier directly - contains the supplied STIX ID, across all object revisions and release - tracks. +- [x] Implement `GET /api/release-tracks/objects/:objectRef/releases` so a + caller can retrieve every tagged snapshot whose `members` tier directly + contains the supplied STIX ID, across all object revisions and release + tracks. ### Design @@ -1256,12 +1385,14 @@ Use `releaseTrackRegistry` as a compact global forward catalogue instead. Its single document per track gains a server-maintained `tagged_releases` array: ```javascript -tagged_releases: [{ - snapshot_modified: Date, // (track_id, snapshot_modified) identifies the snapshot - version: String, - tagged_at: Date, - tagged_by: String -}] +tagged_releases: [ + { + snapshot_modified: Date, // (track_id, snapshot_modified) identifies the snapshot + version: String, + tagged_at: Date, + tagged_by: String, + }, +]; ``` `tagged_release_count` is derived from `tagged_releases.length`. The actual @@ -1295,25 +1426,25 @@ therefore incur no index cost, and draft squashing does not affect the lookup. - Support `order=asc|desc` by `snapshot_modified`, plus `limit` and `offset`. - Return 200 with an empty result for a valid STIX ID with no tagged releases; malformed IDs return 400. -- Ascending order describes first *published/tagged* appearance, not the time +- Ascending order describes first _published/tagged_ appearance, not the time the object first entered an untagged draft. ### Checklist - [x] Registry schema/repository: add `tagged_releases`, reconciliation, and - derived count/latest-version maintenance. + derived count/latest-version maintenance. - [x] Dynamic snapshot schema/repository: add the tagged-member partial index - and a projected `findTaggedSnapshotsContainingObject` query. + and a projected `findTaggedSnapshotsContainingObject` query. - [x] Versioning: reconcile registry metadata after tagging and validate - version progression against track-wide tagged releases rather than a - potentially stale historical snapshot's embedded `version_history`. + version progression against track-wide tagged releases rather than a + potentially stale historical snapshot's embedded `version_history`. - [x] API: route, controller Zod validation, facade/service orchestration, - deterministic pagination, and OpenAPI contract. + deterministic pagination, and OpenAPI contract. - [x] Migration: backfill registry tagged-release refs and ensure the new index - on all existing dynamic track collections. + on all existing dynamic track collections. - [x] Regression tests: multiple tracks/releases/revisions, removal after an - earlier release, retroactive tag, virtual track, draft/non-member exclusion, - filtering/order/pagination, empty/malformed input, and backfill behavior. + earlier release, retroactive tag, virtual track, draft/non-member exclusion, + filtering/order/pagination, empty/malformed input, and backfill behavior. - [x] User/developer docs and Bruno request. - [ ] Verification: targeted spec first, then the complete `npm test` suite. - Targeted endpoint spec: 8 passing; release-track directory: 69 passing; @@ -1328,8 +1459,8 @@ therefore incur no index cost, and draft squashing does not affect the lookup. ## Snapshot Retention (Squash on Tag) - [ ] Implement draft-snapshot squashing so release cycles don't accumulate - unbounded snapshot storage. Design captured 2026-07-15; assessed as sound — - see analysis below. + unbounded snapshot storage. Design captured 2026-07-15; assessed as sound — + see analysis below. ### Why @@ -1356,7 +1487,7 @@ Mitigating facts (verified in code): docs, but no code change needed there. - `::created` events for brand-new objects are no-ops for member sync (`findTracksReferencingObject` only matches already-tracked `stix.id`s). - The O(N²) trap is bulk *re-imports/updates* of already-tracked objects + The O(N²) trap is bulk _re-imports/updates_ of already-tracked objects (e.g. re-importing a modified 20k-object bundle → 20k snapshots × MBs each). - `version_history` is embedded in and carried forward by every clone, so the release ledger survives squashing — tagged snapshots and the latest draft @@ -1414,36 +1545,36 @@ the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. ### Alternatives considered -- *Amend-in-place* (member sync mutates the latest draft instead of cloning): +- _Amend-in-place_ (member sync mutates the latest draft instead of cloning): attacks the root cause but breaks the "every modification is a new snapshot" invariant, complicates concurrent reads, and silently degrades the audit trail for everyone. Rejected for now. -- *Delta/structural-sharing storage*: large refactor of the snapshot store; +- _Delta/structural-sharing storage_: large refactor of the snapshot store; revisit only if squash proves insufficient. -- *TTL/retention config* (e.g. `config.retention.auto_squash_on_tag`, +- _TTL/retention config_ (e.g. `config.retention.auto_squash_on_tag`, max-draft-age): natural follow-on once manual squash exists. ### Checklist - [ ] Repo: `deleteDraftSnapshotsBefore(trackId, boundary)` in - `release-track-dynamic.repository.js` (deleteMany on - `{ id, version: null, modified: { $lt: boundary } }`, excluding the latest - snapshot's `modified`). + `release-track-dynamic.repository.js` (deleteMany on + `{ id, version: null, modified: { $lt: boundary } }`, excluding the latest + snapshot's `modified`). - [ ] Service: squash logic in `versioning-service.js` (`squash` option on - `_doBump`) + standalone squash operation (probably `snapshot-service.js`); - reject for virtual tracks; return `squashed_count`. + `_doBump`) + standalone squash operation (probably `snapshot-service.js`); + reject for virtual tracks; return `squashed_count`. - [ ] Controller/routes: `squash` in the Zod bump body schema; new - `POST /api/release-tracks/:id/snapshots/squash` route with Zod-validated - optional `before`. + `POST /api/release-tracks/:id/snapshots/squash` route with Zod-validated + optional `before`. - [ ] OpenAPI: bump request body + new squash path. - [ ] Regression tests (`release-tracks-squash.spec.js`): squash-on-tag - deletes only pre-tag drafts; tagged snapshots survive; drafts newer than - the tagged snapshot survive; latest-draft never deleted by maintenance - squash; registry counters resync; backrefs untouched; virtual track - rejected; no-tagged-release + no `before` → 400; idempotent re-squash. + deletes only pre-tag drafts; tagged snapshots survive; drafts newer than + the tagged snapshot survive; latest-draft never deleted by maintenance + squash; registry counters resync; backrefs untouched; virtual track + rejected; no-tagged-release + no `before` → 400; idempotent re-squash. - [ ] Docs: `docs/user/release-tracks/versioning.md` (squash behavior + - the bulk-endpoints-vs-per-object-loop warning for initial population), - `docs/developer/release-tracks/` (why, trade-offs, provenance loss). + the bulk-endpoints-vs-per-object-loop warning for initial population), + `docs/developer/release-tracks/` (why, trade-offs, provenance loss). - [ ] Bruno: bump `.bru` gains `~squash` toggle; new squash request file. ### Future (not in scope) @@ -1459,18 +1590,21 @@ the latest snapshot. Like `git rebase --squash`ing the commits behind a tag. ## Small Fixes - [x] **Composition schema mismatch: `priority`.** Resolved 2026-07-29 by - requiring a unique, non-negative integer priority in request validation, - persistence, OpenAPI, documentation, and Bruno examples. + requiring a unique, non-negative integer priority in request validation, + persistence, OpenAPI, documentation, and Bruno examples. -- [ ] **`deleteSnapshot` lacks a tagged-release guard.** `DELETE /api/release-tracks/:id/snapshots/:modified` (`snapshot-service.deleteSnapshot`) deletes any snapshot, including tagged releases — contradicting the "immutable once set" versioning rule. Should 409 on `version != null` (a squash implementation must also filter `version: null`; see Snapshot Retention section). Found 2026-07-15 while designing squash. +- [x] **Restrict `deleteSnapshot` to the latest untagged draft.** + `DELETE /api/release-tracks/:id/snapshots/:modified` now returns `409` + for tagged releases and historical drafts, preserving immutable history. + Completed by the immutable-history slice. -- [ ] **`syncRegistryCounters` scales with snapshot count.** It fetches *all* snapshots (`getAllSnapshots` with projection) on every clone to recount — O(snapshot_count) reads per write, on the hottest path (member sync). Fine post-squash; consider a count query or incremental counters if draft accumulation between tags is large. +- [ ] **`syncRegistryCounters` scales with snapshot count.** It fetches _all_ snapshots (`getAllSnapshots` with projection) on every clone to recount — O(snapshot_count) reads per write, on the hottest path (member sync). Fine post-squash; consider a count query or incremental counters if draft accumulation between tags is large. ## Diffing Endpoint - [ ] Implement object diffing endpoints for snapshots. Users should be able to effectively preview changes to objects before tier transitions (candidates, staged, members). -### Idea 1 - Diffing endpoint specifically for release tracks +### Idea 1 - Diffing endpoint specifically for release tracks In this approach, we would implement a workflow-driven diffing endpoint that is specific to release tracks. The endpoint would allow users to diff objects in the candidate snapshot against their previous revisions in the staged or member snapshots. @@ -1483,12 +1617,12 @@ If `:objectRef` is a reference to an object that is not part of the candidate sn To clarify, snapshot objects transition linearly and unidirectionally through the following tier transitions: Candidate -> Staged -> Member -An object exists as a set of one or more revisions. An object is identified by its `stix.id` field, whereas an object revision is identified by its `stix.id` and `stix.modified` fields. +An object exists as a set of one or more revisions. An object is identified by its `stix.id` field, whereas an object revision is identified by its `stix.id` and `stix.modified` fields. A revision can exist in exactly one tier at a time. - If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. -- If it exists in the staged snapshot, it will not exist in the candidate or member snapshots. +- If it exists in the staged snapshot, it will not exist in the candidate or member snapshots. - If it exists in the member snapshot, it will not exist in the candidate or staged snapshots. If a revision exists in the candidate snapshot, it will not exist in the staged or member snapshots. However, a _previous_ revision may exist in the staged or member tiers (though it is not guaranteed). Because the tier transitions are unidirectional, revisions must be temporally ordered as it relates to how they are distributed across the tiers. It should not be possible for a newer revision to exist in a previous tier. For example, if a revision exists in the candidate snapshot, it is not possible for a newer revision to exist in the staged or member snapshots. @@ -1508,6 +1642,7 @@ To stick with the example, if a revision exists as a candidate, a previous revis ### Idea 2 - Diffing endpoint for all objects (not just release tracks) Type-centric: + ``` GET /api/:type/:id/diff GET /api/:type/:id/modified/:modified/diff @@ -1515,7 +1650,8 @@ GET /api/:type/:id/modified/:modified/diff Type-agnostic: -Embed the +Embed the + ``` GET /api/attack-objects/:id/diff GET /api/attack-objects/:id/modified/:modified/diff @@ -1529,6 +1665,7 @@ GET /api/attack-objects/:id/modified/:modified/diff ``` Set up a diffing endpoint that is type-agnostic and allows users to compare any two revisions of an object. The endpoint should accept a request body that specifies the `compareTo` revision, and the endpoint should return a diff between the current revision and the specified `compareTo` revision. + ``` GET /api/compare { @@ -1545,8 +1682,6 @@ GET /api/compare } ``` - - ## Repurposing the `note` object - [ ] Implement support for tracking notes on snapshot objects (can be candidates, staged, or members). Notes should be stored in a separate Mongo collection and linked to the snapshot object via a reference field. Users should be able to add, edit, and delete notes via the API. Notably, we already have a notes service that can be leveraged for this purpose. However, it needs some modifications. The service was originally implemented with STIX in mind. The idea was to treat/represent notes as STIX objects and enable users to include them in emitted STIX bundles. However, the concept never really took off. We should modify the service to treat notes as second-class objects that are entirely separate from STIX, but rather as Workbench-native objects. Notes should be capable of being linked/attached to snapshot objects (candidates, staged, or members) as well as to objects independent of snapshots (documents in the `attackObjects` collection). @@ -1575,8 +1710,8 @@ Links/references between notes and snapshot objects will be one-to-many. A singl { "_id": "ObjectId", "workspace": { - "notes": ["ObjectId"] // Array of references to notes linked to this attack object + "notes": ["ObjectId"] // Array of references to notes linked to this attack object }, - "stix": "StixObject", + "stix": "StixObject" } ``` diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md index 11fec021..0d6f62d1 100644 --- a/docs/developer/release-tracks/authorization.md +++ b/docs/developer/release-tracks/authorization.md @@ -2,8 +2,8 @@ Release-track access follows the existing Workbench roles. Read operations are available to visitors and higher. Normal draft workflow operations require an -editor, team lead, or administrator. Operations that can replace authoritative -membership or destroy history require an administrator. +editor, team lead, or administrator. Deleting an entire track and all of its +history requires an administrator. ## Authorization matrix @@ -13,23 +13,17 @@ membership or destroy history require an administrator. | Preview releases and export snapshots | Yes | Yes | Yes | | Create tracks and drafts; manage candidates/staged/config/composition | No | Yes | Yes | | Tag a standard or virtual snapshot | No | Yes | Yes | -| Delete an untagged individual snapshot | No | Yes | Yes | -| Replace standard-track members directly | No | No | Yes | +| Delete the latest untagged draft snapshot | No | Yes | Yes | | Delete an entire track and all snapshot history | No | No | Yes | -The two direct replacement routes and full-track deletion also require -`confirm_track_id` to equal the `:id` path parameter. Authorization runs before -the controller, and confirmation runs before request-body validation or -persistence. +Full-track deletion also requires `confirm_track_id` to equal the `:id` path +parameter. Authorization runs before the controller, and confirmation runs +before persistence. ## Audited destructive actions -The following actions create a `releaseTrackAuditEvents` record before their -business operation begins: - -- `replace_members_latest` -- `replace_members_historical` -- `delete_track` +The `delete_track` action creates a `releaseTrackAuditEvents` record before +the business operation begins. Each event records the authenticated actor, confirmation value, target track, request summary, timestamps, and a `pending`, `completed`, or `failed` status. diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index 6e2d3a9d..d5a98790 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -18,9 +18,9 @@ tracks follow that precedent but maintain the pointers event-driven. Membership changes through many routes: add/remove candidates, review, manual and auto promotion, demotion, release (staged → members), member sync, -`updateContents`, track cloning, bundle import, snapshot deletion, and track -deletion. Patching each route with a bespoke incremental backref update would -be error-prone and would drift. +track cloning, bundle import, latest-draft deletion, and track deletion. +Patching each route with a bespoke incremental backref update would be +error-prone and would drift. Instead, every route already funnels through a small set of persistence choke points, and each choke point triggers a full **snapshot-driven reconciliation**: @@ -34,7 +34,7 @@ self-healing — a missed or failed pass is corrected by the next one. ``` snapshot-service.cloneSnapshot ┐ (every tier/config/metadata mutation, snapshot-service._cloneToNewTrack │ member sync, auto-promotion, -snapshot-service.deleteSnapshot │ bundle import, updateContents, ...) +snapshot-service.deleteSnapshot │ bundle import, ...) snapshot-service.deleteTrack │ versioning-service.releaseLatest/releaseByModified ┘ (staged → members via tagSnapshotInPlace) │ diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 5f9abb0c..81e984c7 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -100,22 +100,19 @@ Implemented in filter, mirroring the fact that members are inherently reviewed. `state` never affects members. `reviewed` is intentionally not a valid `state` value for this reason. -2. **Hydration** — any selected candidate/staged `"latest"` selectors are - resolved for this export request, then the concrete - `{object_ref, object_modified}` pairs are batch-fetched per STIX type via - each repository's `findManyByIdAndModified`. The stored draft selectors are - not mutated. Hydration is fail-closed: if any selected primary revision is - missing, the request returns `409` with `missing_references` and emits no - partial bundle. Database failures propagate as server errors. -3. **Relationships** — the relationship service fetches the latest active - relationship revisions whose `source_ref` and `target_ref` are both among - the selected objects. Deprecated data-component `detects` relationships - are excluded. Relationships remain indirect export-time content; they are - not added to the snapshot tiers. -4. **Supporting objects** — referenced identities and marking definitions - that are not themselves tier entries are fetched and appended. -5. **LinkById conversion** — same behavior as the legacy exporter, preferring - objects already in the export before falling back to a database lookup. +2. **Manifest replay** — the snapshot identifies an active graph manifest, or + a complete linked pending manifest recovering from an interrupted + activation, created at the same persistence boundary. The manifest records exact + primary, relationship, secondary, supporting, and LinkById dependency + revisions. Export hydrates those entries and performs no live graph + expansion. +3. **Bounded secondary selection** — replay starts from the requested primary + tiers, follows only dependency edges frozen in the manifest, and emits a + relationship only when both exact endpoint revisions are selected. +4. **Supporting objects** — only identities and marking definitions frozen in + the manifest and referenced by the selected graph are appended. +5. **LinkById conversion** — conversion uses only exact render targets frozen + in the manifest and never falls back to a current database lookup. 6. **Assembly** (Zod transform) — notes are dropped, objects are conformed to `stixVersion` via the shared `lib/stix-conformance.js` helpers, and the bundle envelope is emitted (with `spec_version: "2.0"` only when @@ -134,9 +131,10 @@ Implemented in - `x_mitre_contents`: every bundle object except marking definitions (which are recorded in `object_marking_refs`), sorted by `object_ref` -Because snapshot contents are explicitly curated, the export intentionally -does **not** apply the legacy attack-id / deprecated / revoked filters — if a -revision is in the snapshot, it is exported. +Because snapshot contents are explicitly curated, primary entries do **not** +receive the legacy attack-id / deprecated / revoked filters. Secondary graph +capture retains the established bounded ATT&CK expansion rules and freezes +the resulting graph at snapshot creation. #### Relationship and secondary-object consistency boundary @@ -146,47 +144,58 @@ Release-track snapshots distinguish **primary** and **secondary** content: record exact `(object_ref, object_modified)` revisions. Standard candidates and staged entries may instead store `"latest"` and are resolved just in time when a draft export includes those tiers. -- Secondary objects are not snapshot members. They are discovered because a - primary object references them through an embedded STIX ID, an SRO connects - two selected primary objects, or the bundle needs a supporting identity or - marking definition. +- Secondary objects are not snapshot members. They are discovered when the + snapshot is created because an exact-pinned SRO connects them to a primary, + the bounded ATT&CK rules identify a detection strategy, or the bundle needs + a supporting identity, marking definition, or LinkById render target. Tagged standard membership is deterministic because release planning resolves staged selectors before promoting them to members. Virtual materialization likewise copies exact member revisions from tagged component snapshots and -never follows a component's later `track_latest` candidate movement. Draft -exports that explicitly include dynamic candidate/staged tiers are snapshots -of the latest revisions at export time. Secondary content is also resolved -just in time during bundle generation. - -Relationships are the largest consistency boundary. Current SRO -`source_ref`/`target_ref` fields identify STIX object IDs, not exact -`(object_id, object_modified)` revisions. An SRO can consequently describe the -whole revision chain of each endpoint rather than one precise pair of SDO -entities. The exporter resolves the latest active relationship revisions when -the bundle is requested. This creates several tradeoffs: - -- exporting the same tagged snapshot at different times can produce different - relationship objects or TOC contents; -- relationship revisions are not represented in snapshot history, - release-track backrefs, or composition audit metadata; -- revoking a relationship can remove it from an older snapshot export, while - creating a relationship can add it to that export; -- each bundle request performs a relationship query, although the query is - constrained to relationships whose two endpoints are already selected. - -Consumers that require byte-for-byte or graph-level reproducibility must -archive the emitted bundle. - -Making bundle graphs deterministic requires a separate, high-risk data-model -change rather than virtual composition re-resolution. A future design must -version-control relationships, pin each SRO endpoint to an exact SDO revision, -and likely clone every affected SRO whenever a new endpoint revision is -created. It must also persist an export manifest containing the selected -relationship and other secondary-object revisions. That one-to-one SDO/SRO -model has significant migration, write-amplification, concurrency, and -database-storage costs and is deliberately deferred pending design and -measurement. +never follows a component's later `track_latest` candidate movement. + +Every relationship revision stores server-controlled exact source and target +pins under `workspace.relationship_endpoints`. These fields identify the +precise `(object_ref, object_modified)` pair represented by each side of the +SRO. They are not emitted because bundle output includes only the `stix` +object. When an endpoint advances, Workbench creates a new SRO revision with +updated pins rather than rewriting the older SRO. + +Each persisted snapshot references a tier-aware manifest. A pending manifest +and all of its entries are written before the snapshot is linked to it, then +activated after persistence succeeds. The snapshot link is the durable commit +record: replay can use and self-activate a complete linked pending manifest +after a process interruption. +A standard release replaces the draft manifest with one built from the +resolved release plan, so dynamic staged selectors become exact members. +Materialized virtual snapshots contain exact roots from the outset. + +Active and pending manifests protect their exact dependencies. In-place +updates and hard deletes that would invalidate a primary or secondary +revision return `409`; lineage deletion is rejected when any version is +protected. Relationship source, target, and type changes are rejected. +Description-only relationship corrections remain allowed because the +relationship STIX payload used by older snapshots is frozen in the manifest. +Deleting a draft snapshot or track removes its manifest and releases +protection that no other snapshot needs. + +Existing data is upgraded by an idempotent migration. Only the latest +revision of each legacy relationship can be endpoint-pinned truthfully. +Pre-existing snapshot manifests are labeled `baseline_reconstruction` +because they describe the graph visible during migration rather than an +unknowable historical graph. + +The deliberate exception is a standard draft export that explicitly includes +a candidate or staged entry stored as `"latest"`. That selector is defined to +move until release, so the selected draft graph is resolved for that request. +Release preview and commit resolve it again; a successful commit stores an +exact manifest. Members, tagged releases, materialized virtual snapshots, and +exact-selector draft tiers replay deterministically. + +The graph and object payload are reproducible, but the bundle is not promised +to be byte-for-byte identical: the bundle envelope receives a newly generated +bundle ID. Consumers should compare the emitted STIX object set and revisions, +not the envelope UUID. ### Where validation happens diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index abd4272d..e6794cb6 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -444,9 +444,13 @@ copies the exact member revisions from the selected tagged component snapshots, and later component activity cannot change the persisted virtual snapshot. -This deterministic guarantee covers primary snapshot membership. Secondary -objects and relationships discovered while rendering `format=bundle` remain -an export-time concern and can change between bundle requests. +Each snapshot also references an internal, tier-aware graph manifest. It +freezes the exact relationship endpoint revisions, bounded secondary objects, +supporting objects, and LinkById render targets needed by `format=bundle`. +Tagged standard snapshots, materialized virtual snapshots, and draft tiers +that use exact selectors therefore replay the same graph. A standard draft +tier explicitly stored as `"latest"` remains intentionally dynamic until the +release boundary. The three valid `snapshot_schedule` shapes are: diff --git a/docs/developer/release-tracks/error-handling.md b/docs/developer/release-tracks/error-handling.md index cc553b03..ebbd6cd8 100644 --- a/docs/developer/release-tracks/error-handling.md +++ b/docs/developer/release-tracks/error-handling.md @@ -14,7 +14,8 @@ } ``` -**Solution:** Create a new snapshot by modifying the collection, then release the new snapshot. +**Solution:** Create a new draft through a supported release-track workflow +operation, then release the new snapshot. ### InvalidVersionError @@ -52,6 +53,33 @@ Tagged snapshots are immutable release records. Create or modify a draft snapshot instead; deleting an entire release track remains a separate track-level operation. +### HistoricalSnapshotDeletionError + +**Thrown when:** Attempting to delete an untagged draft that is no longer the +latest snapshot. + +**HTTP Status:** 409 Conflict + +The response identifies both `snapshot_modified` and +`latest_snapshot_modified`. Refresh the track and continue from the latest +draft; historical drafts cannot be removed. + +### SnapshotGraphPinnedRevisionError + +**Thrown when:** An in-place update or hard delete would change an exact +primary, relationship, secondary, supporting, or LinkById dependency frozen +in a release-track snapshot graph. Full-lineage and collection +`deleteAllContents` operations are preflighted against the same invariant. + +**HTTP Status:** 409 Conflict + +The response includes `snapshot_graph_pins` entries identifying the track, +snapshot timestamp, manifest entry kind, and tier where applicable. Create a +new STIX revision instead. Administrator authorization is not a force-delete +override. Description-only relationship corrections remain allowed because +the older relationship payload is frozen inside each existing manifest; +source, target, and relationship-type changes return 400. + ### NotFoundError **Thrown when:** Collection with specified ID does not exist. diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index ec2db9dd..449bfbf7 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -52,7 +52,7 @@ silently dropping it. The error contract distinguishes who can correct the problem: - Request ingress returns `400` with `missing_references` when candidate - selection or direct member replacement names a revision that does not exist. + selection names a revision that does not exist. - Operations over already-persisted content return `409` with `missing_references` when a release preview/commit, track clone, virtual materialization, quarantine promotion, or bundle export encounters a @@ -162,10 +162,11 @@ a standard component track. Snapshot retrieval never re-runs composition, so there is no `resolve` query parameter or `resolved_content` response wrapper. Workbench retrieval returns -the persisted primary membership. Bundle export is a separate consistency -boundary: secondary relationships and supporting objects are discovered at -request time and are not deterministic until relationships become -version-controlled against exact endpoint revisions. +the persisted primary membership. Bundle export replays a graph manifest +captured with the snapshot. Relationship revisions carry server-controlled +exact endpoint pins in `workspace.relationship_endpoints`, and the manifest +freezes the bounded secondary/supporting graph without emitting those internal +fields in STIX output. Snapshot schedules use the same strict, mode-discriminated Zod schema at the controller and service boundaries. `manual` has no selector field, `cron` @@ -212,7 +213,8 @@ There is no side-effect-free virtual snapshot-creation preview. Once a virtual draft is persisted, it uses the same retrieval and release endpoints as a standard draft. Release planning never resolves composition and rejects a virtual draft without `composition_resolution` with `409 Conflict`. Generic -latest and historical `/contents` mutations are standard-only; virtual +snapshot member replacement is not supported for either track type. Standard +membership enters through the candidate/staged/release lifecycle; virtual membership has composition resolution as its sole authority. Quarantine promotion is a snapshot mutation, not a composition diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index c0288a9d..023a4f32 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -130,6 +130,10 @@ sync strategy determines what workflow action (if any) to take: > `BaseService` rejects `PUT`/`DELETE` of a members-pinned revision with > 409 (`MemberPinnedRevisionError`) — released content is immutable in > place. +> - A candidate or staged revision remains editable unless it is also a +> secondary/supporting dependency frozen in a snapshot graph manifest. In +> that case graph integrity takes precedence and the operation returns 409 +> (`SnapshotGraphPinnedRevisionError`). ### Relationship to Existing Features diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index f722075a..9a1bc26f 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -47,7 +47,6 @@ POST /api/release-tracks/new POST /api/release-tracks/new-from-bundle POST /api/release-tracks/import POST /api/release-tracks/:id/meta -POST /api/release-tracks/:id/contents?confirm_track_id=:id POST /api/release-tracks/:id/snapshots/latest/release POST /api/release-tracks/:id/clone DELETE /api/release-tracks/:id?confirm_track_id=:id @@ -59,7 +58,6 @@ DELETE /api/release-tracks/:id?confirm_track_id=:id GET /api/release-tracks/:id/snapshots GET /api/release-tracks/:id/snapshots/latest GET /api/release-tracks/:id/snapshots/:modified -POST /api/release-tracks/:id/snapshots/:modified/meta POST /api/release-tracks/:id/snapshots/:modified/release POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified @@ -462,46 +460,18 @@ Creates new snapshot with updated metadata. } ``` -### Update Contents +### Snapshot content is append-only -``` -POST /api/release-tracks/:id/contents -``` - -Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** The main workflow for enrolling new member objects into `x_mitre_contents` is through the candidate-staging promotion cycle described in [versioning.md](./versioning.md). - -This operation requires the administrator role. The -`confirm_track_id` query parameter must exactly equal the `:id` path -parameter. Every accepted attempt is recorded in the durable release-track -destructive audit trail. - -This operation is available only for standard tracks. Virtual membership is -computed from component releases and can only be updated by materializing a -virtual draft with `POST /api/release-tracks/:id/virtual/snapshots/create`. -Using either contents endpoint with a virtual track returns `400 Bad Request`. - -**Request Body:** - -```json -{ - "x_mitre_contents": [ - { - "obj_ref": "attack-pattern--uuid1", - "obj_modified": "2024-02-01T10:00:00.000Z" - }, - { - "obj_ref": "malware--uuid2", - "obj_modified": "latest" - } - ] -} -``` +There is no endpoint for replacing a persisted snapshot's `members` tier. +Standard tracks add or revise content through candidates, promote those +objects to staged, and freeze them into members during release. Virtual tracks +derive members only when a composition is materialized. -Every entry must include an object ID and either an ISO `obj_modified` -timestamp or the request-time shorthand `"latest"`. The server resolves -`"latest"` to the object's actual latest `stix.modified` value before -persisting the new standard-track snapshot. Snapshot members never store a -moving reference. +If an operator makes an unwanted draft, delete it while it is still the latest +untagged snapshot or continue with a newer corrective draft. Historical drafts +and tagged releases remain part of the immutable track history. Bootstrapping a +new track from a bundle is the supported way to start with an existing member +set. ### Release Latest Snapshot @@ -621,30 +591,6 @@ GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle&include=staged ``` -### Update Metadata (Specific Snapshot) - -``` -POST /api/release-tracks/:id/snapshots/:modified/meta -``` - -Creates new snapshot with updated metadata. - -**Request Body:** Same as [Update Metadata](#update-metadata) for latest snapshot. - -### Update Contents (Specific Snapshot) - -``` -POST /api/release-tracks/:id/snapshots/:modified/contents?confirm_track_id=:id -``` - -Creates new snapshot with updated member objects. **This is intended for retroactive hotfixes only.** - -**Request Body:** Same as [Update Contents](#update-contents) for latest snapshot. - -Like the latest form, this operation is restricted to standard tracks, -requires the administrator role and exact track-ID confirmation, and creates -a durable audit event. - ### Release/Tag Specific Snapshot Converts a specific draft snapshot to a tagged release. Tags snapshot in-place (does not create new snapshot). @@ -665,12 +611,15 @@ POST /api/release-tracks/:id/snapshots/:modified/clone ### Delete Specific Snapshot -**TODO**: further consideration needs to be given here. We need to be careful to avoid breaking contextual continuity between snapshots. - ``` DELETE /api/release-tracks/:id/snapshots/:modified ``` +Deletes the selected snapshot only when it is both the latest snapshot and an +untagged draft. Deletion reverts the track to the immediately preceding +snapshot. Tagged releases and older drafts return `409 Conflict`; they cannot +be removed or rewritten. + --- ## Candidate Management @@ -1332,10 +1281,16 @@ retrieval never recomputes virtual composition. As long as the track does not acquire a newer snapshot, `/snapshots/latest` selects the same primary revision set, and `/snapshots/:modified` addresses that set explicitly. -This determinism does not extend to the complete `format=bundle` graph. -Secondary relationships, identities, marking definitions, and other supporting -objects are resolved during bundle generation and may change independently of -the primary snapshot members. +The server freezes the bounded `format=bundle` graph when it persists the +snapshot. Repeated exports reuse exact relationship, secondary, supporting, +and LinkById dependency revisions rather than discovering the current graph. +Hard deletes and unsafe in-place edits to those protected revisions return +`409 Conflict`. + +A standard draft remains intentionally dynamic only when the request includes +a candidate or staged entry stored with `object_modified: "latest"`. That +selector is resolved at request time until release. Tagged standard members +and all materialized virtual members are exact. `duplicates_found` counts object IDs contributed by more than one component, including repeated contributions of the same exact revision. diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 7d960a68..4fff6916 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -76,8 +76,9 @@ An object referenced by multiple tracks carries one entry per track. Release tracks are never blind to changes in the objects they pin: -- **Members-pinned revisions are immutable in place.** `PUT` and `DELETE` - against a revision that any track pins in its `members` tier return +- **Released and graph-frozen revisions are immutable in place.** `PUT` and + `DELETE` against a revision that any track pins in its `members` tier, or + that a snapshot needs as a secondary/supporting graph dependency, return `409 Conflict` — released content cannot be changed or destroyed under the track. Make changes by creating a new revision (`POST`); retire an object by creating a new revision with `x_mitre_deprecated: true`. Revision sync @@ -86,6 +87,11 @@ Release tracks are never blind to changes in the objects they pin: protected when it belongs only to a historical tagged release, when a newer draft has removed it, or when a reconciliation failure temporarily omitted its backref. +- **Standalone candidate/staged roots remain editable.** Merely appearing in + a draft workflow tier does not create a graph-protection conflict, so the + existing in-place review workflow below still applies. If that same revision + is also a frozen secondary dependency of another selected root, graph + protection takes precedence and the edit returns `409`. - **Candidate/staged-pinned revisions can be edited in place, but the track sees it.** An in-place `PUT` (including one that only sets `x_mitre_deprecated`) marks the pinned entry `modified-in-place`: the @@ -104,8 +110,9 @@ Release tracks are never blind to changes in the objects they pin: workflow creates one new revision of the revoked object (`revoked: true`); revision sync enrolls it as a candidate in tracks where the object is a member and moves candidate/staged pins to it. The revoking - object and the `revoked-by` relationship are not tracked explicitly — - bundle export pulls secondary objects and their SROs in dynamically. + object and the `revoked-by` relationship are not direct track members. + Snapshot creation captures them as bounded secondary graph dependencies + when applicable; later bundle export replays that frozen graph. ## Lifecycle example diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index 55353ba9..336e067c 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -68,26 +68,25 @@ GET /api/release-tracks/:id/snapshots/latest POST /api/release-tracks/:id/config POST /api/release-tracks/:id/meta POST /api/release-tracks/:id/clone -PUT /api/release-tracks/:id/snapshots/latest/release -POST /api/release-tracks/:id/archive -DELETE /api/release-tracks/:id +POST /api/release-tracks/:id/snapshots/latest/release +DELETE /api/release-tracks/:id?confirm_track_id=:id # Candidate/workflow management POST /api/release-tracks/:id/candidates POST /api/release-tracks/:id/candidates/review +POST /api/release-tracks/:id/candidates/promote +POST /api/release-tracks/:id/staged/demote # Snapshot-specific operations GET /api/release-tracks/:id/snapshots/:modified -POST /api/release-tracks/:id/snapshots/:modified/config -POST /api/release-tracks/:id/snapshots/:modified/meta POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified -PUT /api/release-tracks/:id/snapshots/:modified/release +POST /api/release-tracks/:id/snapshots/:modified/release ``` ### 2. Git-Inspired Versioning -We borrow heavily concepts from git. Snapshots are sort of like commits and tagged releases are like git tags. A release track contains snapshots: delta permutations that can be linearly tracked to deduce how the release track has evolved over time. A snapshot is generated every time a change is made, whether that be adding/removing objects, updating the release track configuration, or renaming the release track altogether. +We borrow heavily concepts from git. Snapshots are sort of like commits and tagged releases are like git tags. A release track contains snapshots: delta permutations that can be linearly tracked to deduce how the release track has evolved over time. A snapshot is generated every time a supported draft operation changes state, such as adding or promoting candidates, updating release-track configuration, or renaming the release track. **Snapshots** (like Git commits) - Every modification creates a new snapshot @@ -130,6 +129,13 @@ and committing are separate operations, so a newer object revision created between them can legitimately produce a different plan; the committed release records the revision resolved by the commit itself. +At snapshot persistence, the server also freezes the bounded bundle graph: +exact relationship endpoint revisions, secondary objects, supporting objects, +and LinkById render targets. Tagged releases and materialized virtual +snapshots therefore reproduce the same STIX object graph on later +`format=bundle` retrievals. The generated bundle-envelope ID itself is not +stable. + Virtual snapshots are stricter still: they copy only exact member revisions from tagged standard component snapshots. They never inherit `track_latest`, and retrieving a persisted virtual snapshot does not re-resolve its component diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 6885e5ae..fa06c205 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -954,10 +954,12 @@ Consequently, while the track does not acquire a newer snapshot, `latest` path segment selects the most recent snapshot; it is not a dynamic object-revision selector. -This guarantee applies to the persisted primary snapshot contents. -`format=bundle` also discovers secondary relationships and supporting objects -at export time, so the complete bundle graph is not currently reproducible. -See [Bundle Export](../../developer/release-tracks/bundle-export.md#relationship-and-secondary-object-consistency-boundary). +This guarantee also covers the bounded `format=bundle` object graph. +Relationship endpoint revisions, secondary objects, supporting objects, and +LinkById render targets are frozen in the snapshot's graph manifest. +Repeated exports may use a different bundle-envelope UUID, but replay the same +snapshot object graph. See +[Bundle Export](../../developer/release-tracks/bundle-export.md#relationship-and-secondary-object-consistency-boundary). ## Quarantine Management @@ -1151,8 +1153,7 @@ POST /api/release-tracks/:id/snapshots/:modified/release ``` If composition changes after materialization, repeat the create step. Direct -member replacement through either standard-track `/contents` endpoint is -rejected for virtual tracks. +member replacement is not supported. ### 2. Use Scheduled Snapshots for Consistency diff --git a/docs/user/release-tracks/workflow-examples.md b/docs/user/release-tracks/workflow-examples.md index d3e0c0fc..d1d796f4 100644 --- a/docs/user/release-tracks/workflow-examples.md +++ b/docs/user/release-tracks/workflow-examples.md @@ -8,76 +8,73 @@ POST /api/release-tracks/new { "name": "My Release", ... } # Creates: snapshot 1, x_mitre_version: null -# 2. Update contents -POST /api/release-tracks/release--123/contents -{ "x_mitre_contents": [...] } -# Creates: snapshot 2, x_mitre_version: null +# 2. Add objects as candidates +POST /api/release-tracks/release--123/candidates +{ "object_refs": [{ "id": "attack-pattern--...", "modified": "latest" }] } +# Creates: snapshot 2, version: null -# 3. Update metadata +# 3. Promote accepted candidates to staged +POST /api/release-tracks/release--123/candidates/promote +{ "object_refs": ["attack-pattern--..."] } +# Creates: snapshot 3, version: null + +# 4. Update metadata POST /api/release-tracks/release--123/meta { "description": "Updated description" } -# Creates: snapshot 3, x_mitre_version: null +# Creates: snapshot 4, version: null -# 4. Ready for first release - tag as v1.0 +# 5. Ready for first release - staged objects become members POST /api/release-tracks/release--123/snapshots/latest/release { "increment": "major" } -# Updates: snapshot 3, x_mitre_version: "1.0" (IN-PLACE) - -# 5. Continue development -POST /api/release-tracks/release--123/contents -{ "x_mitre_contents": [...] } -# Creates: snapshot 4, x_mitre_version: null - -# 6. Minor release -POST /api/release-tracks/release--123/snapshots/latest/release -{ "increment": "minor" } -# Updates: snapshot 4, x_mitre_version: "1.1" (IN-PLACE) +# Updates: snapshot 4, version: "1.0" (in place) -# 7. More changes -POST /api/release-tracks/release--123/contents -{ "x_mitre_contents": [...] } -# Creates: snapshot 5, x_mitre_version: null +# 6. Continue development through the same candidate workflow +POST /api/release-tracks/release--123/candidates +{ "object_refs": [{ "id": "malware--...", "modified": "latest" }] } +POST /api/release-tracks/release--123/candidates/promote +{ "object_refs": ["malware--..."] } +# Creates snapshots 5 and 6 -# 8. Another minor release +# 7. Minor release POST /api/release-tracks/release--123/snapshots/latest/release { "increment": "minor" } -# Updates: snapshot 5, x_mitre_version: "1.2" (IN-PLACE) +# Updates: snapshot 6, version: "1.1" (in place) ``` **Resulting Timeline:** ``` -snapshot 1: modified: T1, x_mitre_version: null -snapshot 2: modified: T2, x_mitre_version: null -snapshot 3: modified: T3, x_mitre_version: "1.0" ← RELEASE -snapshot 4: modified: T4, x_mitre_version: "1.1" ← RELEASE -snapshot 5: modified: T5, x_mitre_version: "1.2" ← RELEASE +snapshot 1: initial empty draft +snapshot 2: candidate added +snapshot 3: candidate staged +snapshot 4: version "1.0" ← RELEASE +snapshot 5: next candidate added +snapshot 6: version "1.1" ← RELEASE ``` ### Example 2: Selective Release Tagging ```bash -# Create several snapshots -POST /api/collections/collection--456/contents # snapshot 1 -POST /api/collections/collection--456/contents # snapshot 2 -POST /api/collections/collection--456/contents # snapshot 3 -POST /api/collections/collection--456/contents # snapshot 4 -POST /api/collections/collection--456/contents # snapshot 5 - -# Only tag snapshots 2 and 5 as releases -POST /api/collections/collection--456/modified//snapshots/latest/release +# Create several drafts through ordinary metadata/workflow changes +POST /api/release-tracks/release--456/meta # draft 2 +POST /api/release-tracks/release--456/meta # draft 3 +POST /api/release-tracks/release--456/meta # draft 4 +POST /api/release-tracks/release--456/meta # draft 5 + +# Tag draft 2 retroactively and then tag the latest draft +POST /api/release-tracks/release--456/snapshots//release { "version": "1.0" } -POST /api/collections/collection--456/snapshots/latest/release # Latest = snapshot 5 +POST /api/release-tracks/release--456/snapshots/latest/release { "version": "1.1" } ``` **Resulting Timeline:** ``` -snapshot 1: x_mitre_version: null (skipped) -snapshot 2: x_mitre_version: "1.0" ← RELEASE -snapshot 3: x_mitre_version: null (skipped) -snapshot 4: x_mitre_version: null (skipped) -snapshot 5: x_mitre_version: "1.1" ← RELEASE +snapshot 1: version: null (skipped) +snapshot 2: version: "1.0" ← RELEASE +snapshot 3: version: null (skipped) +snapshot 4: version: null (skipped) +snapshot 5: version: "1.1" ← RELEASE ``` This mirrors Git's ability to tag any commit, not just the latest. @@ -86,22 +83,22 @@ This mirrors Git's ability to tag any commit, not just the latest. ```bash # Tag latest snapshot -POST /api/collections/collection--789/snapshots/latest/release +POST /api/release-tracks/release--789/snapshots/latest/release { "version": "1.0" } # Success: snapshot tagged as v1.0 # Attempt to release the same snapshot again -POST /api/collections/collection--789/snapshots/latest/release +POST /api/release-tracks/release--789/snapshots/latest/release { "version": "1.1" } # Error: AlreadyReleasedError - "This snapshot has already been tagged as version 1.0" -# Solution: Make a change first (creates new snapshot) -POST /api/collections/collection--789/contents -{ "x_mitre_contents": [...] } +# Solution: Make a supported draft change first +POST /api/release-tracks/release--789/meta +{ "description": "Prepare the next release" } # Creates new snapshot # Now release the new snapshot -POST /api/collections/collection--789/snapshots/latest/release +POST /api/release-tracks/release--789/snapshots/latest/release { "version": "1.1" } # Success: new snapshot tagged as v1.1 ``` diff --git a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js new file mode 100644 index 00000000..2643e96d --- /dev/null +++ b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js @@ -0,0 +1,276 @@ +'use strict'; + +/** + * Backfill exact endpoint revision pins on the latest revision of each + * relationship, then reconstruct a baseline graph manifest for every + * pre-existing release-track snapshot. + * + * Historical relationships cannot be reconstructed truthfully because their + * endpoint revision was not recorded when they were created. Snapshot + * manifests produced here are therefore explicitly marked as baseline + * reconstructions of the graph visible at migration time. + */ + +const TRACK_COLLECTION_PATTERN = + /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const CONCURRENCY = 4; + +async function mapWithConcurrency(items, mapper) { + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + await mapper(items[index]); + } + } + + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, items.length) }, () => worker())); +} + +async function latestRelationships(db) { + const latestViewExists = await db + .listCollections({ name: 'view.relationships.latest' }, { nameOnly: true }) + .hasNext(); + if (latestViewExists) { + return db.collection('view.relationships.latest').find({}).toArray(); + } + + return db + .collection('relationships') + .aggregate([ + { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, + { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, + { $replaceRoot: { newRoot: '$document' } }, + ]) + .toArray(); +} + +async function latestEndpoint(db, objectRef) { + return db + .collection('attackObjects') + .findOne( + { 'stix.id': objectRef }, + { projection: { 'stix.id': 1, 'stix.modified': 1 }, sort: { 'stix.modified': -1 } }, + ); +} + +async function buildRelationshipPinOperations(db) { + const relationships = await latestRelationships(db); + const endpointCache = new Map(); + const missing = []; + const operations = []; + + async function endpoint(objectRef) { + if (!endpointCache.has(objectRef)) { + endpointCache.set(objectRef, await latestEndpoint(db, objectRef)); + } + return endpointCache.get(objectRef); + } + + for (const relationship of relationships) { + const [source, target] = await Promise.all([ + endpoint(relationship.stix.source_ref), + endpoint(relationship.stix.target_ref), + ]); + if (!source || !target) { + missing.push({ + relationship_ref: relationship.stix.id, + relationship_modified: relationship.stix.modified, + missing_endpoints: [ + ...(!source ? [relationship.stix.source_ref] : []), + ...(!target ? [relationship.stix.target_ref] : []), + ], + }); + continue; + } + + operations.push({ + updateOne: { + filter: { _id: relationship._id }, + update: { + $set: { + 'workspace.relationship_endpoints': { + source: { + object_ref: source.stix.id, + object_modified: source.stix.modified, + }, + target: { + object_ref: target.stix.id, + object_modified: target.stix.modified, + }, + }, + }, + }, + }, + }); + } + + return { relationships, operations, missing }; +} + +async function findTrackIds(db) { + const [registeredTracks, collections] = await Promise.all([ + db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), + db.listCollections({}, { nameOnly: true }).toArray(), + ]); + return [ + ...new Set([ + ...registeredTracks.map((track) => track.track_id), + ...collections + .map((collection) => collection.name) + .filter((name) => TRACK_COLLECTION_PATTERN.test(name)), + ]), + ].sort(); +} + +async function ensureManifestIndexes(db) { + await Promise.all([ + db + .collection('releaseTrackGraphManifests') + .createIndex({ manifest_id: 1 }, { name: 'manifest_id_1', unique: true }), + db + .collection('releaseTrackGraphManifests') + .createIndex( + { track_id: 1, snapshot_modified: 1, state: 1 }, + { name: 'manifest_by_snapshot' }, + ), + db + .collection('releaseTrackGraphManifestEntries') + .createIndex( + { manifest_id: 1, revision_key: 1, kind: 1, tier: 1 }, + { name: 'unique_manifest_entry', unique: true }, + ), + db + .collection('releaseTrackGraphManifestEntries') + .createIndex( + { object_ref: 1, object_modified: 1, manifest_id: 1 }, + { name: 'manifest_revision_protection' }, + ), + db + .collection('releaseTrackGraphManifestEntries') + .createIndex({ manifest_id: 1, kind: 1, tier: 1 }, { name: 'manifest_id_1_kind_1_tier_1' }), + ]); +} + +async function backfillSnapshotManifests(db, options) { + const graphManifestService = require('../app/services/release-tracks/graph-manifest-service'); + const trackIds = await findTrackIds(db); + const report = { tracks: trackIds.length, snapshots: 0, manifests_created: 0 }; + + await mapWithConcurrency(trackIds, async (trackId) => { + const collectionExists = await db + .listCollections({ name: trackId }, { nameOnly: true }) + .hasNext(); + if (!collectionExists) return; + + const snapshots = await db.collection(trackId).find({}).toArray(); + report.snapshots += snapshots.length; + for (const snapshot of snapshots) { + if (snapshot.graph_manifest_id) { + const linkedManifest = await db.collection('releaseTrackGraphManifests').findOne({ + manifest_id: snapshot.graph_manifest_id, + state: { $in: ['pending', 'active'] }, + }); + if (linkedManifest) { + if (!options.dryRun && linkedManifest.state === 'pending') { + await graphManifestService.activate(linkedManifest.manifest_id); + } + continue; + } + } + if (options.dryRun) { + report.manifests_created++; + continue; + } + + const manifestId = await graphManifestService.prepare(snapshot, { + baselineReconstruction: true, + }); + try { + await db + .collection(trackId) + .updateOne({ _id: snapshot._id }, { $set: { graph_manifest_id: manifestId } }); + await graphManifestService.activate(manifestId); + report.manifests_created++; + } catch (err) { + await graphManifestService.discard(manifestId); + throw err; + } + } + }); + + return report; +} + +async function run(db, options = {}) { + const relationshipPins = await buildRelationshipPinOperations(db); + if (relationshipPins.missing.length > 0) { + const error = new Error( + 'Cannot pin latest relationship endpoints because one or more referenced objects are missing', + ); + error.missing_relationship_endpoints = relationshipPins.missing; + throw error; + } + + if (!options.dryRun && relationshipPins.operations.length > 0) { + await db.collection('relationships').bulkWrite(relationshipPins.operations, { + ordered: false, + }); + await ensureManifestIndexes(db); + } + const manifests = await backfillSnapshotManifests(db, options); + + return { + relationships_scanned: relationshipPins.relationships.length, + relationship_pins_written: relationshipPins.operations.length, + ...manifests, + dry_run: options.dryRun === true, + }; +} + +module.exports = { + async up(db) { + const report = await run(db); + console.log( + `Pinned ${report.relationship_pins_written} latest relationship revision(s) and ` + + `created ${report.manifests_created} baseline snapshot manifest(s)`, + ); + }, + + async down(db) { + const baselineManifests = await db + .collection('releaseTrackGraphManifests') + .find({ baseline_reconstruction: true }) + .project({ manifest_id: 1, track_id: 1, snapshot_modified: 1, _id: 0 }) + .toArray(); + + await mapWithConcurrency(baselineManifests, async (manifest) => { + if (await db.listCollections({ name: manifest.track_id }, { nameOnly: true }).hasNext()) { + await db.collection(manifest.track_id).updateOne( + { + modified: manifest.snapshot_modified, + graph_manifest_id: manifest.manifest_id, + }, + { $unset: { graph_manifest_id: '' } }, + ); + } + }); + const manifestIds = baselineManifests.map((manifest) => manifest.manifest_id); + if (manifestIds.length > 0) { + await db + .collection('releaseTrackGraphManifestEntries') + .deleteMany({ manifest_id: { $in: manifestIds } }); + await db + .collection('releaseTrackGraphManifests') + .deleteMany({ manifest_id: { $in: manifestIds } }); + } + }, + + _private: { + run, + latestRelationships, + buildRelationshipPinOperations, + findTrackIds, + }, +}; diff --git a/package.json b/package.json index deeb337a..bec0dca8 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "test:scheduler": "mocha --timeout 60000 --recursive ./app/tests/scheduler --exit", "test:file": "mocha --timeout 10000 --exit", "repair:release-track-backrefs": "node scripts/reconcileReleaseTrackBackrefs.js", + "preview:deterministic-snapshot-graphs": "node scripts/previewDeterministicSnapshotGraphMigration.js", "check:lockfile": "bash scripts/check-package-lock.sh" }, "dependencies": { diff --git a/scripts/previewDeterministicSnapshotGraphMigration.js b/scripts/previewDeterministicSnapshotGraphMigration.js new file mode 100644 index 00000000..348298e5 --- /dev/null +++ b/scripts/previewDeterministicSnapshotGraphMigration.js @@ -0,0 +1,22 @@ +'use strict'; + +const mongoose = require('mongoose'); +const database = require('../app/lib/database-connection'); +const migration = require('../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); + +async function main() { + await database.initializeConnection(); + const report = await migration._private.run(mongoose.connection.db, { + dryRun: true, + }); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} + +main() + .catch((err) => { + process.stderr.write(`${err.stack || err.message}\n`); + process.exitCode = 1; + }) + .finally(async () => { + await mongoose.disconnect(); + }); From 9ba3bb7d17b6ea4a2ad0c976bdda5e12f4a85fa0 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:47:59 -0400 Subject: [PATCH 38/55] fix(release-tracks): scope graph migration to active relationships Ignore deprecated and revoked dangling relationship history during deterministic graph backfill while preserving strict validation for active relationships. Improve migration diagnostics and ensure manifest indexes are always established. --- .../deterministic-graph-migration.spec.js | 65 ++++++++++++++++++- docs/admin/release-track-graph-migration.md | 28 ++++---- docs/developer/TODO.md | 23 +++++++ ...-backfill-deterministic-snapshot-graphs.js | 40 ++++++++++-- ...viewDeterministicSnapshotGraphMigration.js | 9 +++ 5 files changed, 145 insertions(+), 20 deletions(-) diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 7a149c2e..9e4689e2 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -25,6 +25,9 @@ describe('Deterministic snapshot graph migration', function () { let group; let relationship; let trackId; + const deprecatedDanglingRelationshipId = 'relationship--f7a41277-6599-49df-9567-82c9227fb8b5'; + const activeDanglingRelationshipId = 'relationship--932fabf0-2868-46ed-9453-41e33dab7f39'; + const missingEndpointId = 'campaign--5f4e747c-11d7-49ae-a947-a0f436879d62'; before(async function () { await database.initializeConnection(); @@ -110,9 +113,65 @@ describe('Deterministic snapshot graph migration', function () { ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }), ReleaseTrackGraphManifestEntry.deleteMany({ track_id: trackId }), ]); + await mongoose.connection.db.collection('relationships').insertOne({ + workspace: {}, + stix: { + type: 'relationship', + spec_version: '2.1', + id: deprecatedDanglingRelationshipId, + created: new Date(timestamp), + modified: new Date(timestamp), + relationship_type: 'uses', + source_ref: missingEndpointId, + target_ref: technique.stix.id, + revoked: false, + x_mitre_deprecated: true, + object_marking_refs: [markingDefinitionId], + }, + }); + }); + + it('fails closed when an active latest relationship has a dangling endpoint', async function () { + const timestamp = new Date(); + await mongoose.connection.db.collection('relationships').insertOne({ + workspace: {}, + stix: { + type: 'relationship', + spec_version: '2.1', + id: activeDanglingRelationshipId, + created: timestamp, + modified: timestamp, + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: missingEndpointId, + revoked: false, + x_mitre_deprecated: false, + object_marking_refs: [markingDefinitionId], + }, + }); + + try { + await expect( + migration._private.run(mongoose.connection.db, { + dryRun: true, + }), + ).rejects.toMatchObject({ + message: expect.stringContaining(activeDanglingRelationshipId), + missing_relationship_endpoints: [ + expect.objectContaining({ + relationship_ref: activeDanglingRelationshipId, + missing_endpoints: [missingEndpointId], + }), + ], + }); + } finally { + await mongoose.connection.db + .collection('relationships') + .deleteOne({ 'stix.id': activeDanglingRelationshipId }); + } }); - it('supports a non-mutating dry run', async function () { + it('supports a non-mutating dry run with unrelated deprecated dangling data', async function () { const report = await migration._private.run(mongoose.connection.db, { dryRun: true, }); @@ -145,6 +204,10 @@ describe('Deterministic snapshot graph migration', function () { object_ref: technique.stix.id, object_modified: new Date(technique.stix.modified), }); + const deprecatedDanglingRelationship = await mongoose.connection.db + .collection('relationships') + .findOne({ 'stix.id': deprecatedDanglingRelationshipId }); + expect(deprecatedDanglingRelationship.workspace.relationship_endpoints).toBeUndefined(); const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId, diff --git a/docs/admin/release-track-graph-migration.md b/docs/admin/release-track-graph-migration.md index 227e562f..91855ca5 100644 --- a/docs/admin/release-track-graph-migration.md +++ b/docs/admin/release-track-graph-migration.md @@ -14,21 +14,25 @@ DATABASE_URL='mongodb://host/database' \ npm run preview:deterministic-snapshot-graphs ``` -The report includes the latest relationship revisions scanned, endpoint pins -that would be written, release-track snapshots found, and baseline manifests -that would be created. No database writes or indexes are created by this -command. - -The preview fails if a latest relationship references a source or target -object that no longer exists. Repair those dangling endpoints before -deployment. Snapshot graph capture fails closed rather than silently producing -an incomplete deterministic baseline. +The report includes the latest active relationship revisions scanned, endpoint +pins that would be written, release-track snapshots found, and baseline +manifests that would be created. No database writes or indexes are created by +this command. + +The preview fails if an active latest relationship references a source or +target object that no longer exists. The error identifies the affected +relationship and missing endpoint IDs; repair those dangling endpoints before +deployment. Deprecated and revoked relationships are not eligible for bundle +graphs, so the migration leaves that inactive legacy history untouched. +Snapshot graph capture fails closed rather than silently producing an +incomplete deterministic baseline. ## What the migration writes -- Exact source and target revision metadata is added only to the latest - revision of each relationship in the underlying `relationships` collection. - `view.relationships.latest` may be used for discovery but is never written. +- Exact source and target revision metadata is added only to active latest + relationship revisions in the underlying `relationships` collection. + `view.relationships.latest.active` may be used for discovery but is never + written. - Each existing release-track snapshot receives a graph manifest containing its exact primary, relationship, secondary, supporting, and LinkById dependencies. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 1ad59401..591677be 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -258,6 +258,29 @@ revision on which that bundle depends. Bruno request definitions require no transport change. - [x] Run focused specs while iterating, then lint, OpenAPI validation, and the complete `npm test` suite. + +### Production-shaped migration repair + +- [x] Scope legacy endpoint pinning to active latest relationships, matching + the relationship set eligible for deterministic snapshot graphs. +- [x] Preserve fail-closed behavior for active dangling relationships while + allowing deprecated or revoked dangling history to remain untouched. +- [x] Surface actionable missing-endpoint diagnostics in the migration preview + and startup failure. +- [x] Establish manifest indexes independently of whether relationship pin + updates happen to be required. +- [x] Add production-shaped migration regressions and update the operator + documentation. + +Verification result (2026-07-30): + +- The focused migration spec passes all 4 cases. +- A read-only preview against the restored production database scans 24,818 + active relationship revisions without encountering the 118 dangling + endpoints confined to deprecated relationship history. +- Lint, formatting, diff checks, and the complete `npm test` suite pass; the + API suite passes all 972 cases. + - [x] Propose conventional commits split by independently reviewable architectural slice; do not commit until requested. diff --git a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js index 2643e96d..ff5ba792 100644 --- a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js +++ b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js @@ -14,6 +14,11 @@ const TRACK_COLLECTION_PATTERN = /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const CONCURRENCY = 4; +const ACTIVE_RELATIONSHIP_FILTER = { + 'stix.x_mitre_deprecated': { $in: [null, false] }, + 'stix.revoked': { $in: [null, false] }, +}; +const ERROR_SAMPLE_LIMIT = 10; async function mapWithConcurrency(items, mapper) { let nextIndex = 0; @@ -29,11 +34,18 @@ async function mapWithConcurrency(items, mapper) { } async function latestRelationships(db) { + const latestActiveViewExists = await db + .listCollections({ name: 'view.relationships.latest.active' }, { nameOnly: true }) + .hasNext(); + if (latestActiveViewExists) { + return db.collection('view.relationships.latest.active').find({}).toArray(); + } + const latestViewExists = await db .listCollections({ name: 'view.relationships.latest' }, { nameOnly: true }) .hasNext(); if (latestViewExists) { - return db.collection('view.relationships.latest').find({}).toArray(); + return db.collection('view.relationships.latest').find(ACTIVE_RELATIONSHIP_FILTER).toArray(); } return db @@ -42,6 +54,7 @@ async function latestRelationships(db) { { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, { $replaceRoot: { newRoot: '$document' } }, + { $match: ACTIVE_RELATIONSHIP_FILTER }, ]) .toArray(); } @@ -109,6 +122,21 @@ async function buildRelationshipPinOperations(db) { return { relationships, operations, missing }; } +function missingEndpointError(missing) { + const sample = missing + .slice(0, ERROR_SAMPLE_LIMIT) + .map((entry) => `${entry.relationship_ref} -> ${entry.missing_endpoints.join(', ')}`) + .join('; '); + const remaining = missing.length - ERROR_SAMPLE_LIMIT; + const suffix = remaining > 0 ? `; and ${remaining} more` : ''; + const error = new Error( + `Cannot pin ${missing.length} active latest relationship(s) because referenced objects are ` + + `missing: ${sample}${suffix}`, + ); + error.missing_relationship_endpoints = missing; + return error; +} + async function findTrackIds(db) { const [registeredTracks, collections] = await Promise.all([ db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), @@ -206,17 +234,15 @@ async function backfillSnapshotManifests(db, options) { async function run(db, options = {}) { const relationshipPins = await buildRelationshipPinOperations(db); if (relationshipPins.missing.length > 0) { - const error = new Error( - 'Cannot pin latest relationship endpoints because one or more referenced objects are missing', - ); - error.missing_relationship_endpoints = relationshipPins.missing; - throw error; + throw missingEndpointError(relationshipPins.missing); } if (!options.dryRun && relationshipPins.operations.length > 0) { await db.collection('relationships').bulkWrite(relationshipPins.operations, { ordered: false, }); + } + if (!options.dryRun) { await ensureManifestIndexes(db); } const manifests = await backfillSnapshotManifests(db, options); @@ -233,7 +259,7 @@ module.exports = { async up(db) { const report = await run(db); console.log( - `Pinned ${report.relationship_pins_written} latest relationship revision(s) and ` + + `Pinned ${report.relationship_pins_written} active latest relationship revision(s) and ` + `created ${report.manifests_created} baseline snapshot manifest(s)`, ); }, diff --git a/scripts/previewDeterministicSnapshotGraphMigration.js b/scripts/previewDeterministicSnapshotGraphMigration.js index 348298e5..60ffb74b 100644 --- a/scripts/previewDeterministicSnapshotGraphMigration.js +++ b/scripts/previewDeterministicSnapshotGraphMigration.js @@ -15,6 +15,15 @@ async function main() { main() .catch((err) => { process.stderr.write(`${err.stack || err.message}\n`); + if (err.missing_relationship_endpoints) { + process.stderr.write( + `${JSON.stringify( + { missing_relationship_endpoints: err.missing_relationship_endpoints }, + null, + 2, + )}\n`, + ); + } process.exitCode = 1; }) .finally(async () => { From fee3f8be5f27b19d2a9adc9384bcf04718af717d Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:21:08 -0400 Subject: [PATCH 39/55] fix(release-tracks): allow ATT&CK-branded track names Permit ampersands in release-track names across request, persistence, and OpenAPI validation. Add regression coverage and document the accepted naming contract. --- .../definitions/components/release-tracks.yml | 6 ++++-- .../release-tracks/release-track-schemas.js | 4 ++-- .../release-track-validators.js | 3 ++- .../api/release-tracks/release-tracks.spec.js | 20 +++++++++++++++++++ docs/developer/TODO.md | 9 +++++++++ docs/developer/release-tracks/entities.md | 7 ++++--- 6 files changed, 41 insertions(+), 8 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index d973f9d4..91c09b91 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -31,7 +31,8 @@ components: with this snapshot. Clients should treat this value as opaque. name: type: string - description: 'Human-readable track name' + pattern: '^[a-zA-Z0-9 &]+$' + description: 'Human-readable track name containing alphanumeric characters, spaces, and ampersands' example: 'Enterprise ATT&CK' description: type: string @@ -523,7 +524,8 @@ components: description: 'Track type' name: type: string - description: 'Track name' + pattern: '^[a-zA-Z0-9 &]+$' + description: 'Track name containing alphanumeric characters, spaces, and ampersands' description: type: string description: 'Track description' diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index aaf9d8df..e0d3e49a 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -69,8 +69,8 @@ const releaseTrackIdSchema = createCustomStixIdValidator('release-track'); const trackNameSchema = z .string() .min(1, { message: 'Release track name must not be empty' }) - .regex(/^[a-zA-Z0-9 ]+$/, { - message: 'Release track name may only contain alphanumeric characters and spaces', + .regex(/^[a-zA-Z0-9 &]+$/, { + message: 'Release track name may only contain alphanumeric characters, spaces, and ampersands', }); // ----------------------------------------------------------------------------- diff --git a/app/lib/release-tracks/release-track-validators.js b/app/lib/release-tracks/release-track-validators.js index e8e6465e..24e771fa 100644 --- a/app/lib/release-tracks/release-track-validators.js +++ b/app/lib/release-tracks/release-track-validators.js @@ -34,7 +34,8 @@ const validateTrackId = { const validateTrackName = { validator: (v) => trackNameSchema.safeParse(v).success, message: (props) => - `"${props.value}" is not a valid release track name (only alphanumeric characters and spaces allowed)`, + `"${props.value}" is not a valid release track name ` + + '(only alphanumeric characters, spaces, and ampersands allowed)', }; const validateStixId = { diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 33079058..33c5175b 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -235,6 +235,26 @@ describe('Release Tracks API', function () { .expect(501); }); + it('accepts ATT&CK branding in release-track names', async function () { + const response = await request(app) + .post('/api/release-tracks/new') + .send({ + name: 'Enterprise ATT&CK', + description: 'Aggregate Enterprise ATT&CK release track.', + type: 'virtual', + snapshot_schedule: { mode: 'manual' }, + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201) + .expect('Content-Type', /json/); + + expect(response.body).toMatchObject({ + name: 'Enterprise ATT&CK', + type: 'virtual', + }); + }); + after(async function () { await database.closeConnection(); }); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 591677be..7d5f223d 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,14 @@ # Release Track TODOs +## Bootstrap hotfix — ATT&CK-branded track names + +- [x] Permit ampersands in release-track names at the request and persistence + validation boundaries. +- [x] Add an API regression using the production bootstrap name + `Enterprise ATT&CK`. +- [x] Align OpenAPI and developer naming documentation with the accepted + contract. + ## Production-readiness branch — `fix/release-tracks-production-readiness` This branch implements the prioritized findings in diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index e6794cb6..bc0743f2 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -10,9 +10,10 @@ This document tracks new database schemas, interfaces, etc.; as well as changes **Release Track Names:** -- Must contain only alphanumeric characters and spaces: `[a-zA-Z0-9 ]` -- No special characters allowed (no hyphens, underscores, or other punctuation) -- Examples: `Enterprise`, `Groups Monthly`, `Techniques Quarterly` +- May contain alphanumeric characters, spaces, and ampersands: + `[a-zA-Z0-9 &]` +- Other punctuation remains unsupported, including hyphens and underscores. +- Examples: `Enterprise`, `Groups Monthly`, `Enterprise ATT&CK` **Release Track IDs:** MongoDB Collections and release track IDs follow a simple naming convention: From 1cdc18f3b9d0d4da689dcf1c99031ac6f708272d Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 19:49:33 -0400 Subject: [PATCH 40/55] feat(release-tracks): accept config when creating tracks Validate caller-supplied configuration using the existing track config contract and persist it on the initial release-track snapshot. --- .../paths/release-tracks-paths.yml | 18 ++++---- .../release-tracks/release-track-schemas.js | 28 ++++++------ .../release-tracks/snapshot-service.js | 4 +- .../api/release-tracks/release-tracks.spec.js | 36 +++++++++++++++ docs/developer/TODO.md | 20 +++++++++ .../release-tracks/implementation-notes.md | 5 +++ docs/user/release-tracks/api-reference.md | 45 +++++++++++++++++-- 7 files changed, 128 insertions(+), 28 deletions(-) diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 4a001ef4..39884070 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -223,14 +223,16 @@ paths: operationId: 'release-tracks-create' description: | Create a new standard or virtual release track with an initial empty draft snapshot. - Request body is validated via Zod (not OpenAPI). Virtual composition - objects are strict, and component selectors must match their - resolution_strategy. Component IDs and priorities must be unique, - every priority is required, and referenced components must already - exist as standard tracks; virtual-track nesting and native members are - unsupported. Virtual snapshot schedules are strict by mode: manual - accepts no selector, cron requires cron, and dates requires at least - one date. Standard tracks reject snapshot_schedule. + Request body is validated via Zod (not OpenAPI). An optional config + object accepts the same fields and validation rules as PUT + /api/release-tracks/:id/config; omitted values receive model defaults. + Virtual composition objects are strict, and component selectors must + match their resolution_strategy. Component IDs and priorities must be + unique, every priority is required, and referenced components must + already exist as standard tracks; virtual-track nesting and native + members are unsupported. Virtual snapshot schedules are strict by + mode: manual accepts no selector, cron requires cron, and dates + requires at least one date. Standard tracks reject snapshot_schedule. tags: - 'Release Tracks' # Request body validation moved to Zod in controller diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index e0d3e49a..58c151f5 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -253,6 +253,19 @@ const memberSyncConfigSchema = z.object({ supplant: memberSyncSupplantSchema.optional(), }); +const promotionConflictsSchema = z.object({ + into_candidates: conflictPolicySchema.optional(), + candidates_to_staged: conflictPolicySchema.exclude(['abort']).optional(), + staged_to_members: conflictPolicySchema.optional(), +}); + +const updateConfigBodySchema = z.object({ + candidacy_threshold: candidacyThresholdSchema.optional(), + auto_promote: z.boolean().optional(), + promotion_conflicts: promotionConflictsSchema.optional(), + member_sync: memberSyncConfigSchema.optional(), +}); + // ============================================================================= // Request body schemas (used inline by controller handlers) // ============================================================================= @@ -372,6 +385,7 @@ const createTrackBodySchema = z object_marking_refs: z.array(stixIdentifierSchema).optional(), composition: compositionSchema.optional(), snapshot_schedule: snapshotScheduleSchema.optional(), + config: updateConfigBodySchema.optional(), }) .strict() .superRefine((track, context) => { @@ -471,20 +485,6 @@ const updateCandidateVersionBodySchema = z.object({ new_modified: z.iso.datetime().or(z.literal('latest')), }); -/** PUT /release-tracks/:id/config */ -const promotionConflictsSchema = z.object({ - into_candidates: conflictPolicySchema.optional(), - candidates_to_staged: conflictPolicySchema.exclude(['abort']).optional(), - staged_to_members: conflictPolicySchema.optional(), -}); - -const updateConfigBodySchema = z.object({ - candidacy_threshold: candidacyThresholdSchema.optional(), - auto_promote: z.boolean().optional(), - promotion_conflicts: promotionConflictsSchema.optional(), - member_sync: memberSyncConfigSchema.optional(), -}); - /** PUT /release-tracks/:id/virtual/composition */ const updateCompositionBodySchema = compositionSchema; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index ddf8e334..1a4ba38d 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -167,7 +167,7 @@ exports.listTracks = async function listTracks(options) { /** * Create a new release track with an initial empty draft snapshot. * - * @param {Object} data - { name, description?, type, userAccountId?, object_marking_refs?, composition?, snapshot_schedule? } + * @param {Object} data - { name, description?, type, userAccountId?, object_marking_refs?, composition?, snapshot_schedule?, config? } * @returns {Promise} The initial snapshot document */ exports.createTrack = async function createTrack(data) { @@ -190,7 +190,7 @@ exports.createTrack = async function createTrack(data) { candidates: trackType === 'standard' ? [] : undefined, quarantine: trackType === 'virtual' ? [] : undefined, composition: trackType === 'virtual' ? data.composition : undefined, - config: {}, + config: data.config || {}, version_history: [], }; diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 33c5175b..5aef3bc0 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -255,6 +255,42 @@ describe('Release Tracks API', function () { }); }); + it('creates a release track with caller-supplied config', async function () { + const suppliedConfig = { + candidacy_threshold: 'awaiting-review', + auto_promote: false, + promotion_conflicts: { + into_candidates: 'always_reject', + candidates_to_staged: 'always_overwrite', + staged_to_members: 'prefer_latest', + }, + member_sync: { + strategy: 'manual', + supplant: { + behavior: 'queue', + status_policy: 'preserve', + }, + }, + }; + + const response = await request(app) + .post('/api/release-tracks/new') + .send({ + name: 'Custom Config Track', + type: 'standard', + config: suppliedConfig, + }) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201) + .expect('Content-Type', /json/); + + expect(response.body.config).toEqual(suppliedConfig); + + const persistedSnapshot = await snapshotService.getLatestSnapshot(response.body.id); + expect(persistedSnapshot.config).toEqual(suppliedConfig); + }); + after(async function () { await database.closeConnection(); }); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 7d5f223d..84a743de 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,25 @@ # Release Track TODOs +## Caller-supplied configuration on track creation + +- [x] Add a regression proving `POST /api/release-tracks/new` accepts and + persists supported `config` options on the initial snapshot. +- [x] Reuse the release-track config validation contract in the create request + and pass the validated config through the snapshot creation service. +- [x] Update OpenAPI guidance, user documentation, and the Bruno request. +- [ ] Run the focused regression and the complete `npm test` suite. + +Verification (2026-07-30): + +- Focused release-track API regression passes: 3 cases. +- Backend lint and OpenAPI validation pass. +- The complete suite was run and reached API 975 passing with four failures + in unrelated, pre-existing work: three canonical-domain migration failures + and one roaming virtual-composition failure. +- The virtual-composition spec passes in isolation. The in-progress + canonical-domain migration spec still has three isolated failures, so a + clean aggregate run remains outstanding. + ## Bootstrap hotfix — ATT&CK-branded track names - [x] Permit ampersands in release-track names at the request and persistence diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 449bfbf7..791dd6af 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -28,6 +28,11 @@ supported deployments. ## Validation Rules +- `POST /api/release-tracks/new` and `PUT /api/release-tracks/:id/config` + share the same Zod configuration schema. Creation passes the parsed config + directly into the initial snapshot so Mongoose applies defaults only to + omitted options instead of replacing caller-supplied values with an empty + config. - **Same revision selector** can only be in one tier per release-track snapshot (`members`, `staged`, `candidates`, or `quarantine`) - **Different selectors** for the same object CAN exist in multiple tiers simultaneously diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 9a1bc26f..12e06363 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -236,11 +236,32 @@ POST /api/release-tracks/new { "name": "Release Track Name", "description": "Description", - "external_references": [], - "object_marking_refs": [] + "type": "standard", + "object_marking_refs": [], + "config": { + "candidacy_threshold": "awaiting-review", + "auto_promote": false, + "promotion_conflicts": { + "into_candidates": "always_reject", + "candidates_to_staged": "prefer_latest", + "staged_to_members": "abort" + }, + "member_sync": { + "strategy": "manual", + "supplant": { + "behavior": "queue", + "status_policy": "preserve" + } + } + } } ``` +`config` is optional. When supplied, it uses the same fields and validation +rules as [Update Configuration](#update-configuration), and the validated +values are persisted on the initial draft snapshot. Omitted config fields use +their model defaults. + ### Bootstrap Release Track From Bundle Creates a new release track initialized with objects from a STIX bundle. This is useful for importing existing collections or bootstrapping from published ATT&CK releases. @@ -838,11 +859,27 @@ PUT /api/release-tracks/:id/config ```json { - "candidacy_threshold": "work-in-progress" | "awaiting-review" | "reviewed", - "auto_promote": true | false + "candidacy_threshold": "awaiting-review", + "auto_promote": true, + "promotion_conflicts": { + "into_candidates": "prefer_latest", + "candidates_to_staged": "prefer_latest", + "staged_to_members": "abort" + }, + "member_sync": { + "strategy": "track_latest", + "supplant": { + "behavior": "replace", + "status_policy": "reset" + } + } } ``` +All fields are optional. Configuration updates merge with the latest draft; +nested `promotion_conflicts` and `member_sync.supplant` values are also +merged. The same configuration object may be supplied when creating a track. + --- ## Release Previews From ee5651058842ec00e0dc4183e7e484f51e8c378c Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:16:55 -0400 Subject: [PATCH 41/55] fix(migrations): make canonical domain backfill release agnostic Infer domain unions from persisted canonical collection provenance and cover every domain-bearing ATT&CK type. Let the native migration driver generate inactive clone IDs to prevent cross-version BSON failures. --- app/lib/automation-run-recorder.js | 24 +- app/lib/default-bypass-rules.json | 28 - app/models/campaign-model.js | 1 + .../release-tracks/member-sync-service.js | 52 +- app/services/stix/bundle-graph-resolver.js | 33 +- .../canonical-domain-migration.spec.js | 587 ++++++++++++++ .../release-tracks/ephemeral-bundle.spec.js | 30 +- .../virtual-domain-filters.spec.js | 21 +- .../api/stix-bundles/stix-bundles.spec.js | 15 +- docs/README.md | 1 + docs/admin/canonical-domain-migration.md | 143 ++++ docs/developer/FRONTEND_TODO.md | 40 + docs/developer/TODO.md | 77 ++ docs/developer/data-model.md | 20 +- .../developer/release-tracks/bundle-export.md | 23 +- .../release-tracks/implementation-notes.md | 102 ++- docs/user/release-tracks/api-reference.md | 20 +- docs/user/release-tracks/summary.md | 8 + docs/user/release-tracks/virtual-tracks.md | 25 +- ...0000-backfill-canonical-x-mitre-domains.js | 726 ++++++++++++++++++ 20 files changed, 1875 insertions(+), 101 deletions(-) create mode 100644 app/tests/api/release-tracks/canonical-domain-migration.spec.js create mode 100644 docs/admin/canonical-domain-migration.md create mode 100644 migrations/20260730230000-backfill-canonical-x-mitre-domains.js diff --git a/app/lib/automation-run-recorder.js b/app/lib/automation-run-recorder.js index b5acd1ac..f1f44b47 100644 --- a/app/lib/automation-run-recorder.js +++ b/app/lib/automation-run-recorder.js @@ -81,17 +81,31 @@ class AutomationRunRecorder { } async recordItem(item) { - this.sequence += 1; + await this.recordItems([item]); + } - await this.itemsCollection.insertOne({ + /** + * Persist multiple audit items in one database operation while retaining + * the same stable, monotonically increasing sequence contract as + * recordItem(). + * + * @param {Array} items + */ + async recordItems(items) { + if (!Array.isArray(items) || items.length === 0) return; + + const recordedAt = new Date(); + const documents = items.map((item) => ({ schema_version: AUTOMATION_RUN_SCHEMA_VERSION, run_id: this.runId, automation_type: this.automationType, name: this.name, - recorded_at: new Date(), - sequence: this.sequence, + recorded_at: recordedAt, + sequence: ++this.sequence, ...item, - }); + })); + + await this.itemsCollection.insertMany(documents, { ordered: true }); } async finish({ status, counts, warnings, verification, summary, errorSummary }) { diff --git a/app/lib/default-bypass-rules.json b/app/lib/default-bypass-rules.json index 685be3c0..3e307287 100644 --- a/app/lib/default-bypass-rules.json +++ b/app/lib/default-bypass-rules.json @@ -125,33 +125,5 @@ "suppressError": false, "warningMessage": "Tactic shortname does not match predefined ATT&CK tactics. This may prevent compatibility with official ATT&CK data but can be used for custom taxonomies.", "_comment": "Warn about non-standard tactic shortnames instead of blocking" - }, - { - "fieldPath": ["x_mitre_domains"], - "errorCode": "invalid_type", - "stixType": "intrusion-set", - "suppressError": true, - "_comment": "Server sets x_mitre_domains for intrusion-set (assigned during bundle export)" - }, - { - "fieldPath": ["x_mitre_domains"], - "errorCode": "invalid_type", - "stixType": "campaign", - "suppressError": true, - "_comment": "Server sets x_mitre_domains for campaign (assigned during bundle export)" - }, - { - "fieldPath": ["x_mitre_domains"], - "errorCode": "invalid_type", - "stixType": "x-mitre-matrix", - "suppressError": true, - "_comment": "Server sets x_mitre_domains for x-mitre-matrix (assigned during bundle export)" - }, - { - "fieldPath": ["x_mitre_domains"], - "errorCode": "invalid_type", - "stixType": "x-mitre-detection-strategy", - "suppressError": true, - "_comment": "Server sets x_mitre_domains for x-mitre-detection-strategy (assigned during bundle export)" } ] diff --git a/app/models/campaign-model.js b/app/models/campaign-model.js index 2112f749..df360e68 100644 --- a/app/models/campaign-model.js +++ b/app/models/campaign-model.js @@ -19,6 +19,7 @@ const stixCampaign = { x_mitre_last_seen_citation: String, x_mitre_modified_by_ref: String, x_mitre_deprecated: { type: Boolean, required: true, default: false }, + x_mitre_domains: { type: [String], default: undefined }, x_mitre_version: String, x_mitre_attack_spec_version: String, x_mitre_contributors: { type: [String], default: undefined }, diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index 282e7a8b..2caa2940 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -40,6 +40,30 @@ const logger = require('../../lib/logger'); const EventBus = require('../../lib/event-bus'); const EventConstants = require('../../lib/event-constants'); +// Concurrent object creates can affect the same standard track. Snapshot +// updates are read-modify-write operations, so serialize them per track while +// still allowing unrelated tracks to progress concurrently. +const trackLocks = new Map(); + +async function withTrackLock(trackId, operation) { + const previous = trackLocks.get(trackId) || Promise.resolve(); + let release; + const current = new Promise((resolve) => { + release = resolve; + }); + trackLocks.set(trackId, current); + + await previous; + try { + return await operation(); + } finally { + release(); + if (trackLocks.get(trackId) === current) { + trackLocks.delete(trackId); + } + } +} + // ============================================================================= // Main entry point // ============================================================================= @@ -75,12 +99,27 @@ exports.handleObjectModified = async function handleObjectModified(event) { const results = []; for (const trackInfo of affectedTracks) { try { - const result = await processMemberSync(trackInfo.trackId, trackInfo.snapshot, { - objectRef, - newModified, - modifiedBy, - trigger, - isMember: trackInfo.isMember, + const result = await withTrackLock(trackInfo.trackId, async () => { + // Discovery may have happened while another object was cloning this + // track. Refresh inside the lock so this mutation always builds on the + // authoritative latest snapshot instead of overwriting its peer. + const snapshot = await dynamicRepo.getLatestSnapshot(trackInfo.trackId); + if (!snapshot) return null; + + const isMember = (snapshot.members || []).some((entry) => entry.object_ref === objectRef); + const isTracked = + isMember || + (snapshot.candidates || []).some((entry) => entry.object_ref === objectRef) || + (snapshot.staged || []).some((entry) => entry.object_ref === objectRef); + if (!isTracked) return null; + + return processMemberSync(trackInfo.trackId, snapshot, { + objectRef, + newModified, + modifiedBy, + trigger, + isMember, + }); }); if (result) results.push(result); } catch (err) { @@ -586,6 +625,7 @@ initializeEventListeners(); exports._internal = { findTracksReferencingObject, processMemberSync, + withTrackLock, getMemberSyncConfig, handleStixObjectEvent, handleStixObjectRevokedEvent, diff --git a/app/services/stix/bundle-graph-resolver.js b/app/services/stix/bundle-graph-resolver.js index 9374bafb..bde27257 100644 --- a/app/services/stix/bundle-graph-resolver.js +++ b/app/services/stix/bundle-graph-resolver.js @@ -291,11 +291,10 @@ class BundleGraphResolver { this.options.inferDomains !== false && (secondaryObject.stix.type === 'intrusion-set' || secondaryObject.stix.type === 'campaign') ) { - if (secondaryObject.stix.x_mitre_domains) { - this.domainCache.set(secondaryObject.stix.id, secondaryObject.stix.x_mitre_domains); + if (!this.rememberCanonicalDomains(secondaryObject)) { + secondaryObject.stix.x_mitre_domains = + await this.getDomainsForSecondaryObject(secondaryObject); } - secondaryObject.stix.x_mitre_domains = - await this.getDomainsForSecondaryObject(secondaryObject); } return true; } @@ -379,7 +378,7 @@ class BundleGraphResolver { } } } - this.rememberAndSetDomains(detectionStrategyDoc, [this.options.domain]); + this.setFallbackDomains(detectionStrategyDoc, [this.options.domain]); this.addAttackObject(detectionStrategyDoc, objects, objectsMap); } } @@ -403,7 +402,7 @@ class BundleGraphResolver { groupObject, this.endpointDocument(objectsMap, relationship, 'source'), ); - this.rememberAndSetDomains(groupObject, [this.options.domain]); + this.setFallbackDomains(groupObject, [this.options.domain]); this.addAttackObject(groupObject, objects, objectsMap); } } @@ -426,7 +425,7 @@ class BundleGraphResolver { detectionStrategy, this.endpointDocument(objectsMap, relationship, 'target'), ); - this.rememberAndSetDomains(detectionStrategy, [this.options.domain]); + this.setFallbackDomains(detectionStrategy, [this.options.domain]); this.addAttackObject(detectionStrategy, objects, objectsMap); } } @@ -450,19 +449,21 @@ class BundleGraphResolver { this.endpointDocument(objectsMap, relationship, 'target'), ); if (revokedObject.stix.type === 'intrusion-set' || revokedObject.stix.type === 'campaign') { - this.rememberAndSetDomains(revokedObject, [this.options.domain]); + this.setFallbackDomains(revokedObject, [this.options.domain]); } this.addAttackObject(revokedObject, objects, objectsMap); } - rememberAndSetDomains(attackObject, domains) { - if (this.options.inferDomains === false) { - return; - } - if (attackObject.stix.x_mitre_domains) { - this.domainCache.set(attackObject.stix.id, attackObject.stix.x_mitre_domains); - } - attackObject.stix.x_mitre_domains = domains; + rememberCanonicalDomains(attackObject) { + const domains = attackObject.stix.x_mitre_domains; + if (!Array.isArray(domains) || domains.length === 0) return false; + this.domainCache.set(attackObject.stix.id, domains); + return true; + } + + setFallbackDomains(attackObject, domains) { + if (this.options.inferDomains === false || this.rememberCanonicalDomains(attackObject)) return; + attackObject.stix.x_mitre_domains = [...new Set(domains)]; } } diff --git a/app/tests/api/release-tracks/canonical-domain-migration.spec.js b/app/tests/api/release-tracks/canonical-domain-migration.spec.js new file mode 100644 index 00000000..20e75c7b --- /dev/null +++ b/app/tests/api/release-tracks/canonical-domain-migration.spec.js @@ -0,0 +1,587 @@ +'use strict'; + +const mongoose = require('mongoose'); +const { MongoClient } = require('mongodb'); +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const migration = require('../../../../migrations/20260730230000-backfill-canonical-x-mitre-domains'); +const defaultBypassRules = require('../../../lib/default-bypass-rules.json'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; +const collectionIds = { + enterprise: 'x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019', + ics: 'x-mitre-collection--90c00720-636b-4485-b342-8751d232bf09', + mobile: 'x-mitre-collection--dac0d2d7-8653-445c-9bff-82f934c1e858', +}; +const objectFixtures = [ + { + path: '/api/techniques', + id: 'attack-pattern--10000000-0000-4000-8000-000000000001', + type: 'attack-pattern', + name: 'Active migration technique', + lifecycle: 'active', + collectionRefs: [collectionIds.mobile], + expectedDomains: ['mobile-attack'], + }, + { + path: '/api/groups', + id: 'intrusion-set--00f67a77-86a4-4adf-be26-1a54fc713340', + type: 'intrusion-set', + name: 'Active migration group', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise, collectionIds.mobile], + expectedDomains: ['enterprise-attack', 'mobile-attack'], + }, + { + path: '/api/campaigns', + id: 'campaign--0257b35b-93ef-4a70-80dd-ad5258e6045b', + type: 'campaign', + name: 'Active migration campaign', + lifecycle: 'active', + collectionRefs: [collectionIds.ics], + expectedDomains: ['ics-attack'], + }, + { + path: '/api/mitigations', + id: 'course-of-action--10000000-0000-4000-8000-000000000002', + type: 'course-of-action', + name: 'Active migration mitigation', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise], + expectedDomains: ['enterprise-attack'], + }, + { + path: '/api/software', + id: 'malware--10000000-0000-4000-8000-000000000003', + type: 'malware', + name: 'Active migration malware', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise, collectionIds.ics], + expectedDomains: ['enterprise-attack', 'ics-attack'], + }, + { + path: '/api/software', + id: 'tool--10000000-0000-4000-8000-000000000004', + type: 'tool', + name: 'Active migration tool', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise], + expectedDomains: ['enterprise-attack'], + }, + { + path: '/api/analytics', + id: 'x-mitre-analytic--10000000-0000-4000-8000-000000000005', + type: 'x-mitre-analytic', + name: 'Active migration analytic', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise], + expectedDomains: ['enterprise-attack'], + }, + { + path: '/api/assets', + id: 'x-mitre-asset--10000000-0000-4000-8000-000000000006', + type: 'x-mitre-asset', + name: 'Active migration asset', + lifecycle: 'active', + collectionRefs: [collectionIds.ics], + expectedDomains: ['ics-attack'], + }, + { + path: '/api/data-components', + id: 'x-mitre-data-component--10000000-0000-4000-8000-000000000007', + type: 'x-mitre-data-component', + name: 'Active migration data component', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise], + expectedDomains: ['enterprise-attack'], + }, + { + path: '/api/data-sources', + id: 'x-mitre-data-source--10000000-0000-4000-8000-000000000008', + type: 'x-mitre-data-source', + name: 'Active migration data source', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise], + expectedDomains: ['enterprise-attack'], + }, + { + path: '/api/detection-strategies', + id: 'x-mitre-detection-strategy--00060b87-7f99-45aa-9553-a4d94139195c', + type: 'x-mitre-detection-strategy', + name: 'Revoked migration detection strategy', + lifecycle: 'revoked', + collectionRefs: [collectionIds.enterprise, collectionIds.mobile], + expectedDomains: ['enterprise-attack', 'mobile-attack'], + }, + { + path: '/api/matrices', + id: 'x-mitre-matrix--eafc1b4c-5e56-4965-bd4e-66a6a89c88cc', + type: 'x-mitre-matrix', + name: 'Deprecated migration matrix', + lifecycle: 'deprecated', + collectionRefs: [collectionIds.ics], + expectedDomains: ['ics-attack'], + }, + { + path: '/api/tactics', + id: 'x-mitre-tactic--10000000-0000-4000-8000-000000000009', + type: 'x-mitre-tactic', + name: 'Active migration tactic', + lifecycle: 'active', + collectionRefs: [collectionIds.enterprise, collectionIds.ics, collectionIds.mobile], + expectedDomains: ['enterprise-attack', 'ics-attack', 'mobile-attack'], + }, +]; +const groupFixture = objectFixtures.find((fixture) => fixture.type === 'intrusion-set'); +const campaignFixture = objectFixtures.find((fixture) => fixture.type === 'campaign'); + +describe('Canonical ATT&CK domain migration', function () { + let app; + let migrationClient; + let migrationDb; + let passportCookie; + const created = new Map(); + let trackId; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + const migrationUri = + `mongodb://${mongoose.connection.host}:${mongoose.connection.port}/` + + mongoose.connection.name; + migrationClient = new MongoClient(migrationUri); + await migrationClient.connect(); + migrationDb = migrationClient.db(mongoose.connection.name); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + before('create representative legacy revisions and a member track', async function () { + for (const fixture of objectFixtures) { + const timestamp = new Date().toISOString(); + const stix = { + type: fixture.type, + id: fixture.id, + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: fixture.name, + x_mitre_deprecated: false, + object_marking_refs: [markingDefinitionId], + }; + if (fixture.type === 'x-mitre-matrix') { + stix.external_references = [ + { + source_name: 'mitre-attack', + external_id: 'enterprise-attack', + }, + ]; + } + + const document = await post(fixture.path, { + workspace: { workflow: { state: 'work-in-progress' } }, + stix, + }); + created.set(fixture.id, document); + const provenanceResult = await mongoose.connection.db.collection('attackObjects').updateOne( + { 'stix.id': fixture.id, 'stix.modified': new Date(document.stix.modified) }, + { + $set: { + 'workspace.collections': fixture.collectionRefs.map((collectionRef) => ({ + collection_ref: collectionRef, + collection_modified: new Date('2026-01-01T00:00:00.000Z'), + })), + }, + }, + ); + expect(provenanceResult.matchedCount).toBe(1); + } + + const revokedFixture = objectFixtures.find((fixture) => fixture.lifecycle === 'revoked'); + await mongoose.connection.db.collection('attackObjects').updateOne( + { 'stix.id': revokedFixture.id }, + { + $set: { + 'stix.revoked': true, + 'workspace.release_tracks': [ + { + id: 'release-track--ffffffff-ffff-4fff-8fff-ffffffffffff', + type: 'standard', + tier: 'members', + status: 'reviewed', + }, + ], + 'workspace.validation': { + errors: [ + { + message: 'x_mitre_domains is required', + path: ['x_mitre_domains'], + code: 'invalid_type', + }, + { + message: 'another retained issue', + path: ['description'], + code: 'invalid_type', + }, + ], + }, + }, + }, + ); + const deprecatedFixture = objectFixtures.find((fixture) => fixture.lifecycle === 'deprecated'); + await mongoose.connection.db.collection('attackObjects').updateOne( + { 'stix.id': deprecatedFixture.id }, + { + $set: { + 'stix.x_mitre_deprecated': true, + 'workspace.validation': { + errors: [ + { + message: 'x_mitre_domains is required', + path: ['x_mitre_domains'], + code: 'invalid_type', + }, + ], + }, + }, + }, + ); + + const track = await post('/api/release-tracks/new', { + name: 'Domain migration track', + type: 'standard', + }); + trackId = track.id; + const group = created.get(groupFixture.id); + const campaign = created.get(campaignFixture.id); + const memberSeed = await mongoose.connection.db.collection(trackId).updateOne( + { id: trackId, modified: new Date(track.modified) }, + { + $set: { + members: [ + { + object_ref: group.stix.id, + object_modified: new Date(group.stix.modified), + }, + { + object_ref: campaign.stix.id, + object_modified: new Date(campaign.stix.modified), + }, + ], + }, + }, + ); + expect(memberSeed.matchedCount).toBe(1); + }); + + it('rejects reviewed objects that omit required ATT&CK domains', async function () { + const timestamp = new Date().toISOString(); + const response = await post( + '/api/groups', + { + workspace: { workflow: { state: 'reviewed' } }, + stix: { + type: 'intrusion-set', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Domainless reviewed group', + aliases: ['Domainless reviewed group'], + x_mitre_deprecated: false, + x_mitre_version: '1.0', + object_marking_refs: [markingDefinitionId], + }, + }, + 400, + ); + + expect(JSON.stringify(response)).toContain('x_mitre_domains'); + }); + + it('does not seed missing-domain validation bypasses', function () { + const retiredRules = defaultBypassRules.filter( + (rule) => + rule.errorCode === 'invalid_type' && + rule.fieldPath?.join('.') === 'x_mitre_domains' && + migration._private.TARGET_TYPES.includes(rule.stixType), + ); + expect(retiredRules).toEqual([]); + }); + + it('covers every domain-bearing ATT&CK object type and infers domain unions from provenance', function () { + expect(migration._private.TARGET_TYPES).toEqual([ + 'attack-pattern', + 'campaign', + 'course-of-action', + 'intrusion-set', + 'malware', + 'tool', + 'x-mitre-analytic', + 'x-mitre-asset', + 'x-mitre-data-component', + 'x-mitre-data-source', + 'x-mitre-detection-strategy', + 'x-mitre-matrix', + 'x-mitre-tactic', + ]); + + expect( + migration._private.domainsFromCollectionProvenance({ + workspace: { + collections: [ + { collection_ref: collectionIds.mobile }, + { collection_ref: collectionIds.enterprise }, + { collection_ref: 'x-mitre-collection--ffffffff-ffff-4fff-8fff-ffffffffffff' }, + ], + }, + }), + ).toEqual(['enterprise-attack', 'mobile-attack']); + }); + + it('leaves inactive clone ids to the native database driver', async function () { + const original = await mongoose.connection.db + .collection('attackObjects') + .findOne({ 'stix.id': objectFixtures[0].id }); + const prepared = migration._private.prepareInactiveClone({ + document: original, + domains: ['enterprise-attack'], + }); + + expect(Object.prototype.hasOwnProperty.call(prepared.document, '_id')).toBe(false); + }); + + it('chunks work and caps active service concurrency', async function () { + const work = Array.from({ length: migration._private.BATCH_SIZE * 2 + 1 }, (_, index) => index); + expect(migration._private.chunkItems(work).map((batch) => batch.length)).toEqual([ + migration._private.BATCH_SIZE, + migration._private.BATCH_SIZE, + 1, + ]); + + let active = 0; + let maximumActive = 0; + const results = await migration._private.mapWithConcurrency( + work.slice(0, 12), + migration._private.ACTIVE_CONCURRENCY, + async (value) => { + active++; + maximumActive = Math.max(maximumActive, active); + await new Promise((resolve) => setTimeout(resolve, 2)); + active--; + return value * 2; + }, + ); + + expect(maximumActive).toBe(migration._private.ACTIVE_CONCURRENCY); + expect(results).toEqual(work.slice(0, 12).map((value) => value * 2)); + }); + + it('backfills active and inactive revisions without mutating history or lifecycle state', async function () { + await mongoose.connection.db.collection('validationbypassrules').insertMany( + migration._private.TARGET_TYPES.map((stixType) => ({ + fieldPath: ['x_mitre_domains'], + errorCode: 'invalid_type', + stixType, + suppressError: true, + })), + ); + + const report = await migration._private.run(migrationDb, migrationClient); + + expect(report.counts).toMatchObject({ + scanned_candidates: 13, + active_reposts: 11, + inactive_clones: 2, + active_batches: 2, + inactive_batches: 1, + revoked: 1, + deprecated: 1, + bypasses_removed: 13, + updated: 13, + failed: 0, + }); + expect(report.verification).toEqual({ + remaining_latest_domainless_target_objects: 0, + remaining_domain_validation_bypasses: 0, + }); + + for (const fixture of objectFixtures) { + const revisions = await mongoose.connection.db + .collection('attackObjects') + .find({ 'stix.id': fixture.id }) + .sort({ 'stix.modified': -1 }) + .toArray(); + + expect(revisions).toHaveLength(2); + expect(revisions[0].stix.x_mitre_domains).toEqual(fixture.expectedDomains); + expect(revisions[1].stix.x_mitre_domains).toBeUndefined(); + expect(new Date(revisions[0].stix.modified).getTime()).toBeGreaterThan( + new Date(revisions[1].stix.modified).getTime(), + ); + expect(revisions[0].stix.revoked === true).toBe(fixture.lifecycle === 'revoked'); + expect(revisions[0].stix.x_mitre_deprecated === true).toBe( + fixture.lifecycle === 'deprecated', + ); + + if (fixture.lifecycle === 'revoked') { + expect(revisions[0].workspace.release_tracks).toBeUndefined(); + expect(revisions[0].workspace.validation.errors).toEqual([ + expect.objectContaining({ path: ['description'] }), + ]); + expect(revisions[1].workspace.release_tracks).toHaveLength(1); + expect(revisions[1].workspace.validation.errors).toHaveLength(2); + } + if (fixture.lifecycle === 'deprecated') { + expect(revisions[0].workspace.validation).toBeUndefined(); + expect(revisions[1].workspace.validation.errors).toHaveLength(1); + } + } + + const latestTrackSnapshot = await mongoose.connection.db + .collection(trackId) + .findOne({}, { sort: { modified: -1 } }); + expect(latestTrackSnapshot.candidates).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + object_ref: groupFixture.id, + object_modified: 'latest', + }), + expect.objectContaining({ + object_ref: campaignFixture.id, + object_modified: 'latest', + }), + ]), + ); + + const completedRun = await mongoose.connection.db + .collection('automationRuns') + .findOne( + { name: '20260730230000-backfill-canonical-x-mitre-domains' }, + { sort: { started_at: -1 } }, + ); + const auditItems = await mongoose.connection.db + .collection('automationRunItems') + .find({ run_id: completedRun.run_id }) + .sort({ sequence: 1 }) + .toArray(); + expect(auditItems).toHaveLength(13); + expect(auditItems.map((item) => item.sequence)).toEqual( + Array.from({ length: 13 }, (_, index) => index + 1), + ); + }); + + it('is idempotent after canonical revisions and bypass removal are complete', async function () { + const report = await migration._private.run(migrationDb, migrationClient); + expect(report.counts.scanned_candidates).toBe(0); + expect(report.counts.updated).toBe(0); + expect(report.counts.bypasses_removed).toBe(0); + expect(await migration._private.countRemainingDomainlessTargets(migrationDb)).toBe(0); + }); + + it('defaults unmapped active and inactive domainless objects to Enterprise', async function () { + const unknownActiveId = 'intrusion-set--ffffffff-ffff-4fff-8fff-ffffffffffff'; + const unknownInactiveId = 'campaign--eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee'; + const now = new Date(); + await mongoose.connection.db.collection('attackObjects').insertMany([ + { + __t: 'Intrusion-Set', + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: unknownActiveId, + type: 'intrusion-set', + spec_version: '2.1', + created: now, + modified: now, + name: 'Unsupported custom group', + revoked: false, + x_mitre_deprecated: false, + }, + }, + { + __t: 'Campaign', + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: unknownInactiveId, + type: 'campaign', + spec_version: '2.1', + created: now, + modified: now, + name: 'Unsupported deprecated campaign', + revoked: false, + x_mitre_deprecated: true, + }, + }, + ]); + await mongoose.connection.db.collection('validationbypassrules').insertOne({ + fieldPath: ['x_mitre_domains'], + errorCode: 'invalid_type', + stixType: 'intrusion-set', + suppressError: true, + }); + + const report = await migration._private.run(migrationDb, migrationClient); + expect(report.counts).toMatchObject({ + scanned_candidates: 2, + active_reposts: 1, + inactive_clones: 1, + enterprise_defaults: 2, + deprecated: 1, + updated: 2, + failed: 0, + bypasses_removed: 1, + }); + + for (const stixId of [unknownActiveId, unknownInactiveId]) { + const revisions = await mongoose.connection.db + .collection('attackObjects') + .find({ 'stix.id': stixId }) + .sort({ 'stix.modified': -1 }) + .toArray(); + expect(revisions).toHaveLength(2); + expect(revisions[0].stix.x_mitre_domains).toEqual(['enterprise-attack']); + expect(revisions[1].stix.x_mitre_domains).toBeUndefined(); + } + expect(await migration._private.countStaleDomainBypasses(migrationDb)).toBe(0); + + const completedRun = await mongoose.connection.db + .collection('automationRuns') + .findOne( + { name: '20260730230000-backfill-canonical-x-mitre-domains' }, + { sort: { started_at: -1 } }, + ); + expect(completedRun.status).toBe('completed'); + expect(completedRun.counts.enterprise_defaults).toBe(2); + + const fallbackItems = await mongoose.connection.db + .collection('automationRunItems') + .find({ run_id: completedRun.run_id }) + .toArray(); + expect(fallbackItems).toHaveLength(2); + expect(fallbackItems.map((item) => item.details.domain_source)).toEqual([ + 'enterprise-default', + 'enterprise-default', + ]); + }); + + after(async function () { + await migrationClient?.close(); + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/release-tracks/ephemeral-bundle.spec.js b/app/tests/api/release-tracks/ephemeral-bundle.spec.js index 3a930581..8c5cb002 100644 --- a/app/tests/api/release-tracks/ephemeral-bundle.spec.js +++ b/app/tests/api/release-tracks/ephemeral-bundle.spec.js @@ -49,6 +49,7 @@ describe('Ephemeral Bundle API', function () { let icsTechnique; let group; let relationship; + let sharedIcsRelationship; let icsGroup; let icsRelationship; @@ -169,6 +170,7 @@ describe('Ephemeral Bundle API', function () { type: 'intrusion-set', description: 'Group used to verify secondary-object inclusion.', object_marking_refs: [staticMarkingDefinitionId], + x_mitre_domains: [enterpriseDomain, icsDomain], }, }); @@ -186,6 +188,20 @@ describe('Ephemeral Bundle API', function () { }, }); + sharedIcsRelationship = await postObject('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: icsTechnique.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + icsGroup = await postObject('/api/groups', { workspace: { workflow: { state: 'work-in-progress' } }, stix: { @@ -231,9 +247,10 @@ describe('Ephemeral Bundle API', function () { expect(ids).toContain(group.stix.id); expect(ids).toContain(relationship.stix.id); - // The group's domains are inferred from the technique it uses + // Canonical multi-domain membership is preserved instead of being + // narrowed to the requested export domain. const bundleGroup = bundle.objects.find((o) => o.id === group.stix.id); - expect(bundleGroup.x_mitre_domains).toEqual([enterpriseDomain]); + expect(bundleGroup.x_mitre_domains).toEqual([enterpriseDomain, icsDomain]); // Referenced supporting objects expect(ids).toContain(enterpriseTechnique.stix.created_by_ref); @@ -278,17 +295,22 @@ describe('Ephemeral Bundle API', function () { expect(enterpriseIds).toContain(group.stix.id); expect(enterpriseIds).toContain(relationship.stix.id); + expect(enterpriseIds).not.toContain(sharedIcsRelationship.stix.id); expect(enterpriseIds).not.toContain(icsGroup.stix.id); expect(enterpriseIds).not.toContain(icsRelationship.stix.id); + expect(icsIds).toContain(group.stix.id); + expect(icsIds).toContain(sharedIcsRelationship.stix.id); expect(icsIds).toContain(icsGroup.stix.id); expect(icsIds).toContain(icsRelationship.stix.id); - expect(icsIds).not.toContain(group.stix.id); expect(icsIds).not.toContain(relationship.stix.id); expect( enterpriseBundle.objects.find((object) => object.id === group.stix.id).x_mitre_domains, - ).toEqual([enterpriseDomain]); + ).toEqual([enterpriseDomain, icsDomain]); + expect( + icsBundle.objects.find((object) => object.id === group.stix.id).x_mitre_domains, + ).toEqual([enterpriseDomain, icsDomain]); expect( icsBundle.objects.find((object) => object.id === icsGroup.stix.id).x_mitre_domains, ).toEqual([icsDomain]); diff --git a/app/tests/api/release-tracks/virtual-domain-filters.spec.js b/app/tests/api/release-tracks/virtual-domain-filters.spec.js index 98e3c984..71c798e3 100644 --- a/app/tests/api/release-tracks/virtual-domain-filters.spec.js +++ b/app/tests/api/release-tracks/virtual-domain-filters.spec.js @@ -91,7 +91,7 @@ describe('Virtual Release Track Domain Filters API', function () { return post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}); } - it('filters exact pinned revisions by normalized ATT&CK domains', async function () { + it('includes exact pinned revisions when any canonical ATT&CK domain matches', async function () { const enterprise = await post( '/api/mitigations', buildMitigation('Enterprise Domain Member', ['enterprise-attack']), @@ -102,7 +102,11 @@ describe('Virtual Release Track Domain Filters API', function () { ); const shared = await post( '/api/mitigations', - buildMitigation('Shared Domain Member', ['enterprise-attack', 'ics-attack']), + buildMitigation('Shared Domain Member', ['enterprise-attack', 'mobile-attack']), + ); + const mobile = await post( + '/api/mitigations', + buildMitigation('Mobile Domain Member', ['mobile-attack']), ); const noDomain = await post('/api/mitigations', buildMitigation('No Domain Member', undefined)); const enterpriseMatrix = await post( @@ -118,6 +122,7 @@ describe('Virtual Release Track Domain Filters API', function () { enterprise, ics, shared, + mobile, noDomain, enterpriseMatrix, ]); @@ -139,16 +144,26 @@ describe('Virtual Release Track Domain Filters API', function () { expect.arrayContaining([enterprise.stix.id, shared.stix.id, enterpriseMatrix.stix.id]), ); expect(enterpriseIds).not.toContain(ics.stix.id); + expect(enterpriseIds).not.toContain(mobile.stix.id); expect(enterpriseIds).not.toContain(noDomain.stix.id); const icsSnapshot = await createVirtualSnapshot('ICS Domain Virtual', component.id, [ 'ics-attack', ]); const icsIds = icsSnapshot.members.map((member) => member.object_ref); - expect(icsIds).toEqual(expect.arrayContaining([ics.stix.id, shared.stix.id])); + expect(icsIds).toEqual(expect.arrayContaining([ics.stix.id])); + expect(icsIds).not.toContain(shared.stix.id); expect(icsIds).not.toContain(enterprise.stix.id); expect(icsIds).not.toContain(enterpriseMatrix.stix.id); expect(icsIds).not.toContain(noDomain.stix.id); + + const mobileSnapshot = await createVirtualSnapshot('Mobile Domain Virtual', component.id, [ + 'mobile', + ]); + const mobileIds = mobileSnapshot.members.map((member) => member.object_ref); + expect(mobileIds).toEqual(expect.arrayContaining([mobile.stix.id, shared.stix.id])); + expect(mobileIds).not.toContain(enterprise.stix.id); + expect(mobileIds).not.toContain(ics.stix.id); }); after(async function () { diff --git a/app/tests/api/stix-bundles/stix-bundles.spec.js b/app/tests/api/stix-bundles/stix-bundles.spec.js index 98b5c656..c415c229 100644 --- a/app/tests/api/stix-bundles/stix-bundles.spec.js +++ b/app/tests/api/stix-bundles/stix-bundles.spec.js @@ -16,11 +16,11 @@ * - Require ATT&CK IDs * * 2. DETECTION STRATEGIES (x-mitre-detection-strategy) - Secondary Objects - * - NOT explicitly assigned to domains (domain is inferred) + * - Canonical domains are preserved when explicitly assigned + * - Legacy domainless objects receive an export-time fallback * - Included in bundle under TWO conditions: * a) They detect a technique in the bundle (via 'detects' relationship) * b) They reference an analytic in the bundle (via x_mitre_analytic_refs) - * - Their x_mitre_domains is set to the domain being exported * * 3. DATA COMPONENTS & DATA SOURCES - Now Primary Objects * - Both are now PRIMARY objects with explicit domain assignment @@ -37,7 +37,8 @@ * ✓ Analytics are retrieved as primary objects by domain * ✓ Detection strategies are included when they detect techniques in bundle * ✓ Detection strategies are included when they reference analytics in bundle - * ✓ Detection strategies get their x_mitre_domains set to the export domain + * ✓ Canonical multi-domain detection strategies retain every assigned domain + * ✓ Legacy domainless detection strategies get an export-domain fallback * ✓ Data components are retrieved as primary objects (not via detects relationships) * ✓ Data sources are optionally included via includeDataSources parameter * ✓ Deprecated detects relationships from data components are ignored @@ -46,7 +47,7 @@ * TEST DATA STRUCTURE: * - 3 Techniques (attack-patterns) across enterprise and ICS domains * - 2 Analytics in enterprise domain - * - 3 Detection Strategies (secondary objects with no domain assignment) + * - 3 Detection Strategies (secondary objects with canonical or legacy domain assignment) * - 2 Data Components with explicit domain assignments * - 2 Data Sources with explicit domain assignments * - Deprecated detects relationships from data components (to prove they're ignored) @@ -81,7 +82,7 @@ const mitreIdentityId = 'identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5'; * This bundle includes: * - 3 techniques (2 enterprise, 1 ICS, with 1 shared) * - 2 analytics (both enterprise) - * - 3 detection strategies (no domain - inferred) + * - 3 detection strategies (canonical domains or legacy inference) * - 2 data components (1 enterprise, 1 ICS) * - 2 data sources (1 enterprise, 1 ICS) * - Valid detects relationships: detection-strategy → technique @@ -294,7 +295,7 @@ const newSpecBundleData = { external_references: [{ source_name: 'mitre-attack', external_id: 'DET0001' }], x_mitre_analytic_refs: ['x-mitre-analytic--44444444-4444-4444-8444-444444444444'], x_mitre_attack_spec_version: config.app.attackSpecVersion, - x_mitre_domains: [enterpriseDomain], + x_mitre_domains: [enterpriseDomain, icsDomain], x_mitre_modified_by_ref: mitreIdentityId, x_mitre_version: '1.0', }, @@ -641,7 +642,7 @@ describe('STIX Bundles New Specification API', function () { ); expect(ds001).toBeDefined(); expect(ds001.name).toBe('Detection Strategy 1 - Detects Technique via Relationship'); - expect(ds001.x_mitre_domains).toEqual([enterpriseDomain]); + expect(ds001.x_mitre_domains).toEqual([enterpriseDomain, icsDomain]); // Verify the 'detects' relationships are included const ds001DetectsRels = stixBundle.objects.filter( diff --git a/docs/README.md b/docs/README.md index b4e64a7e..921c41fa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -62,6 +62,7 @@ Configuration, deployment, and identity provider setup. - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures - [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator track-deletion attempts - [Release-Track Deterministic Graph Migration](admin/release-track-graph-migration.md): Preview and operate the relationship-pin and snapshot-manifest backfill +- [ATT&CK Canonical-Domain Migration](admin/canonical-domain-migration.md): Understand the release-agnostic startup repair, inactive-revision handling, strict validation, and verification procedure ### Authentication diff --git a/docs/admin/canonical-domain-migration.md b/docs/admin/canonical-domain-migration.md new file mode 100644 index 00000000..f3890316 --- /dev/null +++ b/docs/admin/canonical-domain-migration.md @@ -0,0 +1,143 @@ +# ATT&CK Canonical-Domain Migration + +## Purpose + +Workbench formerly allowed some domain-bearing ATT&CK objects to omit +`x_mitre_domains`. Bundle export inferred or projected a domain later. That +produced separate domain-narrowed representations of a single object and made +virtual-track domain filters dependent on export-time behavior. + +Migration +`20260730230000-backfill-canonical-x-mitre-domains.js` replaces that model with +canonical object data: + +- one object revision carries its complete domain union; +- a cross-domain revision can appear unchanged in multiple domain bundles; +- virtual `filters.domains` uses inclusive set intersection; +- new reviewed content cannot rely on a missing-domain validation bypass. + +The migration runs automatically at server startup when database migrations +are enabled. + +## Domain source + +Startup does not access GitHub or another network service, and the migration +is not coupled to a particular ATT&CK release manifest. Workbench records the +canonical ATT&CK collections containing each exact object revision in +`workspace.collections`. The migration maps those persisted Enterprise, ICS, +and Mobile collection references back to their domains. + +Domain membership is inferred from an exact revision's collection provenance: + +- one canonical collection reference produces one domain; +- multiple canonical collection references produce the complete domain union; +- unrelated collection references are ignored. + +The migration examines the latest revision of every domain-bearing ATT&CK +lineage: techniques, campaigns, mitigations, groups, malware, tools, +analytics, assets, data components, data sources, detection strategies, +matrices, and tactics. This includes active, revoked, and deprecated content. +Identities, marking definitions, collections, and relationships are excluded +because their ADM schemas do not define `x_mitre_domains`. + +## Repair behavior + +Only the latest revision in each affected object lineage is repaired. +Historical revisions remain byte-for-byte historical and may still be +domainless. + +Active latest revisions are reposted through the ordinary service `create` +workflow. This creates a new revision, runs ADM validation and lifecycle hooks, +and triggers the same relationship and standard-track member-sync behavior as +an API POST. Reposts are processed in batches of 50 with at most four service +creates in flight. Analytics, data components, and detection strategies are +serialized because their backref hooks perform read-modify-write updates. +Release-track member sync is serialized per track, so concurrent group, +campaign, or matrix reposts cannot overwrite one another's candidate changes. + +Revoked and deprecated latest revisions use a narrow exception. The migration +duplicates the stored entity directly, preserves `revoked` and +`x_mitre_deprecated`, assigns canonical domains, and advances +`stix.modified`. It removes copied `workspace.release_tracks` pointers because +those backrefs belong to an exact old revision, removes the resolved +`x_mitre_domains` validation issue, and invokes release-track member sync +directly. It does not emit a generic created event or claim that ordinary +inactive-content hooks ran. + +Inactive replacements use bounded concurrent native-driver inserts. +The migration deliberately omits `_id` and lets the native MongoDB driver +performing the insert generate it. This avoids passing a Mongoose BSON value +to migrate-mongo when those dependencies use different BSON major versions. +Replacement/original verification is performed once per batch, and per-object +automation audit records are inserted together with stable sequence numbers. + +The old revision is never updated or deleted in either path. + +## Unmapped-object fallback + +Before creating any object revision, the migration resolves the complete +latest domainless candidate set from persisted collection provenance. If an +object has no recognized canonical collection reference, the migration assigns +`["enterprise-attack"]`. This permits legacy custom content to satisfy the +stricter contract without blocking startup. The fallback is explicit in the +per-object audit record as `domain_source: "enterprise-default"` and increments +the run's `enterprise_defaults` counter. + +Persisted missing-domain validation bypasses are deleted only after all object +repairs succeed and verification finds no remaining latest domainless target. +A partial repair therefore leaves enforcement unchanged and fails startup. On +restart, already repaired lineages are skipped and only the remaining work is +retried. + +## Verification and audit + +Inspect the latest run: + +```javascript +db.automationRuns.findOne( + { name: '20260730230000-backfill-canonical-x-mitre-domains' }, + { sort: { started_at: -1 } }, +); +``` + +Important counters are: + +- `active_reposts` +- `inactive_clones` +- `active_batches` +- `inactive_batches` +- `enterprise_defaults` +- `revoked` +- `deprecated` +- `bypasses_removed` +- `failed` + +A completed run reports both verification values as zero: + +```javascript +{ + remaining_latest_domainless_target_objects: 0, + remaining_domain_validation_bypasses: 0 +} +``` + +Inspect per-object actions by using the run's `run_id`: + +```javascript +db.automationRunItems.find({ run_id: '' }).sort({ sequence: 1 }); +``` + +## After migration + +Active reposts and inactive clones may become candidates on standard tracks +that already contain those object lineages. Review and release those +candidates through the normal standard-track workflow before materializing +the next virtual baseline. + +Exact historical domainless revisions remain retrievable. Legacy graph +rendering retains a compatibility fallback for those pins, but current +canonical revisions and new releases do not depend on that fallback. + +The down migration is intentionally a no-op. Removing the replacement +revisions or restoring permission to create invalid reviewed content would +discard history and weaken the new contract. diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 53920a37..d04bf799 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -25,6 +25,46 @@ Keep these rules in mind while updating the connector: convention and do not currently include `/standard/`. - A release preview is a read-only `GET`. A release commit is a `POST`. +## P0 — Send canonical domains for domain-bearing content + +### [ ] Require `x_mitre_domains` in affected reviewed-object forms + +The backend no longer suppresses the ATT&CK Data Model error for a missing +`x_mitre_domains` property on campaigns, intrusion sets, detection strategies, +or matrices. Existing latest v19.1 content is repaired automatically at server +startup, including revoked and deprecated lineages, but new reviewed revisions +must carry their own canonical domain membership. + +Update the affected Angular create/edit payloads so the field contains the +object's complete domain union: + +```ts +x_mitre_domains: Array<'enterprise-attack' | 'ics-attack' | 'mobile-attack'>; +``` + +Do not reduce a cross-domain object to the currently selected screen or bundle +domain. For example, one object used by Enterprise and Mobile should persist +`['enterprise-attack', 'mobile-attack']`; both virtual domain filters will +include that same exact revision by set intersection. + +Workbench still permits incomplete `work-in-progress` objects under the +existing partial-ADM workflow contract. Before a form advances an affected +object to `awaiting-review` or `reviewed`, require at least one domain and +surface the backend's `x_mitre_domains` validation detail if it is missing. + +Done when: + +- Campaign, group, detection-strategy, and matrix form models expose canonical + domain selection. +- Reviewed create and new-revision payloads always include a nonempty domain + array. +- Multi-select state preserves every selected domain instead of choosing one + based on route context. +- Validation errors for `x_mitre_domains` are displayed next to the domain + control. +- Tests cover a cross-domain payload and rejection of a reviewed domainless + payload. + ## P0 — Model draft revision selectors separately from released member pins ### [ ] Preserve `"latest"` in candidate and staged frontend state diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 84a743de..66a27c32 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -20,6 +20,83 @@ Verification (2026-07-30): canonical-domain migration spec still has three isolated failures, so a clean aggregate run remains outstanding. +## Embedded canonical-domain migration and enforcement + +- [x] Batch the canonical-domain migration so active revisions use bounded + service-layer concurrency and verification/audit records avoid + unnecessary per-object database round trips. +- [x] Replace the v19.1 object manifest with persisted canonical collection + provenance and scan the latest revision of all 13 domain-bearing ATT&CK + types, irrespective of active, deprecated, or revoked state. +- [x] Let the native migration driver generate inactive-clone `_id` values so + Mongoose BSON 6 values are never passed to MongoDB driver/BSON 7 writes. +- [x] Serialize concurrent release-track member-sync mutations per track so + batched reposts cannot overwrite candidates created by sibling workers. +- [x] Add batch-size, audit-sequence, shared-track concurrency, and + idempotency regressions; update operator documentation and rerun the + focused and complete test suites. +- [x] Add an idempotent startup migration that reposts every active latest + domainless object through its normal service create lifecycle. +- [x] Include deprecated and revoked latest revisions as immutable direct + clones, preserving lifecycle state and creating a new `modified` + revision without relying on inactive-content POST guardrails. +- [x] Initialize release-track member synchronization during the migration so + newly created revisions follow ordinary track-driven candidacy behavior. +- [x] Remove static `x_mitre_domains` validation bypasses and delete their + already-persisted database copies during migration. +- [x] Default a latest domainless object that cannot be mapped to canonical + collection provenance to `["enterprise-attack"]`, and audit the fallback. +- [x] Add migration, idempotency, inactive-state, member-sync, and ADM + enforcement regressions. +- [x] Update migration and domain-contract documentation, then run focused + tests, lint, and the complete `npm test` suite. + +Verification (2026-07-30): + +- Release-agnostic canonical-domain migration regression: 8 passing, covering + all 13 domain-bearing types and the MongoDB 7/Mongoose MongoDB 6 driver + boundary. +- Focused virtual-domain and bundle regressions: 23 passing. +- Backend and migration lint plus diff checks pass. +- Complete suite passes: OpenAPI 2, config 21, API 982, middleware 29, and + scheduler 10. +- Existing release-track change-capture regression: 13 passing. +- Batch-related lint and formatting checks pass. +- Two complete-suite runs reached 977 and 975 API passes respectively. The + remaining failures were the documented roaming HTTP/Mongo test-harness + failures in unrelated specs; every affected spec, including virtual + determinism, passes in isolation. +- Focused canonical-domain migration, virtual-filter, and bundle regressions: + 110 passing. +- Application and migration lint plus diff checks pass. +- The pre-batching clean full-suite baseline was OpenAPI 2, config 21, API 978, + middleware 29, and scheduler 10. + +## ATT&CK v19.1 canonical domain repair + +- [x] Add a dry-run/apply operational migration that derives canonical + `x_mitre_domains` values from object presence across the Enterprise, ICS, + and Mobile v19.1 collection TOCs. +- [x] Repost each affected latest active object through its normal create + endpoint so the repair creates a new revision and triggers ordinary + release-track member synchronization. +- [x] Preserve canonical multi-domain arrays during legacy and ephemeral bundle + export instead of narrowing them to the requested bundle domain. +- [x] Keep virtual `filters.domains` matching inclusive: any matching canonical + domain includes an object, while no matching domain excludes it. +- [x] Document the canonical-domain contract and the required follow-up + standard-track release after the repair creates new candidate revisions. +- [x] Run focused migration and API regressions, then lint, OpenAPI validation, + and the complete `npm test` suite. + +Verification (2026-07-30): + +- Migration/bootstrap Python regressions: 20 passing. +- Focused bundle and virtual-domain API regressions: 23 passing. +- Lint and diff checks pass. +- Complete server suite passes: OpenAPI 2, config 21, API 973, middleware 29, + and scheduler 10. + ## Bootstrap hotfix — ATT&CK-branded track names - [x] Permit ampersands in release-track names at the request and persistence diff --git a/docs/developer/data-model.md b/docs/developer/data-model.md index 9f787445..29768fef 100644 --- a/docs/developer/data-model.md +++ b/docs/developer/data-model.md @@ -24,6 +24,23 @@ The ATT&CK Workbench database supports the following ATT&CK object types (with t Most ATT&CK object types should be updated by creating a new object with a new `modified` timestamp (POST request). The Collection Index is different and should be updated by modifying (overwriting) the current object (PUT request). +## Canonical Domain Membership + +`stix.x_mitre_domains` is authoritative object data for domain-bearing ATT&CK +objects. Cross-domain content has one revision containing the complete domain +union, such as `["enterprise-attack", "mobile-attack"]`; Workbench does not +store separate domain-narrowed copies of that revision. + +ADM validation requires the property before a domain-bearing object leaves the +partial `work-in-progress` workflow. Workbench does not suppress the +missing-domain error for any domain-bearing ATT&CK type. +Migration +`20260730230000-backfill-canonical-x-mitre-domains.js` creates replacement +latest revisions for all domainless lineages without rewriting historical +revisions. Domain unions come from persisted canonical collection provenance; +unmappable content defaults to Enterprise. See the +[operator guide](../admin/canonical-domain-migration.md). + ## Database Structure ### attackObjects Collection @@ -133,6 +150,7 @@ The REST API supports linking between objects using a reference mechanism called When one object references another, it uses the format `(LinkById: ref)` where `ref` is the external ID of the referenced object. This is stored in the database as part of the object's text properties (typically the description). Additionally, an external reference is added to the object with: + - `source_name`: the external ID of the referenced object - `url`: the URL of the referenced object - `description`: the name of the referenced object @@ -164,4 +182,4 @@ Object containing the LinkById to another object: ### Export Behavior -When exporting objects, LinkById references are converted to Markdown links in the format `[description](url)`, making them human-readable in exported content. \ No newline at end of file +When exporting objects, LinkById references are converted to Markdown links in the format `[description](url)`, making them human-readable in exported content. diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 81e984c7..509e82be 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -15,9 +15,9 @@ implements the ATT&CK bundle-composition rules: 1. **Primary objects** are retrieved by domain (`x_mitre_domains`): techniques, tactics, mitigations, software, matrices, analytics, data components, data sources. -2. **Secondary objects** (groups, campaigns, detection strategies) cannot be - assigned domains by users; they are discovered through relationships to - primary objects and their `x_mitre_domains` is inferred at export time. +2. **Secondary objects** (groups, campaigns, detection strategies) were + historically discovered through relationships to primary objects and their + `x_mitre_domains` was projected at export time. 3. **Relationship referential integrity**: a relationship is only emitted if both its `source_ref` and `target_ref` are present in the bundle. 4. **Supporting objects**: identities (`created_by_ref`) and marking @@ -131,6 +131,23 @@ Implemented in - `x_mitre_contents`: every bundle object except marking definitions (which are recorded in `object_marking_refs`), sorted by `object_ref` +### Canonical domains and the legacy graph renderer + +Domain membership is object data, not an export projection. A cross-domain +object has one revision whose `x_mitre_domains` contains the complete domain +union. That same revision may appear in multiple domain bundles; its array is +not narrowed to the domain requested by a particular export. + +The legacy and ephemeral graph renderer now preserves every nonempty +`x_mitre_domains` array it hydrates. Export-time inference remains only as a +compatibility fallback for exact historical domainless revisions pinned +before canonical-domain enforcement, including historical matrix revisions. +The fallback affects the rendered copy and does not update the stored +revision. The release-agnostic startup migration creates canonical replacement +revisions for the latest domainless object in every domain-bearing chain; all +subsequent content must persist canonical domains so virtual composition, +snapshot export, and ephemeral export observe the same membership. + Because snapshot contents are explicitly curated, primary entries do **not** receive the legacy attack-id / deprecated / revoked filters. Secondary graph capture retains the established bounded ATT&CK expansion rules and freezes diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 791dd6af..74a3c130 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -33,6 +33,11 @@ supported deployments. directly into the initial snapshot so Mongoose applies defaults only to omitted options instead of replacing caller-supplied values with an empty config. +- `x_mitre_domains` is required by ADM for domain-bearing ATT&CK objects. + Workbench no longer suppresses the missing-field validation error for + campaigns, intrusion sets, detection strategies, or matrices. The established + partial-ADM contract still permits an incomplete `work-in-progress` draft, + but it cannot advance as valid reviewed content without domains. - **Same revision selector** can only be in one tier per release-track snapshot (`members`, `staged`, `candidates`, or `quarantine`) - **Different selectors** for the same object CAN exist in multiple tiers simultaneously @@ -46,6 +51,63 @@ supported deployments. and `version-utils.calculateNextVersion` repeats the invariant so internal release-planning callers cannot silently choose one selector. +### ATT&CK canonical-domain migration + +Migration +`20260730230000-backfill-canonical-x-mitre-domains.js` establishes the stricter +domain contract for data created under the former validation bypasses. It +derives domain unions from the canonical Enterprise, ICS, and Mobile +collection references already persisted on exact object revisions; startup +never downloads release data and is not tied to a release-specific manifest. + +The migration examines the latest revision of every ADM domain-bearing ATT&CK +type, including active, revoked, and deprecated content: + +- Active domainless revisions are reposted through their normal service + `create` workflow. This performs ADM validation and emits the ordinary + created event, so standard release tracks enroll the new revision according + to their member-sync configuration. +- Revoked or deprecated domainless revisions are copied directly into + `attackObjects` as a new immutable revision. This intentionally avoids + lifecycle guardrails that can reject inactive content. The copy preserves + both lifecycle flags, advances `stix.modified`, removes revision-specific + release-track backrefs and the now-resolved validation error, and invokes + release-track member sync directly. It does not emit a generic created event, + because that would falsely imply that every ordinary lifecycle hook ran and + could trigger unrelated active-content side effects. The original revision + remains unchanged. +- Already canonical latest revisions are skipped. A rerun after partial + completion therefore processes only the remaining domainless chains. + +The repair is batch-oriented to keep startup bounded on production-sized +datasets. It processes 50 candidates at a time, permits four concurrent active +service reposts, inserts inactive clones with bounded native-driver +concurrency, verifies replacement/original pairs with one read per batch, and +bulk-inserts the corresponding automation audit items. Inactive clone `_id` +values are generated by the same native driver performing the insert; this +prevents BSON-major incompatibilities between Mongoose and migrate-mongo. +Analytics, data-component, and detection-strategy reposts remain serial +because their backref hooks are read-modify-write operations. +Member-sync mutations use a per-track lock and refresh the latest snapshot +inside that lock; otherwise two concurrent reposts affecting the same track +could each clone a stale snapshot and lose one candidate update. + +Before changing data, the migration resolves the complete candidate set from +persisted canonical collection provenance. A latest domainless target object +without recognized provenance defaults to `["enterprise-attack"]` so legacy +or custom content does not block startup. Each fallback is identified as +`domain_source: "enterprise-default"` in its automation item and counted by +the run's `enterprise_defaults` counter. + +The migration deletes database copies of retired `x_mitre_domains` bypass +rules only after every target object has been repaired. Removing the rules +only from `default-bypass-rules.json` would be insufficient because static +rules are seeded additively. If object repair is partial, the persisted +bypasses remain and startup fails; the next boot safely retries the remaining +chains. Completion is recorded in `automationRuns` and +`automationRunItems`, with active reposts and inactive clones reported +separately. + ### Primary revision integrity boundary `app/services/release-tracks/primary-revision-service.js` is the shared @@ -197,6 +259,14 @@ means no type filter. Materialization compares each value to the type prefix already encoded in the resolved snapshot member's `object_ref`; it never re-resolves that member to the latest database revision. +Component `filters.domains` is also evaluated against the exact pinned +revision. It normalizes public and STIX domain names, then uses set +intersection (any-match) semantics. A canonical multi-domain revision is +therefore eligible for every matching domain composition without being cloned +or narrowed. Domainless revisions fail a configured domain filter, except for +the established matrix fallback through +`external_references[].external_id`. + Virtual deduplication distinguishes duplicate contributions from revision conflicts. Entries are grouped first by `object_ref`, then by the exact `object_modified` timestamp. Multiple components contributing the same exact @@ -292,29 +362,29 @@ versions. eventBus.emit('release-track:status-changed', { collectionId: 'x-mitre-collection--123', objectId: 'attack-pattern--eee', - objectModified: '2024-01-12T09:00:00Z', // Version pin + objectModified: '2024-01-12T09:00:00Z', // Version pin oldStatus: 'work-in-progress', newStatus: 'awaiting-review', changedBy: 'user@example.com', - changedAt: '2024-01-15T10:00:00Z' + changedAt: '2024-01-15T10:00:00Z', }); // When object version is added to collection candidates eventBus.emit('release-track:candidate-added', { collectionId: 'x-mitre-collection--123', objectId: 'attack-pattern--eee', - objectModified: '2024-01-12T09:00:00Z', // Version pin + objectModified: '2024-01-12T09:00:00Z', // Version pin status: 'work-in-progress', - addedBy: 'user@example.com' + addedBy: 'user@example.com', }); // When object is promoted to staged eventBus.emit('release-track:object-staged', { collectionId: 'x-mitre-collection--123', objectId: 'attack-pattern--ddd', - objectModified: '2024-01-14T10:00:00Z', // Version pin + objectModified: '2024-01-14T10:00:00Z', // Version pin status: 'reviewed', - promotedBy: 'auto' // or user email + promotedBy: 'auto', // or user email }); // When collection is released @@ -325,10 +395,10 @@ eventBus.emit('release-track:released', { promotedObjects: [ { objectId: 'attack-pattern--ddd', - objectModified: '2024-01-14T10:00:00Z' // Version included in release - } + objectModified: '2024-01-14T10:00:00Z', // Version included in release + }, ], - releasedBy: 'admin@example.com' + releasedBy: 'admin@example.com', }); ``` @@ -345,7 +415,7 @@ eventBus.on('release-track:status-changed', async (event) => { await promoteToStaged( collection, event.objectId, - event.objectModified // Preserve version pin + event.objectModified, // Preserve version pin ); } } @@ -359,17 +429,17 @@ eventBus.on('release-track:object-staged', async (event) => { { $set: { 'workspace.referenced_by.$[elem].tier': 'staged', - 'workspace.referenced_by.$[elem].status': event.status - } + 'workspace.referenced_by.$[elem].status': event.status, + }, }, { arrayFilters: [ { 'elem.collection_id': event.collectionId, - 'elem.tier': 'candidates' - } - ] - } + 'elem.tier': 'candidates', + }, + ], + }, ); }); ``` diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 12e06363..749befba 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1162,11 +1162,21 @@ POST /api/release-tracks/new } ``` -`filters.domains` matches the exact pinned revision's `x_mitre_domains`. -Short names (`enterprise`, `ics`, `mobile`) and STIX names ending in -`-attack` are equivalent. Objects without a matching domain are excluded. -For primary matrices, which omit `x_mitre_domains` in published ATT&CK data, -the domain is read from `external_references[].external_id`. +`filters.domains` matches the exact pinned revision's canonical +`x_mitre_domains`. Short names (`enterprise`, `ics`, `mobile`) and STIX names +ending in `-attack` are equivalent. The comparison is inclusive: any +intersection between the object's domains and the configured domains includes +the object. Thus, `["enterprise-attack", "mobile-attack"]` matches either an +Enterprise or Mobile component filter; `["mobile-attack"]` does not match an +Enterprise filter. Objects without a matching domain are excluded. + +Cross-domain objects retain the complete domain array in every representation. +The filter selects an exact revision; it does not narrow or rewrite that +revision for the requested virtual track. +Current matrix revisions must also persist `x_mitre_domains`. For an exact +historical matrix revision created before that requirement, the domain can +still be read from `external_references[].external_id` as a compatibility +fallback. `filters.object_types` accepts canonical Workbench STIX type names: `attack-pattern`, `campaign`, `course-of-action`, `identity`, `intrusion-set`, diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index 336e067c..e887a010 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -141,6 +141,14 @@ from tagged standard component snapshots. They never inherit `track_latest`, and retrieving a persisted virtual snapshot does not re-resolve its component tracks. +Domain membership is likewise pinned object data. A cross-domain object has +one revision whose `x_mitre_domains` contains the complete domain union; the +same exact revision can therefore be selected by multiple virtual domain +filters. Workbench does not create separate domain-narrowed copies during +bundle export. Campaigns, intrusion sets, detection strategies, and matrices +can no longer rely on the former missing-domain validation bypass once they +leave the partial `work-in-progress` state. + ## Key Features ### Automatic Promotion diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index fa06c205..00d1929e 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -219,13 +219,24 @@ filters: { ``` Domain filters hydrate the exact revisions pinned by the component's tagged -snapshot; they do not inspect the latest database revision. An object with -multiple matching domains is included in each corresponding virtual track. -Objects without `x_mitre_domains` are excluded when a domain filter is set. -The primary Enterprise, ICS, and Mobile matrices are the exception: published -ATT&CK data identifies their domain through -`external_references[].external_id`, so virtual filtering uses that established -matrix fallback. +snapshot; they do not inspect the latest database revision. Matching uses +inclusive **any-match** semantics, not exact-array equality: an object is +included when at least one value in its canonical `x_mitre_domains` array +matches at least one configured domain. For example, +`["enterprise-attack", "mobile-attack"]` is included by both an Enterprise +filter and a Mobile filter, while `["mobile-attack"]` is excluded by an +Enterprise filter. Objects without `x_mitre_domains` are excluded when a +domain filter is set. + +`x_mitre_domains` is canonical object data. A cross-domain object has one +revision containing the complete domain union; Workbench does not create or +emit separate domain-narrowed revisions of that object. Consequently, the +same exact `(object_ref, object_modified)` member can appear in multiple +domain-filtered virtual snapshots. +Current matrix revisions follow the same canonical-domain requirement. For +exact historical matrix revisions created before enforcement, virtual +filtering retains a compatibility fallback to the domain in +`external_references[].external_id`. `object_types` values are case-sensitive canonical Workbench STIX type names. When the property is present, it must contain at least one value and cannot diff --git a/migrations/20260730230000-backfill-canonical-x-mitre-domains.js b/migrations/20260730230000-backfill-canonical-x-mitre-domains.js new file mode 100644 index 00000000..4ccc69d0 --- /dev/null +++ b/migrations/20260730230000-backfill-canonical-x-mitre-domains.js @@ -0,0 +1,726 @@ +'use strict'; + +/** + * Backfill canonical x_mitre_domains values for every latest domain-bearing + * ATT&CK object, then retire the validation bypasses that historically + * allowed domainless objects. + * + * Domain membership is inferred from the canonical ATT&CK collection + * provenance already persisted on each object revision. This keeps the + * migration release-agnostic while preserving multi-domain unions. Objects + * without mappable provenance default to Enterprise. + * + * Active latest revisions use the ordinary POST/create service pipeline so + * validation, lifecycle hooks, events, release-track member sync, and audit + * behavior match an operator-created revision. + * + * Revoked or deprecated latest revisions cannot reliably traverse that + * workflow's lifecycle guardrails. They are copied directly as a new + * immutable revision with a bumped modified timestamp. Only release-track + * member sync is invoked for this exceptional path: emitting a generic + * created event would falsely imply that every normal lifecycle hook ran and + * could trigger unrelated active-content side effects. The prior revision is + * never mutated. + */ + +const mongoose = require('mongoose'); +const config = require('../app/config/config'); +const { + createAutomationRunRecorder, + serializeError, +} = require('../app/lib/automation-run-recorder'); +const logger = require('../app/lib/logger'); +const systemConfigurationRepository = require('../app/repository/system-configurations-repository'); +const validationBypassesService = require('../app/services/system/validation-bypasses-service'); + +const MIGRATION_NAME = '20260730230000-backfill-canonical-x-mitre-domains'; +const TARGET_TYPES = [ + 'attack-pattern', + 'campaign', + 'course-of-action', + 'intrusion-set', + 'malware', + 'tool', + 'x-mitre-analytic', + 'x-mitre-asset', + 'x-mitre-data-component', + 'x-mitre-data-source', + 'x-mitre-detection-strategy', + 'x-mitre-matrix', + 'x-mitre-tactic', +]; +const TARGET_TYPE_SET = new Set(TARGET_TYPES); +const DEFAULT_DOMAINS = ['enterprise-attack']; +const BATCH_SIZE = 50; +const ACTIVE_CONCURRENCY = 4; +const SERIAL_ACTIVE_TYPES = new Set([ + 'x-mitre-analytic', + 'x-mitre-data-component', + 'x-mitre-detection-strategy', +]); + +const CANONICAL_COLLECTION_DOMAINS = new Map([ + ['x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019', 'enterprise-attack'], + ['x-mitre-collection--90c00720-636b-4485-b342-8751d232bf09', 'ics-attack'], + ['x-mitre-collection--dac0d2d7-8653-445c-9bff-82f934c1e858', 'mobile-attack'], +]); + +const SERVICE_MODULE_BY_TYPE = { + 'attack-pattern': '../app/services/stix/techniques-service', + campaign: '../app/services/stix/campaigns-service', + 'course-of-action': '../app/services/stix/mitigations-service', + 'intrusion-set': '../app/services/stix/groups-service', + malware: '../app/services/stix/software-service', + tool: '../app/services/stix/software-service', + 'x-mitre-analytic': '../app/services/stix/analytics-service', + 'x-mitre-asset': '../app/services/stix/assets-service', + 'x-mitre-data-component': '../app/services/stix/data-components-service', + 'x-mitre-data-source': '../app/services/stix/data-sources-service', + 'x-mitre-detection-strategy': '../app/services/stix/detection-strategies-service', + 'x-mitre-matrix': '../app/services/stix/matrices-service', + 'x-mitre-tactic': '../app/services/stix/tactics-service', +}; + +let memberSyncService; + +function chunkItems(items, size = BATCH_SIZE) { + const chunks = []; + for (let index = 0; index < items.length; index += size) { + chunks.push(items.slice(index, index + size)); + } + return chunks; +} + +async function mapWithConcurrency(items, concurrency, mapper) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + results[index] = await mapper(items[index], index); + } + } + + const workerCount = Math.min(Math.max(concurrency, 1), items.length); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return results; +} + +function hasCanonicalDomains(document) { + return Array.isArray(document?.stix?.x_mitre_domains) && document.stix.x_mitre_domains.length > 0; +} + +function domainsFromCollectionProvenance(document) { + const collectionRefs = new Set( + (document?.workspace?.collections || []).map((collection) => collection?.collection_ref), + ); + return [...CANONICAL_COLLECTION_DOMAINS] + .filter(([collectionRef]) => collectionRefs.has(collectionRef)) + .map(([, domain]) => domain); +} + +function isInactive(document) { + return document?.stix?.revoked === true || document?.stix?.x_mitre_deprecated === true; +} + +function nextModifiedTimestamp(existingModified) { + const now = Date.now(); + const existing = new Date(existingModified).getTime(); + const next = Number.isFinite(existing) ? Math.max(now, existing + 1) : now; + return new Date(next); +} + +function latestTargetDocumentsPipeline() { + return [ + { $match: { 'stix.type': { $in: TARGET_TYPES } } }, + { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, + { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, + { $replaceRoot: { newRoot: '$document' } }, + ]; +} + +async function latestDomainlessTargetDocuments(db) { + const documents = await db + .collection('attackObjects') + .aggregate(latestTargetDocumentsPipeline()) + .toArray(); + return documents.filter((document) => !hasCanonicalDomains(document)); +} + +function resolveCandidates(documents) { + const candidates = []; + + for (const document of documents) { + const stixType = document?.stix?.type; + + if (!TARGET_TYPE_SET.has(stixType)) { + throw new Error(`Unsupported canonical-domain migration type: ${stixType}`); + } + + const provenanceDomains = domainsFromCollectionProvenance(document); + const hasProvenanceMapping = provenanceDomains.length > 0; + candidates.push({ + document, + domains: hasProvenanceMapping ? provenanceDomains : [...DEFAULT_DOMAINS], + domainSource: hasProvenanceMapping ? 'collection-provenance' : 'enterprise-default', + lifecycle: isInactive(document) ? 'inactive' : 'active', + }); + } + + return candidates; +} + +function ensureMongooseUsesClient(client) { + if (client && mongoose.connection.readyState === 0) { + mongoose.connection.setClient(client); + } +} + +async function prepareServiceLayer(client) { + ensureMongooseUsesClient(client); + + // These listeners are ordinarily registered while Express routes load, + // after migrate-mongo has finished. Load them explicitly so active reposts + // have the same relationship, analytic, and release-track side effects as + // a normal API create. + require('../app/services/stix/attack-objects-service'); + require('../app/services/stix/analytics-service'); + memberSyncService = require('../app/services/release-tracks/member-sync-service'); + + await validationBypassesService.loadStaticRules(config.configurationFiles.staticBypassRulesPath); +} + +async function assertOrganizationIdentityConfigured() { + const systemConfig = await systemConfigurationRepository.retrieveOne({ lean: true }); + if (!systemConfig?.organization_identity_ref) { + throw new Error( + 'System configuration is missing organization_identity_ref; cannot repost active ' + + 'domainless objects through the normal create workflow.', + ); + } +} + +function serviceFor(stixType) { + const modulePath = SERVICE_MODULE_BY_TYPE[stixType]; + if (!modulePath) { + throw new Error(`No canonical-domain migration service is configured for ${stixType}`); + } + return require(modulePath); +} + +function cloneForCreate(document, domains, modified) { + const repost = JSON.parse(JSON.stringify(document)); + delete repost._id; + delete repost.__v; + delete repost.__t; + repost.stix.modified = modified.toISOString(); + repost.stix.x_mitre_domains = domains; + return repost; +} + +function removeResolvedDomainValidation(workspace) { + const replacement = { ...(workspace || {}) }; + delete replacement.release_tracks; + + const validation = replacement.validation; + if (!validation || !Array.isArray(validation.errors)) { + return replacement; + } + + const remainingErrors = validation.errors.filter( + (error) => + !( + error?.code === 'invalid_type' && + Array.isArray(error.path) && + error.path.map(String).join('.') === 'x_mitre_domains' + ), + ); + + if (remainingErrors.length === 0) { + delete replacement.validation; + } else { + replacement.validation = { + ...validation, + errors: remainingErrors, + }; + } + + return replacement; +} + +async function repostActive(candidate, recorder) { + const { document, domains } = candidate; + const service = serviceFor(document.stix.type); + const modified = nextModifiedTimestamp(document.stix.modified); + const repost = cloneForCreate(document, domains, modified); + const created = await service.create(repost, { + import: false, + automationContext: { + automationName: MIGRATION_NAME, + runId: recorder.runId, + }, + }); + + return { + method: 'service-create', + modified: new Date(created.stix.modified), + document: created, + }; +} + +function prepareInactiveClone(candidate) { + const { document, domains } = candidate; + const modified = nextModifiedTimestamp(document.stix.modified); + const replacement = { + ...document, + workspace: removeResolvedDomainValidation(document.workspace), + stix: { + ...document.stix, + modified, + x_mitre_domains: domains, + }, + }; + delete replacement._id; + delete replacement.__v; + + return { + method: 'inactive-clone', + modified, + document: replacement, + }; +} + +async function syncInactiveClone(candidate, result, recorder) { + const { document } = candidate; + // The direct clone is intentionally not presented as a generic create. It + // still advances any standard track that references this object, matching + // the part of the ordinary revision workflow that release tracks own. + await memberSyncService.handleObjectModified({ + objectRef: document.stix.id, + newModified: result.modified, + modifiedBy: 'system', + trigger: document.stix.revoked === true ? 'revocation' : 'new-revision', + automationContext: { + automationName: MIGRATION_NAME, + runId: recorder.runId, + }, + }); +} + +async function processActiveBatch(candidates, recorder, concurrency) { + return mapWithConcurrency(candidates, concurrency, async (candidate) => { + try { + return { + candidate, + result: await repostActive(candidate, recorder), + }; + } catch (error) { + return { candidate, error }; + } + }); +} + +async function processInactiveBatch(db, candidates, recorder) { + return mapWithConcurrency(candidates, ACTIVE_CONCURRENCY, async (candidate) => { + try { + const result = prepareInactiveClone(candidate); + // Do not construct _id with Mongoose here. migrate-mongo uses the root + // MongoDB driver, which may carry a different BSON major version. Let + // the native driver performing the insert create its own ObjectId. + const insertResult = await db.collection('attackObjects').insertOne(result.document); + result.document._id = insertResult.insertedId; + await syncInactiveClone(candidate, result, recorder); + return { candidate, result }; + } catch (error) { + return { candidate, error }; + } + }); +} + +function revisionKey(stixId, modified) { + return `${stixId}\0${new Date(modified).toISOString()}`; +} + +function assertReplacement(candidate, result, originalsById, replacementsByRevision) { + const { document, domains } = candidate; + const replacement = replacementsByRevision.get(revisionKey(document.stix.id, result.modified)); + + if (!replacement) { + throw new Error(`Replacement revision was not persisted for ${document.stix.id}`); + } + if (new Date(replacement.stix.modified).getTime() <= new Date(document.stix.modified).getTime()) { + throw new Error(`Replacement revision did not advance modified for ${document.stix.id}`); + } + if (JSON.stringify(replacement.stix.x_mitre_domains) !== JSON.stringify(domains)) { + throw new Error(`Replacement revision has unexpected domains for ${document.stix.id}`); + } + if ((replacement.stix.revoked === true) !== (document.stix.revoked === true)) { + throw new Error(`Replacement revision changed revoked status for ${document.stix.id}`); + } + if ( + (replacement.stix.x_mitre_deprecated === true) !== + (document.stix.x_mitre_deprecated === true) + ) { + throw new Error(`Replacement revision changed deprecated status for ${document.stix.id}`); + } + + const original = originalsById.get(String(document._id)); + if (!original) { + throw new Error(`Original revision was not retained for ${document.stix.id}`); + } + + return replacement; +} + +async function verifyReplacementBatch(db, entries) { + if (entries.length === 0) return []; + + const originalIds = entries.map(({ candidate }) => candidate.document._id); + const replacementSelectors = entries.map(({ candidate, result }) => ({ + 'stix.id': candidate.document.stix.id, + 'stix.modified': result.modified, + })); + const persisted = await db + .collection('attackObjects') + .find({ + $or: [{ _id: { $in: originalIds } }, ...replacementSelectors], + }) + .toArray(); + const originalIdSet = new Set(originalIds.map(String)); + const originalsById = new Map( + persisted + .filter((document) => originalIdSet.has(String(document._id))) + .map((document) => [String(document._id), document]), + ); + const replacementsByRevision = new Map( + persisted.map((document) => [revisionKey(document.stix.id, document.stix.modified), document]), + ); + + return entries.map((entry) => { + try { + return { + ...entry, + replacement: assertReplacement( + entry.candidate, + entry.result, + originalsById, + replacementsByRevision, + ), + }; + } catch (error) { + return { candidate: entry.candidate, error }; + } + }); +} + +function actionFor(candidate) { + return candidate.lifecycle === 'active' + ? 'repost_with_canonical_domains' + : 'clone_inactive_with_domains'; +} + +function changedAuditItem(entry) { + const { candidate, result, replacement } = entry; + const { document, domains, domainSource, lifecycle } = candidate; + return { + status: 'changed', + action: actionFor(candidate), + target: { + kind: 'stix-object', + collection: 'attackObjects', + stix_id: document.stix.id, + stix_type: document.stix.type, + }, + details: { + lifecycle, + domain_source: domainSource, + persistence_method: result.method, + previous_modified: document.stix.modified, + new_modified: replacement.stix.modified, + revoked: document.stix.revoked === true, + deprecated: document.stix.x_mitre_deprecated === true, + changes: [ + { + field: 'stix.x_mitre_domains', + before: document.stix.x_mitre_domains, + after: domains, + }, + ], + }, + }; +} + +function failedAuditItem(candidate, error) { + const { document, domains, domainSource, lifecycle } = candidate; + return { + status: 'failed', + action: actionFor(candidate), + target: { + kind: 'stix-object', + collection: 'attackObjects', + stix_id: document.stix.id, + stix_type: document.stix.type, + }, + details: { + lifecycle, + domain_source: domainSource, + previous_modified: document.stix.modified, + attempted_domains: domains, + }, + error: serializeError(error), + }; +} + +async function finalizeBatch(db, processed, recorder, counts, failures) { + const processingFailures = processed.filter((entry) => entry.error); + const verified = await verifyReplacementBatch( + db, + processed.filter((entry) => !entry.error), + ); + const finalized = [...verified, ...processingFailures]; + const auditItems = []; + + for (const entry of finalized) { + const { candidate, error } = entry; + const { document, domainSource, lifecycle } = candidate; + if (error) { + counts.failed++; + failures.push({ stix_id: document.stix.id, error: error.message }); + auditItems.push(failedAuditItem(candidate, error)); + continue; + } + + counts.updated++; + if (lifecycle === 'active') counts.active_reposts++; + else counts.inactive_clones++; + if (domainSource === 'enterprise-default') counts.enterprise_defaults++; + if (document.stix.revoked === true) counts.revoked++; + if (document.stix.x_mitre_deprecated === true) counts.deprecated++; + auditItems.push(changedAuditItem(entry)); + } + + await recorder.recordItems(auditItems); +} + +async function countRemainingDomainlessTargets(db) { + return (await latestDomainlessTargetDocuments(db)).length; +} + +async function countStaleDomainBypasses(db) { + return db.collection('validationbypassrules').countDocuments({ + fieldPath: ['x_mitre_domains'], + errorCode: 'invalid_type', + stixType: { $in: TARGET_TYPES }, + }); +} + +async function removeStaleDomainBypasses(db) { + return db.collection('validationbypassrules').deleteMany({ + fieldPath: ['x_mitre_domains'], + errorCode: 'invalid_type', + stixType: { $in: TARGET_TYPES }, + }); +} + +async function run(db, client) { + const domainlessDocuments = await latestDomainlessTargetDocuments(db); + + const recorder = await createAutomationRunRecorder(db, { + automationType: 'migration', + name: MIGRATION_NAME, + trigger: { source: 'startup', runner: 'migrate-mongo' }, + scope: { + collections: ['attackObjects', 'validationbypassrules'], + object_kinds: ['stix-object', 'validation-bypass-rule'], + target_types: TARGET_TYPES, + }, + metadata: { + domain_source: 'persisted-canonical-collection-provenance', + canonical_collection_domains: Object.fromEntries(CANONICAL_COLLECTION_DOMAINS), + unmapped_default_domains: DEFAULT_DOMAINS, + active_method: 'service-create', + inactive_method: 'immutable-direct-clone', + batch_size: BATCH_SIZE, + active_concurrency: ACTIVE_CONCURRENCY, + serialized_active_types: [...SERIAL_ACTIVE_TYPES], + latest_domainless_objects_discovered: domainlessDocuments.length, + }, + }); + + const counts = { + scanned_candidates: domainlessDocuments.length, + active_reposts: 0, + inactive_clones: 0, + active_batches: 0, + inactive_batches: 0, + enterprise_defaults: 0, + revoked: 0, + deprecated: 0, + bypasses_removed: 0, + updated: 0, + failed: 0, + }; + const failures = []; + let verification = {}; + + try { + // Resolve the complete plan before deleting bypasses or creating object + // revisions. Persisted canonical collection provenance is authoritative + // when available; custom/unmapped content defaults to Enterprise. + const candidates = resolveCandidates(domainlessDocuments); + if (candidates.some((candidate) => candidate.lifecycle === 'active')) { + ensureMongooseUsesClient(client); + await assertOrganizationIdentityConfigured(); + } + await prepareServiceLayer(client); + + const activeCandidates = candidates.filter((candidate) => candidate.lifecycle === 'active'); + const parallelActiveCandidates = activeCandidates.filter( + (candidate) => !SERIAL_ACTIVE_TYPES.has(candidate.document.stix.type), + ); + const serialActiveCandidates = activeCandidates.filter((candidate) => + SERIAL_ACTIVE_TYPES.has(candidate.document.stix.type), + ); + const inactiveCandidates = candidates.filter((candidate) => candidate.lifecycle === 'inactive'); + + for (const batch of chunkItems(parallelActiveCandidates)) { + counts.active_batches++; + recorder.log('info', 'Processing active canonical-domain batch', { + batch: counts.active_batches, + size: batch.length, + concurrency: ACTIVE_CONCURRENCY, + }); + const processed = await processActiveBatch(batch, recorder, ACTIVE_CONCURRENCY); + await finalizeBatch(db, processed, recorder, counts, failures); + } + + // Analytics, data components, and detection strategies update referenced + // objects through read-modify-write hooks. Keep them serial while allowing + // independent active types to benefit from bounded concurrency. + for (const batch of chunkItems(serialActiveCandidates)) { + counts.active_batches++; + recorder.log('info', 'Processing serialized active canonical-domain batch', { + batch: counts.active_batches, + size: batch.length, + concurrency: 1, + stix_types: [...new Set(batch.map((candidate) => candidate.document.stix.type))], + }); + const processed = await processActiveBatch(batch, recorder, 1); + await finalizeBatch(db, processed, recorder, counts, failures); + } + + for (const batch of chunkItems(inactiveCandidates)) { + counts.inactive_batches++; + recorder.log('info', 'Processing inactive canonical-domain batch', { + batch: counts.inactive_batches, + size: batch.length, + concurrency: ACTIVE_CONCURRENCY, + }); + const processed = await processInactiveBatch(db, batch, recorder); + await finalizeBatch(db, processed, recorder, counts, failures); + } + + const remainingDomainless = await countRemainingDomainlessTargets(db); + if (failures.length > 0 || remainingDomainless > 0) { + const failureSample = failures + .slice(0, 5) + .map((failure) => `${failure.stix_id}: ${failure.error}`) + .join('; '); + throw new Error( + `Canonical-domain object repair is incomplete: ${failures.length} failed item(s), ` + + `${remainingDomainless} latest domainless target object(s). Validation bypasses ` + + `were retained.${failureSample ? ` Failures: ${failureSample}` : ''}`, + ); + } + + // Enforcement is the final step. Leaving persisted bypasses in place until + // every object is repaired prevents a partial run from activating a + // stricter contract against data the same migration has not yet fixed. + const bypassResult = await removeStaleDomainBypasses(db); + counts.bypasses_removed = bypassResult.deletedCount; + + verification = { + remaining_latest_domainless_target_objects: remainingDomainless, + remaining_domain_validation_bypasses: await countStaleDomainBypasses(db), + }; + + if (verification.remaining_domain_validation_bypasses > 0) { + throw new Error( + `Canonical-domain enforcement is incomplete: ` + + `${verification.remaining_domain_validation_bypasses} stale bypass(es).`, + ); + } + + await recorder.finish({ + status: 'completed', + counts, + warnings: {}, + verification, + summary: { + message: + `Assigned canonical ATT&CK domains to ${counts.updated} latest revision(s): ` + + `${counts.active_reposts} active repost(s) and ${counts.inactive_clones} inactive clone(s).`, + }, + errorSummary: null, + }); + + return { counts, verification }; + } catch (error) { + verification = { + ...verification, + remaining_latest_domainless_target_objects: + verification.remaining_latest_domainless_target_objects ?? + (await countRemainingDomainlessTargets(db).catch(() => null)), + remaining_domain_validation_bypasses: + verification.remaining_domain_validation_bypasses ?? + (await countStaleDomainBypasses(db).catch(() => null)), + }; + await recorder.finish({ + status: counts.updated > 0 ? 'partial' : 'failed', + counts, + warnings: {}, + verification, + summary: { message: 'Canonical-domain migration did not complete successfully.' }, + errorSummary: serializeError(error), + }); + throw error; + } +} + +module.exports = { + async up(db, client) { + const report = await run(db, client); + logger.info(`[${MIGRATION_NAME}] ${JSON.stringify(report)}`); + }, + + async down() { + logger.info( + `[${MIGRATION_NAME}] down migration is a no-op: replacement revisions and stricter ` + + `domain validation are retained`, + ); + }, + + _private: { + ACTIVE_CONCURRENCY, + BATCH_SIZE, + CANONICAL_COLLECTION_DOMAINS, + SERIAL_ACTIVE_TYPES, + TARGET_TYPES, + chunkItems, + countRemainingDomainlessTargets, + countStaleDomainBypasses, + domainsFromCollectionProvenance, + hasCanonicalDomains, + isInactive, + latestDomainlessTargetDocuments, + mapWithConcurrency, + nextModifiedTimestamp, + prepareInactiveClone, + processInactiveBatch, + removeResolvedDomainValidation, + removeStaleDomainBypasses, + resolveCandidates, + run, + }, +}; From 2d15fbfc7037ebee46904d539ccd09804cf224b7 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:18:08 -0400 Subject: [PATCH 42/55] docs(release-tracks): generalize canonical domain repair wording --- docs/developer/FRONTEND_TODO.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index d04bf799..1416908c 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -31,9 +31,9 @@ Keep these rules in mind while updating the connector: The backend no longer suppresses the ATT&CK Data Model error for a missing `x_mitre_domains` property on campaigns, intrusion sets, detection strategies, -or matrices. Existing latest v19.1 content is repaired automatically at server -startup, including revoked and deprecated lineages, but new reviewed revisions -must carry their own canonical domain membership. +or matrices. Existing latest domainless content is repaired automatically at +server startup, including revoked and deprecated lineages, but new reviewed +revisions must carry their own canonical domain membership. Update the affected Angular create/edit payloads so the field contains the object's complete domain union: From 054678aa9506aade3feac5a5645b9b6f23f3aba3 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:35:24 -0400 Subject: [PATCH 43/55] fix(release-tracks): persist scheduled materialization metadata Accept scheduled materialization metadata on virtual track creation, composition updates, and explicit materialization. Expose persisted values through track and snapshot retrieval endpoints. --- .../definitions/components/release-tracks.yml | 18 +- .../paths/release-tracks-paths.yml | 12 +- app/controllers/release-tracks-controller.js | 5 +- .../release-tracks/release-track-schemas.js | 96 +++--- .../release-track-snapshot-schema.js | 6 + .../release-track-dynamic.repository.js | 2 + .../release-tracks/release-tracks-service.js | 60 +++- .../release-tracks/snapshot-service.js | 5 +- .../release-tracks/virtual-track-service.js | 13 +- .../virtual-scheduled-materialization.spec.js | 288 ++++++++++++++++++ docs/developer/FRONTEND_TODO.md | 16 +- docs/developer/TODO.md | 23 ++ docs/developer/release-tracks/entities.md | 7 + .../release-tracks/implementation-notes.md | 8 + docs/user/release-tracks/api-reference.md | 30 +- docs/user/release-tracks/virtual-tracks.md | 13 + 16 files changed, 545 insertions(+), 57 deletions(-) create mode 100644 app/tests/api/release-tracks/virtual-scheduled-materialization.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 91c09b91..63abdcdb 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -91,8 +91,10 @@ components: scheduled_materialization: nullable: true description: | - Server-controlled provenance for a virtual draft created by a - snapshot schedule. Manually created snapshots omit this property. + Immutable per-snapshot materialization metadata for virtual + tracks. Clients may set it during virtual-track creation or + composition update; the scheduler sets the same shape on drafts + that it creates. $ref: '#/components/schemas/scheduled-materialization' config: $ref: '#/components/schemas/track-config' @@ -168,6 +170,9 @@ components: type: string enum: - virtual + scheduled_materialization: + nullable: true + $ref: '#/components/schemas/scheduled-materialization' quarantine_count: type: integer minimum: 0 @@ -574,6 +579,10 @@ components: nullable: true description: 'Snapshot creation schedule for virtual tracks' $ref: '#/components/schemas/snapshot-schedule' + scheduled_materialization: + nullable: true + description: 'Materialization metadata from the latest virtual snapshot' + $ref: '#/components/schemas/scheduled-materialization' tagged-release-reference: type: object @@ -670,7 +679,10 @@ components: scheduled-materialization: type: object - description: 'Immutable scheduler occurrence that created a virtual draft' + additionalProperties: false + description: | + Immutable materialization occurrence attached to one virtual snapshot. + It may be supplied by a client write or by the virtual-track scheduler. required: - schedule_mode - scheduled_for diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 39884070..2cbb20d3 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -233,6 +233,9 @@ paths: members are unsupported. Virtual snapshot schedules are strict by mode: manual accepts no selector, cron requires cron, and dates requires at least one date. Standard tracks reject snapshot_schedule. + Virtual tracks may also accept a strict scheduled_materialization + object containing schedule_mode and scheduled_for; it is persisted on + the initial snapshot and returned by snapshot and track-list GETs. tags: - 'Release Tracks' # Request body validation moved to Zod in controller @@ -834,6 +837,8 @@ paths: rejects selector fields; specific_version requires version; specific_snapshot requires snapshot. Component IDs and required non-negative integer priorities must each be unique. + An optional strict scheduled_materialization object is persisted on the + new pending virtual draft and returned by snapshot and track-list GETs. tags: - 'Release Tracks' parameters: @@ -867,7 +872,9 @@ paths: object_ref and object_modified revision. The resulting snapshot never follows later component track_latest activity, and snapshot retrieval does not re-resolve composition. - Request body validated via Zod in controller. + Request body validated via Zod in controller. Clients may attach an + optional strict scheduled_materialization object to the resulting + virtual draft. tags: - 'Release Tracks' parameters: @@ -938,7 +945,8 @@ paths: Return lightweight summaries of every snapshot in the release track, ordered by modified timestamp from newest to oldest. Standard snapshot summaries contain members, staged, and candidates counts. Virtual - snapshot summaries contain members and quarantine counts. + snapshot summaries contain members and quarantine counts, plus + scheduled_materialization when present. tags: - 'Release Tracks' parameters: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 4b1f9fec..0431ea36 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -954,8 +954,11 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(req, res, n ); } + const { scheduled_materialization: scheduledMaterialization, ...snapshotOptions } = + bodyResult.data || {}; const result = await releaseTracksService.createVirtualSnapshot(req.params.id, { - ...(bodyResult.data || {}), + ...snapshotOptions, + scheduledMaterialization, userAccountId: req.user?.userAccountId, }); logger.debug(`Success: Created virtual snapshot for track ${req.params.id}`); diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 58c151f5..62077d35 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -291,6 +291,13 @@ const snapshotScheduleSchema = z.discriminatedUnion('mode', [ .strict(), ]); +const scheduledMaterializationSchema = z + .object({ + schedule_mode: z.enum(['cron', 'dates']), + scheduled_for: z.iso.datetime(), + }) + .strict(); + const releaseTrackObjectTypes = Object.freeze(Object.values(types)); const releaseTrackObjectTypeSchema = z.enum(releaseTrackObjectTypes); const objectTypesFilterSchema = z @@ -341,41 +348,45 @@ const componentTrackSchema = z.discriminatedUnion('resolution_strategy', [ .strict(), ]); +const compositionShape = { + component_tracks: z.array(componentTrackSchema).min(1), + deduplication: z + .object({ + strategy: deduplicationStrategySchema, + }) + .strict() + .optional(), +}; + +function validateCompositionUniqueness(composition, context) { + const trackIds = new Set(); + const priorities = new Set(); + + composition.component_tracks.forEach((component, index) => { + if (trackIds.has(component.track_id)) { + context.addIssue({ + code: 'custom', + path: ['component_tracks', index, 'track_id'], + message: 'Each component track must reference a unique track', + }); + } + trackIds.add(component.track_id); + + if (priorities.has(component.priority)) { + context.addIssue({ + code: 'custom', + path: ['component_tracks', index, 'priority'], + message: 'Each component track must have a unique priority value', + }); + } + priorities.add(component.priority); + }); +} + const compositionSchema = z - .object({ - component_tracks: z.array(componentTrackSchema).min(1), - deduplication: z - .object({ - strategy: deduplicationStrategySchema, - }) - .strict() - .optional(), - }) + .object(compositionShape) .strict() - .superRefine((composition, context) => { - const trackIds = new Set(); - const priorities = new Set(); - - composition.component_tracks.forEach((component, index) => { - if (trackIds.has(component.track_id)) { - context.addIssue({ - code: 'custom', - path: ['component_tracks', index, 'track_id'], - message: 'Each component track must reference a unique track', - }); - } - trackIds.add(component.track_id); - - if (priorities.has(component.priority)) { - context.addIssue({ - code: 'custom', - path: ['component_tracks', index, 'priority'], - message: 'Each component track must have a unique priority value', - }); - } - priorities.add(component.priority); - }); - }); + .superRefine(validateCompositionUniqueness); const createTrackBodySchema = z .object({ @@ -385,6 +396,7 @@ const createTrackBodySchema = z object_marking_refs: z.array(stixIdentifierSchema).optional(), composition: compositionSchema.optional(), snapshot_schedule: snapshotScheduleSchema.optional(), + scheduled_materialization: scheduledMaterializationSchema.optional(), config: updateConfigBodySchema.optional(), }) .strict() @@ -396,6 +408,13 @@ const createTrackBodySchema = z message: 'Snapshot schedules are only available for virtual tracks', }); } + if (track.type !== 'virtual' && track.scheduled_materialization !== undefined) { + context.addIssue({ + code: 'custom', + path: ['scheduled_materialization'], + message: 'Scheduled materialization is only available for virtual tracks', + }); + } }); /** POST /release-tracks/new-from-bundle */ @@ -486,13 +505,21 @@ const updateCandidateVersionBodySchema = z.object({ }); /** PUT /release-tracks/:id/virtual/composition */ -const updateCompositionBodySchema = compositionSchema; +const updateCompositionBodySchema = z + .object({ + ...compositionShape, + scheduled_materialization: scheduledMaterializationSchema.optional(), + }) + .strict() + .superRefine(validateCompositionUniqueness); /** POST /release-tracks/:id/virtual/snapshots/create */ const createVirtualSnapshotBodySchema = z .object({ description: z.string().optional(), + scheduled_materialization: scheduledMaterializationSchema.optional(), }) + .strict() .optional(); /** POST /release-tracks/:id/virtual/quarantine/promote */ @@ -571,6 +598,7 @@ module.exports = { componentTrackSchema, compositionSchema, snapshotScheduleSchema, + scheduledMaterializationSchema, objectRefEntrySchema, promotionConflictsSchema, memberSyncConfigSchema, diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index ee25cfcc..e6d6abf2 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -398,6 +398,12 @@ const releaseTrackSnapshotDefinition = { scheduled_materialization: { type: scheduledMaterializationSchema, default: undefined, + validate: { + validator: function validateScheduledMaterialization(value) { + return value === undefined || this.type === 'virtual'; + }, + message: 'Scheduled materialization is only valid for virtual tracks', + }, }, // --- Shared --- diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 80926151..c3cd981d 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -42,6 +42,7 @@ class ReleaseTrackDynamicRepository { { $project: { _id: 0, + scheduled_materialization: 1, members_count: { $size: { $ifNull: ['$members', []] } }, staged_count: { $size: { $ifNull: ['$staged', []] } }, candidates_count: { $size: { $ifNull: ['$candidates', []] } }, @@ -274,6 +275,7 @@ class ReleaseTrackDynamicRepository { version: 1, name: 1, description: 1, + scheduled_materialization: 1, members_count: { $size: { $ifNull: ['$members', []] } }, staged_count: { $size: { $ifNull: ['$staged', []] } }, candidates_count: { $size: { $ifNull: ['$candidates', []] } }, diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 234f52ca..038946c5 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -18,6 +18,7 @@ const { BadRequestError, NotImplementedError } = require('../../exceptions'); const { compositionSchema, snapshotScheduleSchema, + scheduledMaterializationSchema, } = require('../../lib/release-tracks/release-track-schemas'); const snapshotService = require('./snapshot-service'); const standardTrackService = require('./standard-track-service'); @@ -41,6 +42,22 @@ function notImplemented(methodName) { throw new NotImplementedError(MODULE, methodName); } +function validateScheduledMaterialization(value) { + const scheduledFor = value?.scheduled_for; + const normalizedValue = + scheduledFor instanceof Date && !Number.isNaN(scheduledFor.getTime()) + ? { ...value, scheduled_for: scheduledFor.toISOString() } + : value; + const result = scheduledMaterializationSchema.safeParse(normalizedValue); + if (!result.success) { + throw new BadRequestError({ + message: 'Invalid scheduled materialization', + details: result.error.errors, + }); + } + return result.data; +} + function destructiveIdentity(trackId, actor, confirmation) { return { actor: actor || { @@ -210,6 +227,28 @@ exports.getReleasesByObject = function getReleasesByObject(objectRef, options) { exports.createTrack = async function createTrack(data) { let validatedData = data; + if (data.scheduled_materialization !== undefined) { + if (data.type !== 'virtual') { + throw new BadRequestError({ + message: 'Scheduled materialization is only available for virtual release tracks', + }); + } + + const materializationResult = scheduledMaterializationSchema.safeParse( + data.scheduled_materialization, + ); + if (!materializationResult.success) { + throw new BadRequestError({ + message: 'Invalid scheduled materialization', + details: materializationResult.error.errors, + }); + } + validatedData = { + ...validatedData, + scheduled_materialization: materializationResult.data, + }; + } + if (data.snapshot_schedule !== undefined) { if (data.type !== 'virtual') { throw new BadRequestError({ @@ -421,18 +460,33 @@ exports.updateConfig = function updateConfig(trackId, config, userId) { // ----------------------------------------------------------------------------- exports.updateComposition = function updateComposition(trackId, composition, userId) { - const compositionResult = compositionSchema.safeParse(composition); + const { scheduled_materialization: scheduledMaterialization, ...compositionData } = + composition || {}; + let validatedScheduledMaterialization = scheduledMaterialization; + const compositionResult = compositionSchema.safeParse(compositionData); if (!compositionResult.success) { throw new BadRequestError({ message: 'Invalid virtual track composition', details: compositionResult.error.errors, }); } - return virtualTrackService.updateComposition(trackId, compositionResult.data, userId); + if (scheduledMaterialization !== undefined) { + validatedScheduledMaterialization = validateScheduledMaterialization(scheduledMaterialization); + } + return virtualTrackService.updateComposition(trackId, compositionResult.data, userId, { + scheduledMaterialization: validatedScheduledMaterialization, + }); }; exports.createVirtualSnapshot = function createVirtualSnapshot(trackId, options) { - return virtualTrackService.createVirtualSnapshot(trackId, options); + let validatedOptions = options; + if (options?.scheduledMaterialization !== undefined) { + validatedOptions = { + ...options, + scheduledMaterialization: validateScheduledMaterialization(options.scheduledMaterialization), + }; + } + return virtualTrackService.createVirtualSnapshot(trackId, validatedOptions); }; exports.promoteQuarantinedObject = function promoteQuarantinedObject(trackId, selection) { diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 1a4ba38d..8c4f2ea9 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -153,6 +153,7 @@ exports.listTracks = async function listTracks(options) { const summary = await dynamicRepo.getLatestSnapshotTierSummary(track.track_id); return { ...track, + scheduled_materialization: summary?.scheduled_materialization, summary: normalizeTierSummary(summary), }; }), @@ -167,7 +168,7 @@ exports.listTracks = async function listTracks(options) { /** * Create a new release track with an initial empty draft snapshot. * - * @param {Object} data - { name, description?, type, userAccountId?, object_marking_refs?, composition?, snapshot_schedule?, config? } + * @param {Object} data - { name, description?, type, userAccountId?, object_marking_refs?, composition?, snapshot_schedule?, scheduled_materialization?, config? } * @returns {Promise} The initial snapshot document */ exports.createTrack = async function createTrack(data) { @@ -190,6 +191,7 @@ exports.createTrack = async function createTrack(data) { candidates: trackType === 'standard' ? [] : undefined, quarantine: trackType === 'virtual' ? [] : undefined, composition: trackType === 'virtual' ? data.composition : undefined, + scheduled_materialization: trackType === 'virtual' ? data.scheduled_materialization : undefined, config: data.config || {}, version_history: [], }; @@ -254,6 +256,7 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { if (snapshot.type === 'virtual') { return { ...common, + scheduled_materialization: snapshot.scheduled_materialization, quarantine_count: snapshot.quarantine_count, }; } diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 58a2bf30..05500382 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -426,11 +426,17 @@ async function resolveComposition(snapshot, registryMap) { * * @param {string} trackId * @param {Object} composition - The new composition configuration - * @param {string} [userId] + * @param {string} [_userId] + * @param {Object} [options] + * @param {Object} [options.scheduledMaterialization] * @returns {Promise} The new snapshot */ -// eslint-disable-next-line no-unused-vars -exports.updateComposition = async function updateComposition(trackId, composition, userId) { +exports.updateComposition = async function updateComposition( + trackId, + composition, + _userId, + options = {}, +) { const source = await snapshotService.getLatestSnapshot(trackId); assertVirtualTrack(source); @@ -442,6 +448,7 @@ exports.updateComposition = async function updateComposition(trackId, compositio members: [], quarantine: [], composition_resolution: null, + scheduled_materialization: options.scheduledMaterialization, }); logger.verbose( diff --git a/app/tests/api/release-tracks/virtual-scheduled-materialization.spec.js b/app/tests/api/release-tracks/virtual-scheduled-materialization.spec.js new file mode 100644 index 00000000..9b030104 --- /dev/null +++ b/app/tests/api/release-tracks/virtual-scheduled-materialization.spec.js @@ -0,0 +1,288 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const modelFactory = require('../../../models/release-tracks/model-factory'); +const releaseTracksService = require('../../../services/release-tracks/release-tracks-service'); + +describe('Virtual release-track scheduled materialization API', function () { + let app; + let passportCookie; + + const createdMaterialization = { + schedule_mode: 'dates', + scheduled_for: '2027-01-15T00:00:00.000Z', + }; + const updatedMaterialization = { + schedule_mode: 'cron', + scheduled_for: '2027-07-15T00:00:00.000Z', + }; + const snapshotMaterialization = { + schedule_mode: 'dates', + scheduled_for: '2028-01-15T00:00:00.000Z', + }; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + function api(method, path) { + return request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + it('persists a client-supplied value on virtual-track creation and every GET representation', async function () { + const createResponse = await api('post', '/api/release-tracks/new') + .send({ + name: 'Scheduled Materialization Create', + type: 'virtual', + scheduled_materialization: createdMaterialization, + }) + .expect(201); + + expect(createResponse.body.scheduled_materialization).toEqual(createdMaterialization); + const trackId = createResponse.body.id; + const modified = createResponse.body.modified; + + const latestResponse = await api( + 'get', + `/api/release-tracks/${trackId}/snapshots/latest`, + ).expect(200); + expect(latestResponse.body.scheduled_materialization).toEqual(createdMaterialization); + + const selectedResponse = await api( + 'get', + `/api/release-tracks/${trackId}/snapshots/${modified}`, + ).expect(200); + expect(selectedResponse.body.scheduled_materialization).toEqual(createdMaterialization); + + const historyResponse = await api('get', `/api/release-tracks/${trackId}/snapshots`).expect( + 200, + ); + expect(historyResponse.body.data[0].scheduled_materialization).toEqual(createdMaterialization); + + const listResponse = await api('get', '/api/release-tracks') + .query({ search: 'Scheduled Materialization Create' }) + .expect(200); + expect(listResponse.body.data[0].scheduled_materialization).toEqual(createdMaterialization); + }); + + it('persists a client-supplied value on virtual composition update and every GET representation', async function () { + const componentResponse = await api('post', '/api/release-tracks/new') + .send({ + name: 'Scheduled Materialization Component', + type: 'standard', + }) + .expect(201); + + const virtualResponse = await api('post', '/api/release-tracks/new') + .send({ + name: 'Scheduled Materialization Update', + type: 'virtual', + }) + .expect(201); + + const updateResponse = await api( + 'put', + `/api/release-tracks/${virtualResponse.body.id}/virtual/composition`, + ) + .send({ + component_tracks: [ + { + track_id: componentResponse.body.id, + resolution_strategy: 'latest_tagged', + priority: 0, + }, + ], + scheduled_materialization: updatedMaterialization, + }) + .expect(200); + + expect(updateResponse.body.scheduled_materialization).toEqual(updatedMaterialization); + const trackId = virtualResponse.body.id; + const modified = updateResponse.body.modified; + + const latestResponse = await api( + 'get', + `/api/release-tracks/${trackId}/snapshots/latest`, + ).expect(200); + expect(latestResponse.body.scheduled_materialization).toEqual(updatedMaterialization); + + const selectedResponse = await api( + 'get', + `/api/release-tracks/${trackId}/snapshots/${modified}`, + ).expect(200); + expect(selectedResponse.body.scheduled_materialization).toEqual(updatedMaterialization); + + const historyResponse = await api('get', `/api/release-tracks/${trackId}/snapshots`).expect( + 200, + ); + expect(historyResponse.body.data[0].scheduled_materialization).toEqual(updatedMaterialization); + + const listResponse = await api('get', '/api/release-tracks') + .query({ search: 'Scheduled Materialization Update' }) + .expect(200); + expect(listResponse.body.data[0].scheduled_materialization).toEqual(updatedMaterialization); + + await api('put', `/api/release-tracks/${trackId}/virtual/composition`) + .send({ + component_tracks: [ + { + track_id: componentResponse.body.id, + resolution_strategy: 'latest_tagged', + priority: 0, + }, + ], + scheduled_materialization: { + ...updatedMaterialization, + unexpected: true, + }, + }) + .expect(400); + }); + + it('persists a client-supplied value on explicit virtual snapshot creation', async function () { + const componentResponse = await api('post', '/api/release-tracks/new') + .send({ + name: 'Explicit Materialization Component', + type: 'standard', + }) + .expect(201); + + await api('post', `/api/release-tracks/${componentResponse.body.id}/snapshots/latest/release`) + .send({}) + .expect(200); + + const virtualResponse = await api('post', '/api/release-tracks/new') + .send({ + name: 'Explicit Scheduled Materialization', + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: componentResponse.body.id, + resolution_strategy: 'latest_tagged', + priority: 0, + }, + ], + }, + }) + .expect(201); + + const materializedResponse = await api( + 'post', + `/api/release-tracks/${virtualResponse.body.id}/virtual/snapshots/create`, + ) + .send({ + description: 'Client-attributed materialization', + scheduled_materialization: snapshotMaterialization, + }) + .expect(201); + + expect(materializedResponse.body.scheduled_materialization).toEqual(snapshotMaterialization); + + const latestResponse = await api( + 'get', + `/api/release-tracks/${virtualResponse.body.id}/snapshots/latest`, + ).expect(200); + expect(latestResponse.body.scheduled_materialization).toEqual(snapshotMaterialization); + + await api('post', `/api/release-tracks/${virtualResponse.body.id}/virtual/snapshots/create`) + .send({ + scheduled_materialization: { + schedule_mode: 'manual', + scheduled_for: '2028-07-15T00:00:00.000Z', + }, + }) + .expect(400); + }); + + it('rejects scheduled materialization on standard tracks and malformed virtual payloads', async function () { + await api('post', '/api/release-tracks/new') + .send({ + name: 'Invalid Standard Materialization', + type: 'standard', + scheduled_materialization: createdMaterialization, + }) + .expect(400); + + const malformedValues = [ + { + schedule_mode: 'manual', + scheduled_for: '2027-01-15T00:00:00.000Z', + }, + { + schedule_mode: 'cron', + }, + { + schedule_mode: 'dates', + scheduled_for: 'not-a-date', + }, + { + ...createdMaterialization, + unexpected: true, + }, + ]; + + for (const scheduledMaterialization of malformedValues) { + await api('post', '/api/release-tracks/new') + .send({ + name: 'Invalid Virtual Materialization', + type: 'virtual', + scheduled_materialization: scheduledMaterialization, + }) + .expect(400); + } + }); + + it('repeats validation for non-HTTP service and persistence callers', async function () { + await expect( + releaseTracksService.createTrack({ + name: 'Invalid Service Materialization', + type: 'standard', + scheduled_materialization: createdMaterialization, + }), + ).rejects.toThrow('Scheduled materialization is only available'); + + await expect( + releaseTracksService.createTrack({ + name: 'Malformed Service Materialization', + type: 'virtual', + scheduled_materialization: { + schedule_mode: 'dates', + scheduled_for: 'not-a-date', + }, + }), + ).rejects.toThrow('Invalid scheduled materialization'); + + const trackId = 'release-track--11111111-1111-4111-8111-111111111111'; + const Model = modelFactory.getModel(trackId); + const invalidSnapshot = new Model({ + id: trackId, + type: 'standard', + modified: new Date(), + version: null, + name: 'Invalid Persistence Materialization', + created: new Date(), + scheduled_materialization: createdMaterialization, + }); + + await expect(invalidSnapshot.validate()).rejects.toThrow( + 'Scheduled materialization is only valid for virtual tracks', + ); + }); +}); diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index 1416908c..e891c5e3 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -826,7 +826,7 @@ downtime, and due dates are recovered after restart. A component-resolution failure is retried by the backend; the UI does not need to resubmit the schedule. -Scheduled virtual drafts include read-only provenance: +Virtual drafts may include per-snapshot materialization metadata: ```ts scheduled_materialization?: { @@ -835,8 +835,13 @@ scheduled_materialization?: { }; ``` -Use it to identify scheduled drafts where useful, but never include it in -create or update payloads. +Clients may send this strict shape during `POST /api/release-tracks/new` for a +virtual track, `PUT /api/release-tracks/:id/virtual/composition`, and +`POST /api/release-tracks/:id/virtual/snapshots/create`. Include it only when +deliberately attaching the occurrence to the new snapshot; later snapshot +mutations do not inherit it. Standard tracks must omit it. Read it from track +listing, snapshot history, latest snapshot, or timestamp-selected snapshot +responses. Done when: @@ -848,8 +853,9 @@ Done when: - Tests cover all three modes and mode switching. - User-facing copy explains UTC execution and the difference between cron and restart-recoverable dates. -- Scheduled drafts tolerate and preserve the read-only - `scheduled_materialization` response property. +- Virtual create and composition update forms can deliberately submit + `scheduled_materialization`, and all supported GET representations tolerate + and preserve it. ## P1 — Align virtual component object-type filters diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 66a27c32..8906b022 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,28 @@ # Release Track TODOs +## Client-managed virtual scheduled materialization + +- [x] Add API regressions proving virtual-track POST and composition PUT + requests persist `scheduled_materialization`. +- [x] Validate the client-supplied shape at controller, service, and Mongoose + boundaries and reject it for standard tracks. +- [x] Expose the value through track listing, snapshot history, latest + snapshot, and timestamp-selected snapshot GET responses. +- [x] Align OpenAPI, user/developer documentation, frontend guidance, and + Bruno requests with the client-managed contract. +- [x] Run the focused regression spec followed by the complete `npm test` + suite, review the final diff, and propose a conventional commit message. + +Verification (2026-07-30): + +- Focused scheduler and scheduled-materialization API regressions pass: 13 + cases. +- Previously roaming group-query and virtual-deduplication failures pass in + isolation: 14 cases. +- The complete `npm test` suite passes, including OpenAPI, configuration, API, + middleware, and scheduler stages. +- Backend lint and diff whitespace validation pass. + ## Caller-supplied configuration on track creation - [x] Add a regression proving `POST /api/release-tracks/new` accepts and diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index bc0743f2..13fc10b3 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -483,6 +483,13 @@ scheduled_materialization: { } ``` +The scheduler writes this object for automated occurrences, and API clients +may write the same strict virtual-only shape during initial track creation or +composition update, as well as explicit virtual materialization. It is stored +on the resulting snapshot and projected into track-list and snapshot-history +responses. Snapshot clones clear inherited occurrence metadata unless the +mutation explicitly supplies a replacement. + The track-local unique index on `scheduled_for`, together with the durable `virtualTrackScheduleOccurrences` claim record, makes duplicate delivery and restart recovery idempotent. Failed occurrences remain retryable. diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 74a3c130..3385c7af 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -242,6 +242,14 @@ ISO timestamps. Standard-track creation rejects `snapshot_schedule` instead of silently dropping it. Mongoose repeats the mode and track-type invariants for direct persistence callers. +`scheduled_materialization` uses a separate strict virtual-only schema. Track +creation, composition update, and explicit virtual-materialization requests +can attach it to the snapshot they create; the scheduler uses that same +service input for automated occurrences. Full snapshot reads return the stored +object directly, while track listing and snapshot history explicitly project +it. Ordinary clones clear inherited occurrence metadata so it never migrates +to a different snapshot implicitly. + The virtual snapshot scheduler reconciles persisted schedules at startup and on `VIRTUAL_TRACK_SCHEDULES_CRON`. Cron jobs use `Etc/UTC`; explicit dates at or before the reconciliation time become durable occurrences. Atomic diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 749befba..c092c3b2 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -1158,6 +1158,10 @@ POST /api/release-tracks/new "snapshot_schedule": { "mode": "cron", "cron": "0 0 1 1,7 *" + }, + "scheduled_materialization": { + "schedule_mode": "cron", + "scheduled_for": "2027-01-01T00:00:00.000Z" } } ``` @@ -1202,8 +1206,11 @@ Cron expressions and explicit dates are interpreted in UTC. Cron occurrences run while the scheduler is active; they are not backfilled after downtime. Every due date is recovered after restart and creates exactly one draft. Failed cron and date occurrences are retried by the scheduler. Scheduled -drafts include a server-controlled `scheduled_materialization` object with -`schedule_mode` and `scheduled_for`; manual drafts omit it. +drafts include a `scheduled_materialization` object with `schedule_mode` and +`scheduled_for`. Clients may set the same strict object when creating a +virtual track. Standard tracks reject it. The value is attached immutably to +that snapshot and is returned by track listing, snapshot history, latest +snapshot, and timestamp-selected snapshot GET requests. Composition, component, filter, and deduplication objects are strict. Unknown keys, including the incorrect singular `filters.domain`, return @@ -1238,13 +1245,19 @@ PUT /api/release-tracks/:id/virtual/composition "version": "2.0", "priority": 1 } - ] + ], + "scheduled_materialization": { + "schedule_mode": "dates", + "scheduled_for": "2027-07-01T00:00:00.000Z" + } } ``` The same strict composition and selector validation applies to this update operation. Invalid fields are rejected rather than removed from the persisted -configuration. Component track IDs and priorities must each be unique. +configuration. Component track IDs and priorities must each be unique. The +optional `scheduled_materialization` value uses the same strict shape as +creation and is persisted on the new pending virtual draft. **Note:** Updating composition creates a pending draft containing the new rules. To prevent stale materialization from being released, the draft has @@ -1264,10 +1277,17 @@ POST /api/release-tracks/:id/virtual/snapshots/create ```json { - "description": "Q1 2024 snapshot" + "description": "Q1 2024 snapshot", + "scheduled_materialization": { + "schedule_mode": "dates", + "scheduled_for": "2027-07-01T00:00:00.000Z" + } } ``` +`scheduled_materialization` is optional and follows the same strict, +virtual-only contract as track creation and composition update. + **Response:** ```json diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 00d1929e..8ce04995 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -576,6 +576,19 @@ validation, and persistence path as } ``` +Clients may attach the same strict object to the initial virtual snapshot with +`POST /api/release-tracks/new`, or to the pending draft created by +`PUT /api/release-tracks/:id/virtual/composition`, or to an explicitly +materialized draft with +`POST /api/release-tracks/:id/virtual/snapshots/create`. `schedule_mode` must +be `cron` or `dates`, `scheduled_for` must be an ISO timestamp, and unknown +keys are rejected. Standard tracks cannot set this property. + +The persisted value is observable through `GET /api/release-tracks`, snapshot +history, latest-snapshot retrieval, and timestamp-selected snapshot retrieval. +It belongs to one immutable snapshot occurrence; later snapshot clones omit it +unless the write creating that snapshot supplies a new value. + ### 2. Snapshot Review Before tagging, team reviews the draft snapshot: From bde1bbc43d61af59778f615d9020a2142c38f8e2 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:12:47 -0400 Subject: [PATCH 44/55] test(release-tracks): verify STIX 2.0 virtual bundles Add virtual-specific API coverage confirming the existing shared bundle serializer emits STIX 2.0 on request and preserves the STIX 2.1 default. Clarify the established behavior in OpenAPI and release-track documentation. --- .../paths/release-tracks-paths.yml | 10 +- app/services/release-tracks/export-service.js | 4 +- .../api/release-tracks/virtual-bundle.spec.js | 141 ++++++++++++++++++ docs/developer/TODO.md | 19 +++ .../developer/release-tracks/bundle-export.md | 4 +- docs/user/release-tracks/virtual-tracks.md | 11 ++ 6 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 app/tests/api/release-tracks/virtual-bundle.spec.js diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 2cbb20d3..95b50293 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -1067,7 +1067,10 @@ paths: type: string - name: stixVersion in: query - description: 'STIX version for bundle responses' + description: | + STIX version for standard or materialized virtual snapshot bundle + responses. STIX 2.0 adds spec_version to the bundle envelope and + removes it from each bundled object; STIX 2.1 does the inverse. schema: type: string enum: @@ -1162,7 +1165,10 @@ paths: - name: stixVersion in: query description: | - STIX version that the exported bundle should conform to (bundle format only). + STIX version that the exported standard or materialized virtual + snapshot bundle should conform to (bundle format only). STIX 2.0 + adds spec_version to the bundle envelope and removes it from each + bundled object; STIX 2.1 does the inverse. schema: type: string enum: diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index 06911881..05102fcf 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -5,7 +5,7 @@ // // Hydrates STIX object refs (from snapshot members/staged/candidates tiers) // into full STIX documents, then formats the output as one of: -// - bundle: Standard STIX 2.1 bundle +// - bundle: Standard STIX 2.0 or 2.1 bundle // - workbench: Custom format with workflow metadata // - filesystemstore: Directory structure organized by STIX type // @@ -127,6 +127,8 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd * returns the release-track snapshot shape with UI-friendly tier entry details. * * Bundle exports (see docs/developer/release-tracks/bundle-export.md): + * - The same pipeline applies to standard snapshots and materialized virtual + * snapshots because both persist exact member revisions and graph manifests. * 1. Select tier entries — members always; staged/candidates via * options.include, narrowed by options.state * 2. Hydrate entries into full documents diff --git a/app/tests/api/release-tracks/virtual-bundle.spec.js b/app/tests/api/release-tracks/virtual-bundle.spec.js new file mode 100644 index 00000000..76839c7a --- /dev/null +++ b/app/tests/api/release-tracks/virtual-bundle.spec.js @@ -0,0 +1,141 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); + +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Virtual Release Track Bundle Export API', function () { + let app; + let passportCookie; + let malware; + let virtualTrack; + let virtualSnapshot; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + + malware = await post('/api/software', buildMalware('Virtual Bundle Malware')); + + const componentTrack = await post('/api/release-tracks/new', { + name: 'Virtual Bundle Component', + type: 'standard', + }); + await releaseExactMembers(app, passportCookie, componentTrack.id, [malware]); + + virtualTrack = await post('/api/release-tracks/new', { + name: 'Virtual Bundle Track', + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: componentTrack.id, + resolution_strategy: 'latest_tagged', + priority: 0, + }, + ], + deduplication: { strategy: 'prioritize_latest_object' }, + }, + }); + virtualSnapshot = await post( + `/api/release-tracks/${virtualTrack.id}/virtual/snapshots/create`, + {}, + ); + }); + + async function post(path, body, expectedStatus = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(expectedStatus); + return response.body; + } + + async function get(path) { + const response = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + return response.body; + } + + function buildMalware(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'malware', + is_family: true, + object_marking_refs: [staticMarkingDefinitionId], + x_mitre_version: '1.0', + x_mitre_aliases: [name], + x_mitre_platforms: ['Windows'], + x_mitre_domains: ['enterprise-attack'], + }, + }; + } + + it('emits materialized virtual snapshots as STIX 2.1 bundles by default', async function () { + const bundle = await get( + `/api/release-tracks/${virtualTrack.id}/snapshots/latest?format=bundle`, + ); + + expect(bundle.type).toBe('bundle'); + expect(bundle.spec_version).toBeUndefined(); + expect(bundle.objects[0]).toMatchObject({ + type: 'x-mitre-collection', + spec_version: '2.1', + }); + + const exportedMalware = bundle.objects.find((object) => object.id === malware.stix.id); + expect(exportedMalware).toMatchObject({ + type: 'malware', + spec_version: '2.1', + is_family: true, + }); + expect(exportedMalware.labels).toBeUndefined(); + }); + + it('emits materialized virtual snapshots as STIX 2.0 bundles on request', async function () { + const bundle = await get( + `/api/release-tracks/${virtualTrack.id}/snapshots/` + + `${encodeURIComponent(virtualSnapshot.modified)}?format=bundle&stixVersion=2.0`, + ); + + expect(bundle.type).toBe('bundle'); + expect(bundle.spec_version).toBe('2.0'); + expect(bundle.objects.every((object) => object.spec_version === undefined)).toBe(true); + + const exportedMalware = bundle.objects.find((object) => object.id === malware.stix.id); + expect(exportedMalware).toMatchObject({ + type: 'malware', + labels: ['malware'], + }); + expect(exportedMalware.is_family).toBeUndefined(); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 8906b022..0075b63d 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,24 @@ # Release Track TODOs +## STIX 2.0 virtual snapshot bundles + +- [x] Add a virtual-track regression proving materialized snapshots emit STIX + 2.0 bundles when `stixVersion=2.0` and remain STIX 2.1 by default. +- [x] Align the virtual snapshot OpenAPI, user documentation, and Bruno request + with the explicit STIX-version contract. +- [x] Run the focused regression followed by the complete `npm test` suite, + review the final diff, and propose a conventional commit message. + +Verification (2026-07-30): + +- Virtual STIX-version bundle regression passes: 2 cases. +- Existing snapshot-bundle regression passes: 17 cases. +- Backend lint, Prettier, and diff whitespace validation pass. +- One aggregate attempt exposed the documented roaming References search 404; + the affected spec passed all 17 cases in isolation. +- The clean complete suite passes: OpenAPI 2, config 21, API 989, middleware + 29, and scheduler 10. + ## Client-managed virtual scheduled materialization - [x] Add API regressions proving virtual-track POST and composition PUT diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 509e82be..3ccdcc66 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -91,7 +91,9 @@ Implemented in [export-service.js](../../../app/services/release-tracks/export-service.js) (`exportSnapshot`) with the DTO transformation in [export-schemas.js](../../../app/lib/release-tracks/export-schemas.js) -(`bundleTransformSchema`). The pipeline: +(`bundleTransformSchema`). Standard snapshots and materialized virtual +snapshots use this same pipeline; virtual composition metadata does not alter +STIX version serialization. The pipeline: 1. **Tier selection** — members are always exported. `include` (values `staged` and/or `candidates`; singular forms accepted) adds tiers. diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 8ce04995..d13486f5 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -681,7 +681,11 @@ virtual-only property. Export virtual track snapshot as STIX bundle: ```bash +# STIX 2.1 (default) GET /api/release-tracks/:id/snapshots/:modified?format=bundle + +# STIX 2.0 +GET /api/release-tracks/:id/snapshots/:modified?format=bundle&stixVersion=2.0 ``` **Response:** @@ -708,6 +712,13 @@ GET /api/release-tracks/:id/snapshots/:modified?format=bundle } ``` +The default is STIX 2.1. Set `stixVersion=2.0` to serialize the same exact +materialized revision set under the STIX 2.0 rules used by the legacy bundle +exporter. A STIX 2.0 bundle carries `spec_version: "2.0"` on its envelope and +omits `spec_version` from its objects; a STIX 2.1 bundle omits the envelope +property and declares `spec_version: "2.1"` on each object. Version-specific +object conversion also applies, including malware/tool label handling. + **Note:** The exported bundle is **materialized** - it contains concrete object references, not composition metadata. Consumers see a standard STIX bundle, unaware it came from a virtual track. ## Composition Resolution Details From 8090dbcc9a1dd6e8b55e896a59ba65b82dc12fb4 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:47:57 -0400 Subject: [PATCH 45/55] fix(release-tracks): preserve materialized virtual graphs Reuse the captured graph manifest across virtual release previews and commits so tagged bundles retain the audited materialization. Apply component domain filters to secondary traversal and ignore inactive LinkById collisions. --- app/lib/linkById.js | 10 +- .../release-track-graph-manifest-model.js | 2 + .../release-tracks/graph-manifest-service.js | 41 +++- .../release-tracks/release-tracks-service.js | 5 +- .../release-tracks/versioning-service.js | 28 ++- .../virtual-graph-integrity.spec.js | 223 ++++++++++++++++++ docs/developer/TODO.md | 26 ++ .../developer/release-tracks/bundle-export.md | 14 +- docs/user/release-tracks/virtual-tracks.md | 9 + 9 files changed, 344 insertions(+), 14 deletions(-) create mode 100644 app/tests/api/release-tracks/virtual-graph-integrity.spec.js diff --git a/app/lib/linkById.js b/app/lib/linkById.js index c46188f6..e9ecb392 100644 --- a/app/lib/linkById.js +++ b/app/lib/linkById.js @@ -5,7 +5,15 @@ const config = require('../config/config'); // Default implmentation. Retrieves the attack object from the database. async function getAttackObjectFromDatabase(attackId) { - const attackObject = await AttackObject.findOne({ 'workspace.attack_id': attackId }) + const attackObject = await AttackObject.findOne({ + 'workspace.attack_id': attackId, + 'stix.revoked': { $ne: true }, + 'stix.x_mitre_deprecated': { $ne: true }, + }) + // x_mitre_deprecated lives on discriminator schemas rather than the base + // AttackObject schema. Preserve that predicate when Mongoose strictQuery + // is enabled. + .setOptions({ strictQuery: false }) .sort('-stix.modified') .lean() .exec(); diff --git a/app/models/release-tracks/release-track-graph-manifest-model.js b/app/models/release-tracks/release-track-graph-manifest-model.js index 07f049d1..7e2015ce 100644 --- a/app/models/release-tracks/release-track-graph-manifest-model.js +++ b/app/models/release-tracks/release-track-graph-manifest-model.js @@ -58,6 +58,8 @@ const entrySchema = new mongoose.Schema( // Relationship payloads are frozen so description-only corrections do // not change older bundles. Marking definitions are not STIX-versioned, // so their complete payload is frozen for the same replay guarantee. + // Operational baselines may also freeze an exact source-bundle payload + // while retaining the database revision pin as the integrity boundary. frozen_stix: { type: mongoose.Schema.Types.Mixed }, }, { collection: 'releaseTrackGraphManifestEntries' }, diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js index 1ed69ade..872078fa 100644 --- a/app/services/release-tracks/graph-manifest-service.js +++ b/app/services/release-tracks/graph-manifest-service.js @@ -24,6 +24,42 @@ const MUTATION_PROTECTED_ENTRY_FILTER = { ], }; +function normalizeDomain(domain) { + return domain.endsWith('-attack') ? domain : `${domain}-attack`; +} + +function virtualSnapshotDomains(snapshot) { + if (snapshot.type !== 'virtual') return null; + + const domains = (snapshot.composition?.component_tracks || []).flatMap( + (component) => component.filters?.domains || [], + ); + if (domains.length === 0) return null; + return new Set(domains.map(normalizeDomain)); +} + +function objectDomains(stixObject) { + if (Array.isArray(stixObject.x_mitre_domains)) { + return stixObject.x_mitre_domains; + } + if (stixObject.type === 'x-mitre-matrix') { + return (stixObject.external_references || []) + .map((reference) => reference.external_id) + .filter((externalId) => typeof externalId === 'string' && externalId.endsWith('-attack')); + } + return []; +} + +function secondaryObjectIsValid(document, allowedDomains) { + if (!document) return false; + if (!allowedDomains) return true; + + const domains = objectDomains(document.stix); + return ( + domains.length === 0 || domains.some((domain) => allowedDomains.has(normalizeDomain(domain))) + ); +} + function revisionKey(objectRef, objectModified) { return `${objectRef}::${new Date(objectModified).getTime()}`; } @@ -41,6 +77,7 @@ function endpointFor(relationship, side) { } async function buildManifestEntries(snapshot) { + const allowedDomains = virtualSnapshotDomains(snapshot); const rootRequests = []; for (const tier of TIERS) { for (const entry of snapshot[tier] || []) { @@ -98,7 +135,7 @@ async function buildManifestEntries(snapshot) { policy: { isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, relationshipIsActive: bundleRelationships.relationshipIsActive, - secondaryObjectIsValid: (document) => Boolean(document), + secondaryObjectIsValid: (document) => secondaryObjectIsValid(document, allowedDomains), }, options: { inferDomains: false, @@ -311,7 +348,7 @@ async function replayEntries(entries, manifest, options) { ]), ); for (const entry of entries) { - if (entry.kind === 'relationship' && entry.frozen_stix) { + if (entry.frozen_stix) { documentsByRevision.set(entry.revision_key, { stix: entry.frozen_stix, }); diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 038946c5..ebc3e698 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -423,7 +423,10 @@ async function renderReleasePlan(plan, options) { if (format === 'bundle') { return exportService.exportSnapshot(plan.plannedSnapshot, format, { ...options, - captureGraph: true, + // A virtual release does not alter its members. Replaying the draft's + // persisted graph keeps preview output identical to the graph that will + // be tagged instead of resolving current database state a second time. + captureGraph: plan.sourceSnapshot.type !== 'virtual', }); } return formatWorkbenchSnapshot(plan.plannedSnapshot, options); diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 702511fc..621f68c4 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -281,7 +281,11 @@ async function planLoadedSnapshot(trackId, snapshot, options) { async function commitPlan(plan) { if (plan.blockingError) throw plan.blockingError; - const manifestId = await graphManifestService.prepare(plan.plannedSnapshot); + const reuseVirtualManifest = + plan.sourceSnapshot.type === 'virtual' && Boolean(plan.sourceSnapshot.graph_manifest_id); + const manifestId = reuseVirtualManifest + ? plan.sourceSnapshot.graph_manifest_id + : await graphManifestService.prepare(plan.plannedSnapshot); let tagged; try { @@ -294,12 +298,16 @@ async function commitPlan(plan) { }, }); } catch (err) { - await graphManifestService.discard(manifestId); + if (!reuseVirtualManifest) { + await graphManifestService.discard(manifestId); + } throw err; } if (!tagged) { - await graphManifestService.discard(manifestId); + if (!reuseVirtualManifest) { + await graphManifestService.discard(manifestId); + } await releaseHistoryService.reconcileTaggedReleases(plan.trackId); throw new AlreadyReleasedError('(concurrent release)'); } @@ -307,12 +315,14 @@ async function commitPlan(plan) { // Link the complete pending manifest before activation. The snapshot link // is the durable commit record, and replay can recover a linked pending // manifest if the process stops in this narrow window. - try { - await graphManifestService.activate(manifestId); - } catch (err) { - logger.warn( - `VersioningService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, - ); + if (!reuseVirtualManifest) { + try { + await graphManifestService.activate(manifestId); + } catch (err) { + logger.warn( + `VersioningService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, + ); + } } if ( diff --git a/app/tests/api/release-tracks/virtual-graph-integrity.spec.js b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js new file mode 100644 index 00000000..025bf2cc --- /dev/null +++ b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js @@ -0,0 +1,223 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const AttackObject = require('../../../models/attack-object-model'); +const linkById = require('../../../lib/linkById'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const { + ReleaseTrackGraphManifestEntry, +} = require('../../../models/release-tracks/release-track-graph-manifest-model'); +const { releaseExactMembers } = require('./release-track-test-helpers'); + +const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Virtual release-track graph integrity', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + async function post(path, body, status = 201) { + const response = await request(app) + .post(path) + .send(body) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + async function get(path, status = 200) { + const response = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + function technique(name, domains = ['enterprise-attack'], description = `${name} description`) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [staticMarkingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_domains: domains, + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; + } + + function mitigation(name, domains) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'course-of-action', + object_marking_refs: [staticMarkingDefinitionId], + x_mitre_domains: domains, + }, + }; + } + + async function createVirtual(name, members, domains = ['enterprise-attack']) { + const component = await post('/api/release-tracks/new', { + name: `${name} Component`, + type: 'standard', + }); + await releaseExactMembers(app, passportCookie, component.id, members); + const virtual = await post('/api/release-tracks/new', { + name, + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: component.id, + resolution_strategy: 'latest_tagged', + priority: 0, + filters: { domains }, + }, + ], + deduplication: { strategy: 'prioritize_latest_object' }, + }, + }); + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}); + return virtual; + } + + it('applies virtual domain constraints to relationship secondary objects', async function () { + const enterpriseRoot = await post('/api/techniques', technique('Enterprise Graph Root')); + const mobileSecondary = await post( + '/api/mitigations', + mitigation('Mobile Graph Secondary', ['mobile-attack']), + ); + const relationship = await post('/api/relationships', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: new Date().toISOString(), + modified: new Date().toISOString(), + spec_version: '2.1', + type: 'relationship', + relationship_type: 'mitigates', + source_ref: mobileSecondary.stix.id, + target_ref: enterpriseRoot.stix.id, + object_marking_refs: [staticMarkingDefinitionId], + }, + }); + const virtual = await createVirtual('Enterprise Bounded Graph', [enterpriseRoot]); + + const bundle = await get(`/api/release-tracks/${virtual.id}/snapshots/latest?format=bundle`); + const ids = bundle.objects.map((object) => object.id); + + expect(ids).toContain(enterpriseRoot.stix.id); + expect(ids).not.toContain(mobileSecondary.stix.id); + expect(ids).not.toContain(relationship.stix.id); + }); + + it('does not resolve LinkById through a newer deprecated ATT&CK-ID collision', async function () { + const activeTarget = await post('/api/techniques', technique('Active Link Target')); + const attackId = activeTarget.workspace.attack_id; + const attackReference = activeTarget.stix.external_references.find( + (reference) => reference.external_id === attackId, + ); + const deprecatedCollision = await post( + '/api/mitigations', + mitigation('Deprecated Collision', ['enterprise-attack']), + ); + await AttackObject.collection.updateOne( + { 'stix.id': deprecatedCollision.stix.id }, + { + $set: { + 'workspace.attack_id': attackId, + 'stix.modified': new Date(Date.now() + 60_000), + 'stix.x_mitre_deprecated': true, + }, + }, + ); + const selectedTarget = await linkById.getAttackObjectFromDatabase(attackId); + expect(selectedTarget.stix.id).toBe(activeTarget.stix.id); + const root = await post( + '/api/techniques', + technique('LinkById Root', ['enterprise-attack'], `See (LinkById: ${attackId}).`), + ); + const virtual = await createVirtual('Virtual Link Target Selection', [root]); + + const bundle = await get(`/api/release-tracks/${virtual.id}/snapshots/latest?format=bundle`); + const exportedRoot = bundle.objects.find((object) => object.id === root.stix.id); + + expect(exportedRoot.description).toBe(`See [Active Link Target](${attackReference.url}).`); + }); + + it('reuses the materialized graph for virtual release preview and commit', async function () { + const root = await post('/api/techniques', technique('Frozen Virtual Root')); + const virtual = await createVirtual('Virtual Frozen Release Graph', [root]); + const draft = await dynamicRepo.getLatestSnapshot(virtual.id); + const frozenName = 'Canonical Source-Bundle Name'; + await ReleaseTrackGraphManifestEntry.updateOne( + { + manifest_id: draft.graph_manifest_id, + kind: 'root', + object_ref: root.stix.id, + }, + { + $set: { + frozen_stix: { + ...root.stix, + name: frozenName, + }, + }, + }, + ).exec(); + + const preview = await get( + `/api/release-tracks/${virtual.id}/snapshots/latest/release/preview` + + '?format=bundle&version=1.0', + ); + expect(preview.objects.find((object) => object.id === root.stix.id).name).toBe(frozenName); + + await post( + `/api/release-tracks/${virtual.id}/snapshots/latest/release`, + { version: '1.0' }, + 200, + ); + const released = await dynamicRepo.getLatestSnapshot(virtual.id); + const releasedBundle = await get( + `/api/release-tracks/${virtual.id}/snapshots/latest?format=bundle`, + ); + + expect(released.graph_manifest_id).toBe(draft.graph_manifest_id); + expect(releasedBundle.objects.find((object) => object.id === root.stix.id).name).toBe( + frozenName, + ); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 0075b63d..fe540c0d 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1886,3 +1886,29 @@ Links/references between notes and snapshot objects will be one-to-many. A singl "stix": "StixObject" } ``` +## Deterministic v19.1 virtual-track bootstrap graph + +- [x] Preserve the materialized virtual snapshot graph when previewing and committing a release. +- [x] Enforce virtual component-domain filters throughout graph traversal, including secondary objects. +- [x] Prevent `LinkById` rendering from selecting revoked or deprecated ATT&CK-ID collisions. +- [x] Seed virtual snapshot graph manifests from the canonical v19.1 bundles in the bootstrap script. +- [x] Make bootstrap bundle comparisons detect duplicate revisions and explain unexpected drift. +- [x] Add regression coverage and document the deterministic-primary/non-deterministic-graph boundary. +- [x] Run focused tests and the complete `npm test` suite. +- [x] Delete and recreate only the Enterprise ATT&CK virtual track, then assess its emitted bundle against v19.1. + +Verification (2026-07-30): + +- The clean complete server suite passes: OpenAPI 2, config 21, API 992, + middleware 29, and scheduler 10. +- Focused graph-integrity regressions pass (3), the affected release-track + group passes (40), and the bootstrap regression suite passes (23). +- The guarded bootstrap completed without accepting unexplained bundle drift; + all three standard and virtual baselines are tagged `1.0`. +- Enterprise contains 4,815 exact members, zero quarantine entries, and a + 25,842-object publication graph excluding its generated collection. It has + no missing, additional, or duplicate STIX IDs; type counts and all 25,841 + adjusted TOC pins match canonical v19.1. +- The 312 raw payload differences consist only of the expected canonical-domain + repairs and domain-array ordering. After those agreed normalizations, zero + payloads differ. diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 3ccdcc66..e0cde744 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -172,6 +172,11 @@ Tagged standard membership is deterministic because release planning resolves staged selectors before promoting them to members. Virtual materialization likewise copies exact member revisions from tagged component snapshots and never follows a component's later `track_latest` candidate movement. +When a virtual component declares `filters.domains`, the same allowed-domain +set bounds relationship-discovered secondary objects during graph capture. +An explicitly domain-bearing secondary object from another domain is not +included merely because it has a relationship to an included primary root. +Domainless supporting metadata remains eligible. Every relationship revision stores server-controlled exact source and target pins under `workspace.relationship_endpoints`. These fields identify the @@ -187,7 +192,10 @@ record: replay can use and self-activate a complete linked pending manifest after a process interruption. A standard release replaces the draft manifest with one built from the resolved release plan, so dynamic staged selectors become exact members. -Materialized virtual snapshots contain exact roots from the outset. +Materialized virtual snapshots contain exact roots from the outset. Releasing +a virtual draft does not change those roots, so bundle preview and commit +reuse its existing manifest. This makes the preview the literal graph that +will be tagged rather than a second resolution against newer database state. Active and pending manifests protect their exact dependencies. In-place updates and hard deletes that would invalidate a primary or secondary @@ -195,6 +203,10 @@ revision return `409`; lineage deletion is rejected when any version is protected. Relationship source, target, and type changes are rejected. Description-only relationship corrections remain allowed because the relationship STIX payload used by older snapshots is frozen in the manifest. +Manifest entries may also freeze complete source payloads for an audited +operational baseline. The exact database revision pin remains mandatory and +protected; the frozen payload preserves the reviewed publication +representation for deterministic replay. Deleting a draft snapshot or track removes its manifest and releases protection that no other snapshot needs. diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index d13486f5..dc18d3de 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -228,6 +228,12 @@ filter and a Mobile filter, while `["mobile-attack"]` is excluded by an Enterprise filter. Objects without `x_mitre_domains` are excluded when a domain filter is set. +The domain constraint also bounds the snapshot's publication graph. A +relationship cannot pull a secondary object with an explicit, nonmatching +`x_mitre_domains` value into the virtual bundle. Domainless identities, +marking definitions, and other supporting metadata may still be included +when referenced by an included object. + `x_mitre_domains` is canonical object data. A cross-domain object has one revision containing the complete domain union; Workbench does not create or emit separate domain-narrowed revisions of that object. Consequently, the @@ -956,6 +962,9 @@ quarantined object counts. Use `format=workbench` or `format=bundle` to inspect the literal snapshot or publication artifact that would be tagged. The draft must have a non-null `composition_resolution`, proving that its members and quarantine tiers were materialized from its current composition. +Bundle preview replays the materialized draft's graph manifest, and release +retains that same manifest because tagging a virtual snapshot does not alter +its contents. ### Retrieve a Materialized Virtual Snapshot From cfc037174aee34fdaf5248aebc64900bb54bfb2b Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:20:09 -0400 Subject: [PATCH 46/55] fix(release-tracks): enable programmatic access to snapshot retrieval enable stixExport and readOnly service role access to /snapshots/latest endpoint --- app/routes/release-tracks-routes.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 8204fca0..e5f05452 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -198,7 +198,10 @@ router .route('/release-tracks/:id/snapshots/latest') .get( authn.authenticate, - authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + authz.requireRole(authz.visitorOrHigher, [ + authz.serviceRoles.readOnly, + authz.serviceRoles.stixExport, + ]), releaseTracksController.retrieveLatestSnapshot, ); From e351ddfb8fa62610ae0333d92ef569b8990ba1bb Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:44:50 -0400 Subject: [PATCH 47/55] feat(release-tracks): make deterministic graphs opt in Add tagged-snapshot graph create and delete endpoints with pointer-only member manifests and live graph fallback. Make persisted STIX revisions immutable, retain one rolling standard draft, and expose graph statistics in snapshot history. --- .../definitions/components/release-tracks.yml | 57 +++- app/api/definitions/components/workspace.yml | 2 +- app/api/definitions/openapi.yml | 3 + app/api/definitions/paths/analytics-paths.yml | 8 +- app/api/definitions/paths/assets-paths.yml | 6 +- app/api/definitions/paths/campaigns-paths.yml | 8 +- .../paths/data-components-paths.yml | 8 +- .../definitions/paths/data-sources-paths.yml | 6 +- .../paths/detection-strategies-paths.yml | 8 +- app/api/definitions/paths/groups-paths.yml | 8 +- .../definitions/paths/identities-paths.yml | 6 +- app/api/definitions/paths/matrices-paths.yml | 8 +- .../definitions/paths/mitigations-paths.yml | 8 +- app/api/definitions/paths/notes-paths.yml | 8 +- .../definitions/paths/relationships-paths.yml | 8 +- .../paths/release-tracks-paths.yml | 68 +++- app/api/definitions/paths/software-paths.yml | 8 +- app/api/definitions/paths/tactics-paths.yml | 8 +- .../definitions/paths/techniques-paths.yml | 8 +- app/controllers/release-tracks-controller.js | 27 ++ app/exceptions/index.js | 17 +- app/lib/error-handler.js | 2 + app/models/relationship-model.js | 2 + .../release-track-graph-manifest-model.js | 9 +- app/models/subschemas/workspace.js | 4 +- app/repository/relationships-repository.js | 42 +++ .../release-track-dynamic.repository.js | 79 ++++- app/routes/release-tracks-routes.js | 13 + app/services/meta-classes/base.service.js | 47 +-- app/services/release-tracks/export-service.js | 28 +- .../release-tracks/graph-manifest-service.js | 310 ++++++++++++------ .../release-tracks/release-tracks-service.js | 15 +- .../release-tracks/snapshot-service.js | 143 ++++++-- .../release-tracks/versioning-service.js | 51 +-- app/services/stix/bundle-graph-resolver.js | 7 + app/services/system/notes-service.js | 45 +-- app/tests/api/analytics/analytics.spec.js | 14 +- app/tests/api/assets/assets.spec.js | 14 +- .../update-identity-guard.spec.js | 23 +- app/tests/api/campaigns/campaigns.spec.js | 14 +- .../data-components/data-components.spec.js | 14 +- .../api/data-sources/data-sources.spec.js | 12 +- .../detection-strategies-spec.js | 14 +- app/tests/api/groups/groups.spec.js | 14 +- app/tests/api/identities/identities.spec.js | 14 +- app/tests/api/matrices/matrices.spec.js | 14 +- app/tests/api/mitigations/mitigations.spec.js | 14 +- app/tests/api/notes/notes.spec.js | 14 +- .../api/relationships/relationships.spec.js | 16 +- .../deterministic-graph-migration.spec.js | 9 + .../release-tracks-backrefs.spec.js | 21 +- .../release-tracks-bundle.spec.js | 35 +- .../release-tracks-change-capture.spec.js | 149 +-------- .../release-tracks-release.spec.js | 50 +-- .../api/release-tracks/release-tracks.spec.js | 20 +- .../release-tracks/releases-by-object.spec.js | 32 +- .../release-tracks/snapshot-history.spec.js | 43 +++ .../snapshot-immutability.spec.js | 37 ++- .../tagged-content-immutability.spec.js | 10 +- .../virtual-graph-integrity.spec.js | 64 ++-- app/tests/api/software/software.spec.js | 14 +- app/tests/api/tactics/tactics.spec.js | 14 +- .../api/techniques/techniques.convert.spec.js | 9 +- .../api/techniques/techniques.revoke.spec.js | 8 +- app/tests/api/techniques/techniques.spec.js | 12 +- .../adm-validation-middleware.spec.js | 122 +++---- docs/developer/FRONTEND_TODO.md | 67 ++-- docs/developer/data-model.md | 7 +- docs/developer/event-bus-architecture.md | 32 +- .../release-tracks/backref-reconciliation.md | 20 +- .../developer/release-tracks/bundle-export.md | 97 +++--- docs/developer/release-tracks/entities.md | 16 +- .../release-tracks/error-handling.md | 15 +- .../release-tracks/implementation-notes.md | 28 +- .../release-tracks/member-sync-strategies.md | 30 +- ...x-versioning-and-embedded-relationships.md | 76 ++--- docs/developer/workspace-validation.md | 8 +- docs/user/release-tracks/api-reference.md | 67 +++- docs/user/release-tracks/object-backrefs.md | 53 +-- docs/user/release-tracks/output-formats.md | 9 +- docs/user/release-tracks/release-workflow.md | 2 +- docs/user/release-tracks/summary.md | 19 +- docs/user/release-tracks/versioning.md | 30 +- docs/user/release-tracks/virtual-tracks.md | 9 +- ...-backfill-deterministic-snapshot-graphs.js | 3 + 85 files changed, 1430 insertions(+), 1083 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 63abdcdb..307f671a 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -27,8 +27,9 @@ components: type: string readOnly: true description: | - Server-controlled identifier for the frozen bundle graph associated - with this snapshot. Clients should treat this value as opaque. + Server-controlled identifier for an opt-in deterministic member + graph on a tagged snapshot. Absent on drafts and graphless tagged + snapshots. Clients should treat this value as opaque. name: type: string pattern: '^[a-zA-Z0-9 &]+$' @@ -104,6 +105,43 @@ components: items: $ref: '#/components/schemas/version-history-entry' + graph-statistics: + type: object + readOnly: true + description: 'Counts of exact revision pointers by role in a materialized snapshot graph' + required: + - primary_count + - secondary_count + - relationship_count + - supporting_count + - link_target_count + - total_count + properties: + primary_count: + type: integer + minimum: 0 + description: 'Primary member objects selected for the snapshot' + secondary_count: + type: integer + minimum: 0 + description: 'Related objects reached while resolving the bounded member graph' + relationship_count: + type: integer + minimum: 0 + description: 'Relationships connecting objects in the resolved graph' + supporting_count: + type: integer + minimum: 0 + description: 'Supporting identities and marking definitions required by cached objects' + link_target_count: + type: integer + minimum: 0 + description: 'Objects pinned to resolve LinkById references deterministically' + total_count: + type: integer + minimum: 0 + description: 'Total entries across all graph manifest roles' + snapshot-summary: type: object description: 'Lightweight metadata shared by standard and virtual snapshot summaries' @@ -130,6 +168,17 @@ components: type: string nullable: true description: 'Tagged version, or null for an untagged draft' + graph_manifest_id: + type: string + readOnly: true + description: | + Opaque identifier for the tagged snapshot's deterministic member + graph. Omitted when the snapshot has not been materialized. + graph_statistics: + $ref: '#/components/schemas/graph-statistics' + description: | + High-level statistics for the materialized graph. Omitted when the + snapshot does not reference a graph manifest. name: type: string description: @@ -243,7 +292,7 @@ components: - work-in-progress - awaiting-review - reviewed - description: 'Workflow status (scoped to this track). modified-in-place is server-assigned when the pinned revision is edited via an in-place PUT and needs re-review.' + description: 'Workflow status (scoped to this track). modified-in-place is retained only for legacy data; persisted STIX revisions are now immutable.' object_added_at: type: string format: date-time @@ -272,7 +321,7 @@ components: - work-in-progress - awaiting-review - reviewed - description: 'Workflow status (preserved from candidates). modified-in-place is server-assigned when the pinned revision is edited via an in-place PUT and needs re-review.' + description: 'Workflow status (preserved from candidates). modified-in-place is retained only for legacy data; persisted STIX revisions are now immutable.' object_staged_at: type: string format: date-time diff --git a/app/api/definitions/components/workspace.yml b/app/api/definitions/components/workspace.yml index cdf8cbb2..13a49349 100644 --- a/app/api/definitions/components/workspace.yml +++ b/app/api/definitions/components/workspace.yml @@ -39,7 +39,7 @@ components: status: type: string enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'] - description: 'Track-scoped workflow status. Members are always reviewed; quarantined entries carry no status.' + description: 'Track-scoped workflow status. Members are always reviewed; quarantined entries carry no status. modified-in-place is retained only for legacy data.' required: - id - tier diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 07a7c20f..e1e67b6e 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -409,6 +409,9 @@ paths: /api/release-tracks/{id}/snapshots/{modified}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1clone' + /api/release-tracks/{id}/snapshots/{modified}/graph: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1graph' + /api/release-tracks/{id}/snapshots/{modified}/release: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1release' diff --git a/app/api/definitions/paths/analytics-paths.yml b/app/api/definitions/paths/analytics-paths.yml index dbab5651..fe376059 100644 --- a/app/api/definitions/paths/analytics-paths.yml +++ b/app/api/definitions/paths/analytics-paths.yml @@ -206,7 +206,7 @@ paths: '404': description: 'A analytic with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/analytics/{stixId}/modified/{modified}: get: @@ -242,7 +242,7 @@ paths: summary: 'Update a analytic' operationId: 'analytic-update' description: | - This endpoint updates a single version of a analytic in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Analytics' parameters: @@ -278,7 +278,7 @@ paths: '404': description: 'A analytic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a analytic' operationId: 'analytic-delete' @@ -306,4 +306,4 @@ paths: '404': description: 'A analytic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' diff --git a/app/api/definitions/paths/assets-paths.yml b/app/api/definitions/paths/assets-paths.yml index 4f76dde8..2cd8af83 100644 --- a/app/api/definitions/paths/assets-paths.yml +++ b/app/api/definitions/paths/assets-paths.yml @@ -234,7 +234,7 @@ paths: summary: 'Update an asset' operationId: 'asset-update' description: | - This endpoint updates a single version of an asset in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Assets' parameters: @@ -270,7 +270,7 @@ paths: '404': description: 'An asset with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete an asset' operationId: 'asset-delete' @@ -298,7 +298,7 @@ paths: '404': description: 'An asset with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/assets/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/campaigns-paths.yml b/app/api/definitions/paths/campaigns-paths.yml index e02dff77..e4d5af4f 100644 --- a/app/api/definitions/paths/campaigns-paths.yml +++ b/app/api/definitions/paths/campaigns-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A campaign with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/campaigns/{stixId}/modified/{modified}: get: @@ -214,7 +214,7 @@ paths: summary: 'Update a campaign' operationId: 'campaign-update' description: | - This endpoint updates a single version of a campaign in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Campaigns' parameters: @@ -250,7 +250,7 @@ paths: '404': description: 'A campaign with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a campaign' operationId: 'campaign-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A campaign with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/campaigns/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/data-components-paths.yml b/app/api/definitions/paths/data-components-paths.yml index b110bd3f..d005e6de 100644 --- a/app/api/definitions/paths/data-components-paths.yml +++ b/app/api/definitions/paths/data-components-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A data component with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/data-components/{stixId}/channels: get: @@ -280,7 +280,7 @@ paths: summary: 'Update a data component' operationId: 'data-component-update' description: | - This endpoint updates a single version of a data component in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Data Components' parameters: @@ -316,7 +316,7 @@ paths: '404': description: 'A data component with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a data component' operationId: 'data-component-delete' @@ -344,7 +344,7 @@ paths: '404': description: 'A data component with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/data-components/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/data-sources-paths.yml b/app/api/definitions/paths/data-sources-paths.yml index bc33482c..71e964e6 100644 --- a/app/api/definitions/paths/data-sources-paths.yml +++ b/app/api/definitions/paths/data-sources-paths.yml @@ -250,7 +250,7 @@ paths: summary: 'Update a data source' operationId: 'data-source-update' description: | - This endpoint updates a single version of a data source in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Data Sources' parameters: @@ -286,7 +286,7 @@ paths: '404': description: 'A data source with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a data source' operationId: 'data-source-delete' @@ -314,7 +314,7 @@ paths: '404': description: 'A data source with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/data-sources/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/detection-strategies-paths.yml b/app/api/definitions/paths/detection-strategies-paths.yml index d83dc126..f7f3f928 100644 --- a/app/api/definitions/paths/detection-strategies-paths.yml +++ b/app/api/definitions/paths/detection-strategies-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A detection strategy with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/detection-strategies/{stixId}/modified/{modified}: get: @@ -226,7 +226,7 @@ paths: summary: 'Update a detection strategy' operationId: 'detection-strategy-update' description: | - This endpoint updates a single version of a detection strategy in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Detection Strategies' parameters: @@ -262,7 +262,7 @@ paths: '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a detection strategy' operationId: 'detection-strategy-delete' @@ -290,4 +290,4 @@ paths: '404': description: 'A detection strategy with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' diff --git a/app/api/definitions/paths/groups-paths.yml b/app/api/definitions/paths/groups-paths.yml index c6e39178..65442242 100644 --- a/app/api/definitions/paths/groups-paths.yml +++ b/app/api/definitions/paths/groups-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A group with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/groups/{stixId}/modified/{modified}: get: @@ -214,7 +214,7 @@ paths: summary: 'Update a group' operationId: 'group-update' description: | - This endpoint updates a single version of a group in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Groups' parameters: @@ -250,7 +250,7 @@ paths: '404': description: 'A group with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a group' operationId: 'group-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A group with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/groups/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/identities-paths.yml b/app/api/definitions/paths/identities-paths.yml index 10e83651..5af970ea 100644 --- a/app/api/definitions/paths/identities-paths.yml +++ b/app/api/definitions/paths/identities-paths.yml @@ -191,7 +191,7 @@ paths: summary: 'Update an identity' operationId: 'identity-update' description: | - This endpoint updates a single version of an identity in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Identities' parameters: @@ -227,7 +227,7 @@ paths: '404': description: 'An identity with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a identity' operationId: 'identity-delete' @@ -255,4 +255,4 @@ paths: '404': description: 'An identity with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' diff --git a/app/api/definitions/paths/matrices-paths.yml b/app/api/definitions/paths/matrices-paths.yml index 45cc2118..0bb64034 100644 --- a/app/api/definitions/paths/matrices-paths.yml +++ b/app/api/definitions/paths/matrices-paths.yml @@ -178,7 +178,7 @@ paths: '404': description: 'A matrix with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}: get: @@ -214,7 +214,7 @@ paths: summary: 'Update a matrix' operationId: 'matrix-update' description: | - This endpoint updates a single version of a matrix in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Matrices' parameters: @@ -250,7 +250,7 @@ paths: '404': description: 'A matrix with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a matrix' operationId: 'matrix-delete' @@ -278,7 +278,7 @@ paths: '404': description: 'A matrix with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/matrices/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/mitigations-paths.yml b/app/api/definitions/paths/mitigations-paths.yml index 2e3bde43..f207c7ae 100644 --- a/app/api/definitions/paths/mitigations-paths.yml +++ b/app/api/definitions/paths/mitigations-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A mitigation with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/mitigations/{stixId}/modified/{modified}: get: @@ -226,7 +226,7 @@ paths: summary: 'Update a mitigation' operationId: 'mitigation-update' description: | - This endpoint updates a single version of a mitigation in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Mitigations' parameters: @@ -262,7 +262,7 @@ paths: '404': description: 'A mitigation with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a mitigation' operationId: 'mitigation-delete' @@ -290,7 +290,7 @@ paths: '404': description: 'A mitigation with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/mitigations/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/notes-paths.yml b/app/api/definitions/paths/notes-paths.yml index eebad19a..33f67d34 100644 --- a/app/api/definitions/paths/notes-paths.yml +++ b/app/api/definitions/paths/notes-paths.yml @@ -176,7 +176,7 @@ paths: '404': description: 'A note with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/notes/{stixId}/modified/{modified}: get: @@ -212,7 +212,7 @@ paths: summary: 'Update a note' operationId: 'note-update-version' description: | - This endpoint updates a single version of a note in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Notes' parameters: @@ -247,7 +247,7 @@ paths: '404': description: 'A note with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a note' operationId: 'note-delete-version' @@ -275,4 +275,4 @@ paths: '404': description: 'A note with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' diff --git a/app/api/definitions/paths/relationships-paths.yml b/app/api/definitions/paths/relationships-paths.yml index 69fd5c03..f0d60e35 100644 --- a/app/api/definitions/paths/relationships-paths.yml +++ b/app/api/definitions/paths/relationships-paths.yml @@ -248,7 +248,7 @@ paths: '404': description: 'A relationship with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/relationships/{stixId}/modified/{modified}: get: @@ -284,7 +284,7 @@ paths: summary: 'Update a relationship' operationId: 'relationship-update' description: | - This endpoint updates a single version of a relationship in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Relationships' parameters: @@ -320,7 +320,7 @@ paths: '404': description: 'A relationship with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a relationship' operationId: 'relationship-delete' @@ -348,4 +348,4 @@ paths: '404': description: 'A relationship with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 95b50293..da2740b9 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -946,7 +946,10 @@ paths: ordered by modified timestamp from newest to oldest. Standard snapshot summaries contain members, staged, and candidates counts. Virtual snapshot summaries contain members and quarantine counts, plus - scheduled_materialization when present. + scheduled_materialization when present. Tagged summaries also expose + graph_manifest_id when their deterministic member graph has been + materialized, together with graph_statistics counts for primary, + secondary, relationship, supporting, and LinkById entries. tags: - 'Release Tracks' parameters: @@ -1256,6 +1259,69 @@ paths: schema: $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' + /api/release-tracks/{id}/snapshots/{modified}/graph: + post: + summary: 'Make a tagged snapshot member graph deterministic' + operationId: 'release-tracks-snapshot-graph-create' + description: | + Resolve the tagged snapshot's members into a deterministic graph and + persist exact-revision pointers for its primary objects, + relationships, and secondary objects. The referenced revisions are + write-protected until the graph is deleted. Draft snapshots cannot + have persisted graphs. Repeating this operation for a snapshot that + already has a graph is idempotent and returns the existing snapshot. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: modified + in: path + required: true + schema: + type: string + responses: + '200': + description: 'The tagged snapshot already had a deterministic graph' + '201': + description: 'Deterministic member graph created successfully' + '404': + description: 'Snapshot not found' + '409': + description: 'The snapshot is untagged, changed concurrently, or references missing revisions' + + delete: + summary: 'Remove a tagged snapshot deterministic graph' + operationId: 'release-tracks-snapshot-graph-delete' + description: | + Remove the opt-in deterministic member graph and release its + exact-revision deletion protections. Subsequent exports resolve the live + graph. This operation is idempotent when no graph exists. Draft + snapshots cannot have persisted graphs. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: modified + in: path + required: true + schema: + type: string + responses: + '204': + description: 'Deterministic member graph absent after the request' + '404': + description: 'Snapshot not found' + '409': + description: 'The snapshot is untagged or its graph changed concurrently' + /api/release-tracks/{id}/snapshots/{modified}/release: post: summary: 'Release a specific snapshot' diff --git a/app/api/definitions/paths/software-paths.yml b/app/api/definitions/paths/software-paths.yml index 2d90bc18..43fab37c 100644 --- a/app/api/definitions/paths/software-paths.yml +++ b/app/api/definitions/paths/software-paths.yml @@ -201,7 +201,7 @@ paths: '404': description: 'A tactic with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/software/{stixId}/modified/{modified}: get: @@ -237,7 +237,7 @@ paths: summary: 'Update a software object' operationId: 'software-update' description: | - This endpoint updates a single version of a software object in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Software' parameters: @@ -272,7 +272,7 @@ paths: '404': description: 'A software object with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a software object' operationId: 'software-delete' @@ -300,7 +300,7 @@ paths: '404': description: 'A software object with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/software/{stixId}/revoke: post: diff --git a/app/api/definitions/paths/tactics-paths.yml b/app/api/definitions/paths/tactics-paths.yml index e007ca7d..dea66611 100644 --- a/app/api/definitions/paths/tactics-paths.yml +++ b/app/api/definitions/paths/tactics-paths.yml @@ -190,7 +190,7 @@ paths: '404': description: 'A tactic with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}: get: @@ -226,7 +226,7 @@ paths: summary: 'Update a tactic' operationId: 'tactic-update' description: | - This endpoint updates a single version of a tactic in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Tactics' parameters: @@ -262,7 +262,7 @@ paths: '404': description: 'A tactic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a tactic' operationId: 'tactic-delete' @@ -290,7 +290,7 @@ paths: '404': description: 'A tactic with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/tactics/{stixId}/modified/{modified}/techniques: get: diff --git a/app/api/definitions/paths/techniques-paths.yml b/app/api/definitions/paths/techniques-paths.yml index 773af422..2a4c6ba1 100644 --- a/app/api/definitions/paths/techniques-paths.yml +++ b/app/api/definitions/paths/techniques-paths.yml @@ -214,7 +214,7 @@ paths: '404': description: 'A technique with the requested STIX id was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}: get: @@ -250,7 +250,7 @@ paths: summary: 'Update a technique' operationId: 'technique-update' description: | - This endpoint updates a single version of a technique in the workspace, identified by its STIX id and modified date. + This endpoint updates non-exported workspace metadata for one persisted STIX revision, identified by its STIX id and modified date. STIX content is immutable; create corrections as new POST revisions. tags: - 'Techniques' parameters: @@ -286,7 +286,7 @@ paths: '404': description: 'A technique with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' delete: summary: 'Delete a technique' operationId: 'technique-delete' @@ -314,7 +314,7 @@ paths: '404': description: 'A technique with the requested STIX id and modified date was not found.' '409': - description: 'The version is pinned by release-track membership or a snapshot graph manifest and cannot be modified or deleted in place. Create a new revision instead.' + description: 'STIX-changing PUTs conflict with revision immutability. Deletes conflict when the revision is pinned by release-track membership or an opt-in snapshot graph. Create a new revision instead.' /api/techniques/{stixId}/modified/{modified}/tactics: get: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 0431ea36..c90c8d74 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -614,6 +614,33 @@ exports.cloneByModified = async function cloneByModified(req, res, next) { } }; +/** POST /api/release-tracks/:id/snapshots/:modified/graph */ +exports.createSnapshotGraph = async function createSnapshotGraph(req, res, next) { + try { + const result = await releaseTracksService.createSnapshotGraph( + req.params.id, + req.params.modified, + ); + logger.debug(`Success: Created graph for snapshot ${req.params.modified}`); + return res.status(result.created ? 201 : 200).send(result.snapshot); + } catch (err) { + logger.error('Failed to create snapshot graph: ' + err); + return next(err); + } +}; + +/** DELETE /api/release-tracks/:id/snapshots/:modified/graph */ +exports.deleteSnapshotGraph = async function deleteSnapshotGraph(req, res, next) { + try { + await releaseTracksService.deleteSnapshotGraph(req.params.id, req.params.modified); + logger.debug(`Success: Deleted graph for snapshot ${req.params.modified}`); + return res.status(204).end(); + } catch (err) { + logger.error('Failed to delete snapshot graph: ' + err); + return next(err); + } +}; + /** DELETE /api/release-tracks/:id/snapshots/:modified */ exports.deleteSnapshotByModified = async function deleteSnapshotByModified(req, res, next) { try { diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 88747877..0137f2ef 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -367,7 +367,7 @@ class MemberPinnedRevisionError extends CustomError { constructor(options) { super( 'This revision is pinned in the members tier of a release track and is released content: ' + - 'it cannot be modified or deleted in place. Create a new revision instead ' + + 'it cannot be deleted. Create a new revision instead ' + '(set x_mitre_deprecated on a new revision to retire the object).', options, ); @@ -377,8 +377,18 @@ class MemberPinnedRevisionError extends CustomError { class SnapshotGraphPinnedRevisionError extends CustomError { constructor(options) { super( - 'This revision is frozen in a release-track snapshot graph and cannot be modified or ' + - 'deleted in place. Create a new revision instead.', + 'This revision is referenced by a release-track snapshot graph and cannot be deleted. ' + + 'Create a new revision instead.', + options, + ); + } +} + +class ImmutableStixRevisionError extends CustomError { + constructor(options) { + super( + 'Persisted STIX revisions are immutable and cannot be modified in place. ' + + 'Create a new revision with POST instead.', options, ); } @@ -473,6 +483,7 @@ module.exports = { TrackNotFoundError, MemberPinnedRevisionError, SnapshotGraphPinnedRevisionError, + ImmutableStixRevisionError, //** Database-related errors */ DuplicateIdError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 7ad83715..4e486e31 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -51,6 +51,7 @@ const { TrackNotFoundError, MemberPinnedRevisionError, SnapshotGraphPinnedRevisionError, + ImmutableStixRevisionError, ObjectHasValidationIssuesError, } = require('../exceptions'); @@ -148,6 +149,7 @@ exports.serviceExceptions = function (err, req, res, next) { err instanceof VirtualSnapshotNotMaterializedError || err instanceof MemberPinnedRevisionError || err instanceof SnapshotGraphPinnedRevisionError || + err instanceof ImmutableStixRevisionError || err instanceof ObjectHasValidationIssuesError || err instanceof ActiveOrganizationIdentityDeleteError ) { diff --git a/app/models/relationship-model.js b/app/models/relationship-model.js index 2a794940..60232c73 100644 --- a/app/models/relationship-model.js +++ b/app/models/relationship-model.js @@ -55,6 +55,8 @@ relationshipSchema.index({ 'stix.id': 1, 'stix.modified': -1 }, { unique: true } // Multikey index supporting reverse lookups from release tracks // (release-track backref reconciliation queries by workspace.release_tracks.id) relationshipSchema.index({ 'workspace.release_tracks.id': 1 }, { sparse: true }); +relationshipSchema.index({ 'stix.source_ref': 1 }); +relationshipSchema.index({ 'stix.target_ref': 1 }); relationshipSchema.index({ 'workspace.relationship_endpoints.source.object_ref': 1, 'workspace.relationship_endpoints.source.object_modified': 1, diff --git a/app/models/release-tracks/release-track-graph-manifest-model.js b/app/models/release-tracks/release-track-graph-manifest-model.js index 7e2015ce..c1d37c77 100644 --- a/app/models/release-tracks/release-track-graph-manifest-model.js +++ b/app/models/release-tracks/release-track-graph-manifest-model.js @@ -55,11 +55,10 @@ const entrySchema = new mongoose.Schema( source: { type: exactRevisionSchema }, target: { type: exactRevisionSchema }, discovered_from: { type: [exactRevisionSchema], default: undefined }, - // Relationship payloads are frozen so description-only corrections do - // not change older bundles. Marking definitions are not STIX-versioned, - // so their complete payload is frozen for the same replay guarantee. - // Operational baselines may also freeze an exact source-bundle payload - // while retaining the database revision pin as the integrity boundary. + // Schema-v2 relationships are exact-revision pointers. Marking + // definitions are not STIX-versioned, so their complete payload is frozen + // for the same replay guarantee. Schema-v1 relationships retain frozen + // payloads for backwards-compatible replay. frozen_stix: { type: mongoose.Schema.Types.Mixed }, }, { collection: 'releaseTrackGraphManifestEntries' }, diff --git a/app/models/subschemas/workspace.js b/app/models/subschemas/workspace.js index 833bab58..5eb7726e 100644 --- a/app/models/subschemas/workspace.js +++ b/app/models/subschemas/workspace.js @@ -48,8 +48,8 @@ const releaseTrackRef = { }, // Track-scoped workflow status. Members are inherently 'reviewed'; // quarantined entries (virtual tracks) carry no status; - // 'modified-in-place' marks entries whose pinned revision was changed by - // an in-place PUT and needs re-review. + // 'modified-in-place' is retained for legacy persisted entries. Generic + // STIX-changing PUTs are no longer permitted and do not create new markers. status: { type: String, enum: ['modified-in-place', 'work-in-progress', 'awaiting-review', 'reviewed'], diff --git a/app/repository/relationships-repository.js b/app/repository/relationships-repository.js index a4af9094..1c5dc760 100644 --- a/app/repository/relationships-repository.js +++ b/app/repository/relationships-repository.js @@ -113,6 +113,48 @@ class RelationshipsRepository extends BaseRepository { } } + /** + * Retrieve the current revision of relationship lineages that still touch + * any object in a bounded graph frontier. The first indexed lookup finds + * candidate lineages; the second aggregation deliberately chooses each + * lineage's globally latest revision before reapplying the endpoint filter. + * This avoids treating an older, once-relevant revision as current. + */ + async retrieveLatestTouchingObjectRefs(objectRefs, options = {}) { + if (!Array.isArray(objectRefs) || objectRefs.length === 0) return []; + + try { + const endpointQuery = { + $or: [ + { 'stix.source_ref': { $in: objectRefs } }, + { 'stix.target_ref': { $in: objectRefs } }, + ], + }; + const candidateIds = await this.model.distinct('stix.id', endpointQuery).exec(); + if (candidateIds.length === 0) return []; + + const currentQuery = { ...endpointQuery }; + if (!options.includeRevoked) { + currentQuery['stix.revoked'] = { $in: [null, false] }; + } + if (!options.includeDeprecated) { + currentQuery['stix.x_mitre_deprecated'] = { $in: [null, false] }; + } + + return await this.model + .aggregate([ + { $match: { 'stix.id': { $in: candidateIds } } }, + { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, + { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, + { $replaceRoot: { newRoot: '$document' } }, + { $match: currentQuery }, + ]) + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async retrieveAllWithAttackURLInDescription() { const aggregation = [ { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index c3cd981d..45d5f59e 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -32,6 +32,21 @@ class ReleaseTrackDynamicRepository { } } + async getLatestSnapshotBefore(trackId, modified) { + try { + const Model = this._getModel(trackId); + return await Model.findOne({ id: trackId, modified: { $lt: modified } }) + .sort({ modified: -1 }) + .lean() + .exec(); + } catch (err) { + if (err.name === 'CastError') { + throw new BadlyFormattedParameterError({ parameterName: 'modified' }); + } + throw new DatabaseError(err); + } + } + async getLatestSnapshotTierSummary(trackId) { try { const Model = this._getModel(trackId); @@ -273,6 +288,7 @@ class ReleaseTrackDynamicRepository { type: 1, modified: 1, version: 1, + graph_manifest_id: 1, name: 1, description: 1, scheduled_materialization: 1, @@ -329,16 +345,21 @@ class ReleaseTrackDynamicRepository { Object.assign(setOps, versionData.additionalOps); } + const update = { + $set: setOps, + $push: { version_history: versionData.versionHistoryEntry }, + }; + if (versionData.unsetOps) { + update.$unset = versionData.unsetOps; + } + const result = await Model.findOneAndUpdate( { id: trackId, modified: modified, version: null, // Guard: only tag untagged snapshots }, - { - $set: setOps, - $push: { version_history: versionData.versionHistoryEntry }, - }, + update, { new: true, runValidators: true, @@ -374,6 +395,56 @@ class ReleaseTrackDynamicRepository { } } + async attachGraphManifest(trackId, modified, manifestId) { + try { + const Model = this._getModel(trackId); + return await Model.findOneAndUpdate( + { + id: trackId, + modified, + version: { $type: 'string' }, + graph_manifest_id: { $exists: false }, + }, + { $set: { graph_manifest_id: manifestId } }, + { new: true, runValidators: true, lean: true }, + ).exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async detachGraphManifest(trackId, modified, manifestId) { + try { + const Model = this._getModel(trackId); + return await Model.findOneAndUpdate( + { + id: trackId, + modified, + version: { $type: 'string' }, + graph_manifest_id: manifestId, + }, + { $unset: { graph_manifest_id: '' } }, + { new: true, runValidators: true, lean: true }, + ).exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async deleteOlderDrafts(trackId, modified) { + try { + const Model = this._getModel(trackId); + const query = { id: trackId, version: null, modified: { $lt: modified } }; + const snapshots = await Model.find(query).select('modified graph_manifest_id').lean().exec(); + if (snapshots.length > 0) { + await Model.deleteMany({ _id: { $in: snapshots.map((snapshot) => snapshot._id) } }).exec(); + } + return snapshots; + } catch (err) { + throw new DatabaseError(err); + } + } + async deleteSnapshot(trackId, modified) { try { const Model = this._getModel(trackId); diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index e5f05452..594985b7 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -265,6 +265,19 @@ router releaseTracksController.cloneByModified, ); +router + .route('/release-tracks/:id/snapshots/:modified/graph') + .post( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.createSnapshotGraph, + ) + .delete( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.deleteSnapshotGraph, + ); + router .route('/release-tracks/:id/snapshots/:modified') .get( diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 4bf2512d..225ac0ce 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -1,6 +1,7 @@ 'use strict'; const uuid = require('uuid'); +const _ = require('lodash'); const logger = require('../../lib/logger'); const config = require('../../config/config'); const attackIdGenerator = require('../../lib/attack-id-generator'); @@ -24,6 +25,7 @@ const { SelfRevocationError, MemberPinnedRevisionError, SnapshotGraphPinnedRevisionError, + ImmutableStixRevisionError, } = require('../../exceptions'); const { getSchema } = require('../../lib/validation-schemas'); const { deepFreezeStix } = require('../../lib/import-safety'); @@ -773,9 +775,9 @@ class BaseService extends ServiceWithHooks { /** * Protect every exact revision captured by an active or in-progress graph - * manifest. Relationship payloads are frozen in the manifest, so a - * non-topology PUT remains safe; relationship endpoint changes are rejected - * separately by RelationshipsService. Hard deletion is always rejected. + * manifest. Pointer-only manifests hydrate relationships by exact revision + * just like primary and secondary objects, so no versioned STIX payload is + * exempt from this guard. */ static async assertNotGraphPinned(document, operation) { const graphManifestService = require('../release-tracks/graph-manifest-service'); @@ -783,17 +785,12 @@ class BaseService extends ServiceWithHooks { document.stix.id, document.stix.modified, ); - if ( - pins.length === 0 || - (operation === 'updated' && pins.every((pin) => pin.kind === 'relationship')) - ) { - return; - } + if (pins.length === 0) return; throw new SnapshotGraphPinnedRevisionError({ details: `Revision ${document.stix.id} (modified ` + - `${new Date(document.stix.modified).toISOString()}) is frozen in ` + + `${new Date(document.stix.modified).toISOString()}) is referenced by ` + `${pins.length} release-track snapshot graph manifest(s) and cannot be ${operation} ` + 'in place.', snapshot_graph_pins: pins, @@ -975,15 +972,15 @@ class BaseService extends ServiceWithHooks { } /** - * Updates an existing STIX object version in-place. + * Updates non-exported workspace metadata on an existing STIX revision. * * Pipeline stages: * 1. ANALYZE REQUEST — retrieve existing document by stixId + modified * 2. COMPOSE OBJECT — strip server-controlled fields, compose from existing document * 3. SET SERVER-CONTROLLED FIELDS — (future: bump modified timestamp) * 4. LIFECYCLE HOOKS — subclass data transformations (beforeUpdate) - * 5. VALIDATE WITH ADM — full schema validation on the composed object - * 6. PERSIST — merge and save document, run afterUpdate hook, emit event (skip if dryRun) + * 5. IMMUTABILITY + ADM VALIDATION — reject STIX changes, validate the composed object + * 6. PERSIST — merge and save document, run afterUpdate hook (skip if dryRun) * * @param {string} stixId - The STIX ID of the object to update * @param {string} stixModified - The modified timestamp identifying the specific version @@ -1030,13 +1027,6 @@ class BaseService extends ServiceWithHooks { return null; } - // Members-pinned revisions are released content — immutable in place. - await BaseService.assertNotMemberPinned(document, 'updated'); - await BaseService.assertNotGraphPinned(document, 'updated'); - - // TODO: diff analysis — detect field-level changes vs document - // TODO: if no changes detected, short-circuit (no-op) - // ────────────────────────────────────────────── // 2. COMPOSE OBJECT // ────────────────────────────────────────────── @@ -1095,6 +1085,19 @@ class BaseService extends ServiceWithHooks { // ────────────────────────────────────────────── await this.beforeUpdate(stixId, stixModified, data, document, options); + // A STIX revision is identified by (stix.id, stix.modified). Mutating its + // exportable payload in place makes every persisted reference to that + // revision ambiguous. PUT therefore remains available only for workspace + // metadata; STIX corrections must be posted as a new revision. + const persistedStix = JSON.parse(JSON.stringify(document.stix)); + const proposedStix = JSON.parse(JSON.stringify(data.stix)); + if (!_.isEqual(persistedStix, proposedStix)) { + throw new ImmutableStixRevisionError({ + stix_id: document.stix.id, + stix_modified: new Date(document.stix.modified).toISOString(), + }); + } + // ────────────────────────────────────────────── // 5. VALIDATE WITH ADM // ────────────────────────────────────────────── @@ -1124,7 +1127,9 @@ class BaseService extends ServiceWithHooks { } await this.afterUpdate(newDocument, document); - await this.emitUpdatedEvent(newDocument, document); + // PUT can now change workspace metadata only. STIX-domain update events + // drive relationship advancement and release-track revision sync, so + // emitting one here would misclassify metadata edits as new content. const result = newDocument.toObject ? newDocument.toObject() : newDocument; result.warnings = warnings; await this._refreshReleaseTrackBackrefs(result); diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index 05102fcf..3e15ea80 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -51,7 +51,8 @@ exports.hydrateMembers = async function hydrateMembers(entries) { /** * Convert LinkById tags (e.g. "(LinkById: T1234)") in descriptions to - * markdown citations using only object revisions frozen in the manifest. + * markdown citations using only object revisions supplied by the resolved + * live or persisted graph. * * @param {Array} documents - Hydrated lean documents ({ stix, ... }) */ @@ -69,11 +70,12 @@ async function convertLinkByIdTags(documents, linkTargetDocuments) { } } -function selectedDraftTierIsDynamic(snapshot, options) { - return (options.include || []).some( - (tier) => - ['staged', 'candidates'].includes(tier) && - (snapshot[tier] || []).some((entry) => entry.object_modified === 'latest'), +function requiresLiveGraph(snapshot, options) { + return ( + options.captureGraph || + snapshot.version == null || + !snapshot.graph_manifest_id || + (options.include || []).some((tier) => ['staged', 'candidates'].includes(tier)) ); } @@ -128,11 +130,11 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd * * Bundle exports (see docs/developer/release-tracks/bundle-export.md): * - The same pipeline applies to standard snapshots and materialized virtual - * snapshots because both persist exact member revisions and graph manifests. + * snapshots because both persist exact member revisions. * 1. Select tier entries — members always; staged/candidates via * options.include, narrowed by options.state * 2. Hydrate entries into full documents - * 3. Append current relationships whose endpoints are both selected + * 3. Resolve live relationships or replay exact persisted graph pointers * 4. Append referenced identities and marking definitions * 5. Convert LinkById tags to markdown citations * 6. Assemble the bundle (STIX version conformance + optional TOC) via the @@ -149,10 +151,12 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd */ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options = {}) { if (format === 'bundle') { - const graph = - options.captureGraph || selectedDraftTierIsDynamic(snapshot, options) - ? await graphManifestService.replayPlannedSnapshot(snapshot, options) - : await graphManifestService.replay(snapshot, options); + // A persisted graph is an opt-in guarantee for members only. Graphless + // snapshots and exports that add mutable draft tiers resolve the current + // relationship frontier instead of implying determinism they do not have. + const graph = requiresLiveGraph(snapshot, options) + ? await graphManifestService.replayPlannedSnapshot(snapshot, options) + : await graphManifestService.replay(snapshot, options); const allObjects = graph.documents; await convertLinkByIdTags(allObjects, graph.linkTargetDocuments); diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js index 872078fa..407054aa 100644 --- a/app/services/release-tracks/graph-manifest-service.js +++ b/app/services/release-tracks/graph-manifest-service.js @@ -14,8 +14,16 @@ const { const { ReleaseContentIntegrityError } = require('../../exceptions'); const primaryRevisionService = require('./primary-revision-service'); -const RESOLVER_VERSION = 'bounded-attack-graph-v1'; +const MANIFEST_SCHEMA_VERSION = 2; +const RESOLVER_VERSION = 'bounded-member-graph-v2'; const TIERS = ['members', 'staged', 'candidates', 'quarantine']; +const STATISTIC_FIELDS_BY_KIND = { + root: 'primary_count', + secondary: 'secondary_count', + relationship: 'relationship_count', + supporting: 'supporting_count', + link_target: 'link_target_count', +}; const MUTATION_PROTECTED_ENTRY_FILTER = { $or: [ { kind: { $ne: 'root' } }, @@ -76,10 +84,103 @@ function endpointFor(relationship, side) { }; } -async function buildManifestEntries(snapshot) { +async function resolveBoundedGraph(hydratedRoots, allowedDomains, missing) { + const rootObjectRefs = new Set(hydratedRoots.entries.map((entry) => entry.object_ref)); + let frontierObjectRefs = new Set(rootObjectRefs); + + while (true) { + const relationships = await relationshipsRepository.retrieveLatestTouchingObjectRefs( + [...frontierObjectRefs], + { includeRevoked: false, includeDeprecated: false }, + ); + const pinnedRelationships = []; + for (const relationship of relationships) { + const source = endpointFor(relationship, 'source'); + const target = endpointFor(relationship, 'target'); + if (!source || !target) { + // Legacy relationships outside this snapshot's bounded graph cannot + // affect its replay. Fail closed only when an unpinned relationship + // touches a primary member by STIX ID. + if ( + rootObjectRefs.has(relationship.stix.source_ref) || + rootObjectRefs.has(relationship.stix.target_ref) + ) { + missing.push({ + object_ref: relationship.stix.id, + object_modified: new Date(relationship.stix.modified).toISOString(), + dependency: 'relationship_endpoints', + }); + } + continue; + } + pinnedRelationships.push({ relationship, source, target }); + } + if (missing.length > 0) { + throw new ReleaseContentIntegrityError(missing, { + details: 'Snapshot graph capture found relationships without exact endpoint pins.', + }); + } + + // One batched exact-revision hydration per STIX type replaces the + // resolver's historical one-query-per-secondary behavior. + const hydratedEndpoints = await primaryRevisionService.hydrateEntries( + pinnedRelationships.flatMap(({ source, target }) => [source, target]), + ); + const graphResolver = new BundleGraphResolver({ + attackObjectsRepository, + detectionStrategiesRepository, + repositoryMap: primaryRevisionService.getRepositoryMap(), + policy: { + isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, + relationshipIsActive: bundleRelationships.relationshipIsActive, + secondaryObjectIsValid: (document) => secondaryObjectIsValid(document, allowedDomains), + }, + options: { + inferDomains: false, + includeRevoked: true, + includeDeprecated: true, + includeMissingAttackId: true, + }, + relationships: pinnedRelationships.map((candidate) => candidate.relationship), + prefetchedDocuments: hydratedEndpoints.documents, + onMissingDependency(reference) { + missing.push({ + ...reference, + object_modified: new Date(reference.object_modified).toISOString(), + }); + }, + }); + const resolvedGraph = await graphResolver.resolve(hydratedRoots.documents); + if (missing.length > 0) { + const uniqueMissing = [ + ...new Map( + missing.map((reference) => [ + `${reference.object_ref}::${reference.object_modified}`, + reference, + ]), + ).values(), + ]; + throw new ReleaseContentIntegrityError(uniqueMissing, { + details: 'Snapshot graph capture could not hydrate every exact dependency.', + }); + } + + const resolvedObjectRefs = new Set(resolvedGraph.documents.map((document) => document.stix.id)); + const expanded = [...resolvedObjectRefs].some( + (objectRef) => !frontierObjectRefs.has(objectRef), + ); + if (!expanded) { + return { graphResolver, resolvedGraph }; + } + frontierObjectRefs = new Set([...frontierObjectRefs, ...resolvedObjectRefs]); + } +} + +async function buildManifestEntries(snapshot, options = {}) { const allowedDomains = virtualSnapshotDomains(snapshot); const rootRequests = []; - for (const tier of TIERS) { + const rootTiers = options.memberOnly ? ['members'] : TIERS; + for (const tier of rootTiers) { for (const entry of snapshot[tier] || []) { rootRequests.push({ ...entry, tier }); } @@ -93,78 +194,12 @@ async function buildManifestEntries(snapshot) { ]), ); - const rootObjectRefs = new Set(hydratedRoots.entries.map((entry) => entry.object_ref)); - - const relationships = await relationshipsRepository.retrieveAllForBundle({ - includeRevoked: false, - includeDeprecated: false, - }); const missing = []; - const pinnedRelationships = []; - for (const relationship of relationships) { - const source = endpointFor(relationship, 'source'); - const target = endpointFor(relationship, 'target'); - if (!source || !target) { - // Legacy relationships outside this snapshot's bounded graph cannot - // affect its replay. Fail closed only when an unpinned relationship - // touches a primary member by STIX ID. - if ( - rootObjectRefs.has(relationship.stix.source_ref) || - rootObjectRefs.has(relationship.stix.target_ref) - ) { - missing.push({ - object_ref: relationship.stix.id, - object_modified: new Date(relationship.stix.modified).toISOString(), - dependency: 'relationship_endpoints', - }); - } - continue; - } - pinnedRelationships.push({ relationship, source, target }); - } - if (missing.length > 0) { - throw new ReleaseContentIntegrityError(missing, { - details: 'Snapshot graph capture found relationships without exact endpoint pins.', - }); - } - - const graphResolver = new BundleGraphResolver({ - attackObjectsRepository, - detectionStrategiesRepository, - repositoryMap: primaryRevisionService.getRepositoryMap(), - policy: { - isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, - relationshipIsActive: bundleRelationships.relationshipIsActive, - secondaryObjectIsValid: (document) => secondaryObjectIsValid(document, allowedDomains), - }, - options: { - inferDomains: false, - includeRevoked: true, - includeDeprecated: true, - includeMissingAttackId: true, - }, - relationships: pinnedRelationships.map((candidate) => candidate.relationship), - onMissingDependency(reference) { - missing.push({ - ...reference, - object_modified: new Date(reference.object_modified).toISOString(), - }); - }, - }); - const resolvedGraph = await graphResolver.resolve(hydratedRoots.documents); - if (missing.length > 0) { - const uniqueMissing = [ - ...new Map( - missing.map((reference) => [ - `${reference.object_ref}::${reference.object_modified}`, - reference, - ]), - ).values(), - ]; - throw new ReleaseContentIntegrityError(uniqueMissing, { - details: 'Snapshot graph capture could not hydrate every exact dependency.', - }); - } + const { graphResolver, resolvedGraph } = await resolveBoundedGraph( + hydratedRoots, + allowedDomains, + missing, + ); const selectedDocuments = new Map( resolvedGraph.documents.map((document) => [ revisionKey(document.stix.id, document.stix.modified), @@ -202,7 +237,7 @@ async function buildManifestEntries(snapshot) { object_status: root?.object_status, object_ref: document.stix.id, object_modified: document.stix.modified, - discovered_from: discoverySources.get(key) || [], + discovered_from: options.memberOnly ? undefined : discoverySources.get(key) || [], }); } for (const candidate of selectedRelationships) { @@ -216,7 +251,10 @@ async function buildManifestEntries(snapshot) { object_modified: candidate.relationship.stix.modified, source: candidate.source, target: candidate.target, - frozen_stix: candidate.relationship.stix, + // Live previews reuse the legacy replay selector, which carries the + // request-local relationship payload without persisting it. Persisted + // schema-v2 member manifests deliberately omit this field. + frozen_stix: options.memberOnly ? undefined : candidate.relationship.stix, }); } for (const document of supportingDocuments) { @@ -245,7 +283,10 @@ async function buildManifestEntries(snapshot) { async function prepare(snapshot, options = {}) { const manifestId = `release-track-graph-manifest--${uuidv4()}`; - const entries = await buildManifestEntries(snapshot); + const schemaVersion = options.schemaVersion ?? MANIFEST_SCHEMA_VERSION; + const memberOnly = schemaVersion >= MANIFEST_SCHEMA_VERSION; + const resolverVersion = memberOnly ? RESOLVER_VERSION : 'bounded-attack-graph-v1'; + const entries = await buildManifestEntries(snapshot, { memberOnly }); const common = { manifest_id: manifestId, track_id: snapshot.id, @@ -255,7 +296,8 @@ async function prepare(snapshot, options = {}) { await ReleaseTrackGraphManifest.create({ ...common, state: 'pending', - resolver_version: RESOLVER_VERSION, + schema_version: schemaVersion, + resolver_version: resolverVersion, baseline_reconstruction: options.baselineReconstruction === true, }); try { @@ -264,6 +306,19 @@ async function prepare(snapshot, options = {}) { entries.map((entry) => ({ ...common, ...entry })), ); } + // The pending manifest now protects every inserted pointer from deletion. + // Rehydrate once inside that protection window so a revision deleted + // during graph discovery cannot leave an attachable dangling manifest. + await replayEntries( + entries, + { + ...common, + state: 'pending', + schema_version: schemaVersion, + resolver_version: resolverVersion, + }, + {}, + ); } catch (err) { await discard(manifestId); throw err; @@ -323,6 +378,51 @@ async function discardTrack(trackId) { ]); } +function emptyStatistics() { + return { + primary_count: 0, + secondary_count: 0, + relationship_count: 0, + supporting_count: 0, + link_target_count: 0, + total_count: 0, + }; +} + +/** + * Count manifest entries by semantic role for a page of snapshot summaries. + * One aggregate covers every requested manifest to avoid a per-snapshot query. + * + * @param {string[]} manifestIds + * @returns {Promise>} + */ +async function getStatisticsByManifestIds(manifestIds) { + const uniqueManifestIds = [...new Set(manifestIds.filter(Boolean))]; + const statisticsByManifestId = new Map( + uniqueManifestIds.map((manifestId) => [manifestId, emptyStatistics()]), + ); + if (uniqueManifestIds.length === 0) return statisticsByManifestId; + + const counts = await ReleaseTrackGraphManifestEntry.aggregate([ + { $match: { manifest_id: { $in: uniqueManifestIds } } }, + { + $group: { + _id: { manifest_id: '$manifest_id', kind: '$kind' }, + count: { $sum: 1 }, + }, + }, + ]).exec(); + + for (const result of counts) { + const statistics = statisticsByManifestId.get(result._id.manifest_id); + const field = STATISTIC_FIELDS_BY_KIND[result._id.kind]; + if (!statistics || !field) continue; + statistics[field] = result.count; + statistics.total_count += result.count; + } + return statisticsByManifestId; +} + function rootIsSelected(entry, options) { if (entry.tier === 'members') return true; if (!['staged', 'candidates'].includes(entry.tier)) return false; @@ -332,8 +432,11 @@ function rootIsSelected(entry, options) { } async function replayEntries(entries, manifest, options) { + const pointerOnlyMemberGraph = manifest.schema_version >= MANIFEST_SCHEMA_VERSION; const versionedEntries = entries.filter( - (entry) => entry.object_modified && entry.kind !== 'relationship', + (entry) => + entry.object_modified && + (entry.kind !== 'relationship' || (pointerOnlyMemberGraph && !entry.frozen_stix)), ); const hydrated = await primaryRevisionService.assertStoredEntries( versionedEntries.map((entry) => ({ @@ -357,7 +460,11 @@ async function replayEntries(entries, manifest, options) { const selectedRevisionKeys = new Set( entries - .filter((entry) => entry.kind === 'root' && rootIsSelected(entry, options)) + .filter((entry) => + pointerOnlyMemberGraph + ? ['root', 'secondary'].includes(entry.kind) + : entry.kind === 'root' && rootIsSelected(entry, options), + ) .map((entry) => entry.revision_key), ); @@ -365,26 +472,28 @@ async function replayEntries(entries, manifest, options) { // detection strategy discovered through an analytic that was itself a // relationship secondary). Replay only follows edges frozen in the // manifest; it never asks the live database to expand the graph. - let added; - do { - added = false; - for (const entry of entries) { - if ( - !['root', 'secondary'].includes(entry.kind) || - selectedRevisionKeys.has(entry.revision_key) - ) { - continue; - } - if ( - (entry.discovered_from || []).some((source) => - selectedRevisionKeys.has(revisionKey(source.object_ref, source.object_modified)), - ) - ) { - selectedRevisionKeys.add(entry.revision_key); - added = true; + if (!pointerOnlyMemberGraph) { + let added; + do { + added = false; + for (const entry of entries) { + if ( + !['root', 'secondary'].includes(entry.kind) || + selectedRevisionKeys.has(entry.revision_key) + ) { + continue; + } + if ( + (entry.discovered_from || []).some((source) => + selectedRevisionKeys.has(revisionKey(source.object_ref, source.object_modified)), + ) + ) { + selectedRevisionKeys.add(entry.revision_key); + added = true; + } } - } - } while (added); + } while (added); + } const selectedRelationships = entries.filter( (entry) => @@ -493,6 +602,7 @@ async function replayPlannedSnapshot(snapshot, options = {}) { track_id: snapshot.id, snapshot_modified: snapshot.modified, state: 'preview', + schema_version: 1, resolver_version: RESOLVER_VERSION, }, options, @@ -572,8 +682,10 @@ module.exports = { discardTrack, replay, replayPlannedSnapshot, + getStatisticsByManifestIds, findPinsForRevision, findPinsForObject, buildManifestEntries, + MANIFEST_SCHEMA_VERSION, RESOLVER_VERSION, }; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index ebc3e698..3945d576 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -353,6 +353,14 @@ exports.deleteSnapshot = function deleteSnapshot(trackId, modified) { return snapshotService.deleteSnapshot(trackId, modified); }; +exports.createSnapshotGraph = function createSnapshotGraph(trackId, modified) { + return snapshotService.createGraph(trackId, modified); +}; + +exports.deleteSnapshotGraph = function deleteSnapshotGraph(trackId, modified) { + return snapshotService.deleteGraph(trackId, modified); +}; + // ----------------------------------------------------------------------------- // Ephemeral (Phase 6 → ephemeral-service) // ----------------------------------------------------------------------------- @@ -423,10 +431,9 @@ async function renderReleasePlan(plan, options) { if (format === 'bundle') { return exportService.exportSnapshot(plan.plannedSnapshot, format, { ...options, - // A virtual release does not alter its members. Replaying the draft's - // persisted graph keeps preview output identical to the graph that will - // be tagged instead of resolving current database state a second time. - captureGraph: plan.sourceSnapshot.type !== 'virtual', + // Release previews are intentionally live. Determinism begins only if a + // caller explicitly creates a graph after the snapshot is tagged. + captureGraph: true, }); } return formatWorkbenchSnapshot(plan.plannedSnapshot, options); diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 8c4f2ea9..c796caaf 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -27,6 +27,7 @@ const { NotFoundError, TaggedSnapshotDeletionError, HistoricalSnapshotDeletionError, + ReleaseConflictError, } = require('../../exceptions'); // ============================================================================= @@ -104,38 +105,6 @@ async function emitContentsChanged(trackId, snapshot) { } exports.emitContentsChanged = emitContentsChanged; -/** - * Persist a snapshot and its graph manifest as one logical operation. - * - * Mongo transactions are not available across the dynamically named snapshot - * collections in every supported deployment. A pending manifest plus - * compensation keeps partially completed writes invisible to replay and - * protection queries. - */ -async function saveSnapshotWithManifest(trackId, snapshotData) { - const manifestId = await graphManifestService.prepare(snapshotData); - snapshotData.graph_manifest_id = manifestId; - - try { - const saved = await dynamicRepo.saveSnapshot(trackId, snapshotData); - try { - await graphManifestService.activate(manifestId); - } catch (err) { - // The saved snapshot is already linked to a complete pending manifest. - // Replay can safely activate it later, so do not turn a committed - // snapshot into an ambiguous client-visible failure. - logger.warn( - `SnapshotService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, - ); - } - return saved; - } catch (err) { - await graphManifestService.discard(manifestId); - throw err; - } -} -exports.saveSnapshotWithManifest = saveSnapshotWithManifest; - // ============================================================================= // Track management // ============================================================================= @@ -198,7 +167,7 @@ exports.createTrack = async function createTrack(data) { // Create collection + indexes, then persist the initial snapshot await modelFactory.ensureIndexes(trackId); - const snapshot = await saveSnapshotWithManifest(trackId, initialSnapshot); + const snapshot = await dynamicRepo.saveSnapshot(trackId, initialSnapshot); // Register in the central registry await registryRepo.create({ @@ -226,7 +195,8 @@ exports.createTrack = async function createTrack(data) { * List lightweight summaries of a track's snapshots. * * Standard summaries expose members/staged/candidates counts. Virtual - * summaries expose members/quarantine counts. + * summaries expose members/quarantine counts. Summaries linked to a graph + * manifest also expose counts by manifest entry role. * * @param {string} trackId * @param {Object} options - { tagged?, limit, offset } @@ -240,6 +210,9 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { } const result = await dynamicRepo.getSnapshotSummaries(trackId, options); + const graphStatisticsByManifestId = await graphManifestService.getStatisticsByManifestIds( + result.data.map((snapshot) => snapshot.graph_manifest_id), + ); return { ...result, data: result.data.map((snapshot) => { @@ -248,6 +221,10 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { type: snapshot.type, modified: snapshot.modified, version: snapshot.version, + graph_manifest_id: snapshot.graph_manifest_id, + graph_statistics: snapshot.graph_manifest_id + ? graphStatisticsByManifestId.get(snapshot.graph_manifest_id) + : undefined, name: snapshot.name, description: snapshot.description, members_count: snapshot.members_count, @@ -339,7 +316,15 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov } const normalized = tierRevisionInvariant.normalizeSnapshot(clone); - const saved = await saveSnapshotWithManifest(trackId, normalized.snapshot); + const saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); + if (saved.type === 'standard') { + const prunedDrafts = await dynamicRepo.deleteOlderDrafts(trackId, saved.modified); + await Promise.all( + prunedDrafts + .filter((snapshot) => snapshot.graph_manifest_id) + .map((snapshot) => graphManifestService.discard(snapshot.graph_manifest_id)), + ); + } await syncRegistryCounters(trackId); // The clone (modified = now) is the track's new latest snapshot @@ -408,7 +393,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { ); await modelFactory.ensureIndexes(newTrackId); - const saved = await saveSnapshotWithManifest(newTrackId, normalized.snapshot); + const saved = await dynamicRepo.saveSnapshot(newTrackId, normalized.snapshot); await registryRepo.create({ track_id: newTrackId, @@ -528,6 +513,84 @@ exports.updateConfig = async function updateConfig(trackId, config, _userId) { return exports.cloneSnapshot(trackId, source, { config: mergedConfig }); }; +// ============================================================================= +// Optional deterministic member graphs +// ============================================================================= + +async function createGraph(trackId, modified, prepareManifest, validateExisting) { + const snapshot = await dynamicRepo.getSnapshotByModified(trackId, modified); + if (!snapshot) { + throw new NotFoundError({ + details: `Snapshot with modified '${modified}' not found for track '${trackId}'`, + }); + } + if (snapshot.version == null) { + throw new ReleaseConflictError('Only tagged snapshots can be made deterministic', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + if (snapshot.graph_manifest_id) { + if (validateExisting) await validateExisting(snapshot); + return { snapshot, created: false }; + } + + const manifestId = await prepareManifest(snapshot); + const attached = await dynamicRepo.attachGraphManifest(trackId, snapshot.modified, manifestId); + if (!attached) { + await graphManifestService.discard(manifestId); + const current = await dynamicRepo.getSnapshotByModified(trackId, modified); + if (current?.graph_manifest_id) return { snapshot: current, created: false }; + throw new ReleaseConflictError('Snapshot changed while its graph was being created', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + + try { + await graphManifestService.activate(manifestId); + } catch (err) { + logger.warn( + `SnapshotService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, + ); + } + return { snapshot: attached, created: true }; +} + +exports.createGraph = function createLiveGraph(trackId, modified) { + return createGraph(trackId, modified, (snapshot) => graphManifestService.prepare(snapshot)); +}; + +exports.deleteGraph = async function deleteGraph(trackId, modified) { + const snapshot = await dynamicRepo.getSnapshotByModified(trackId, modified); + if (!snapshot) { + throw new NotFoundError({ + details: `Snapshot with modified '${modified}' not found for track '${trackId}'`, + }); + } + if (snapshot.version == null) { + throw new ReleaseConflictError('Only tagged snapshots can have deterministic graphs', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + if (!snapshot.graph_manifest_id) return false; + + const detached = await dynamicRepo.detachGraphManifest( + trackId, + snapshot.modified, + snapshot.graph_manifest_id, + ); + if (!detached) { + throw new ReleaseConflictError('Snapshot graph changed while it was being deleted', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + await graphManifestService.discard(snapshot.graph_manifest_id); + return true; +}; + // ============================================================================= // Deletion // ============================================================================= @@ -584,6 +647,14 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { throw new HistoricalSnapshotDeletionError(snapshot.modified, latest?.modified); } + const predecessor = await dynamicRepo.getLatestSnapshotBefore(trackId, snapshot.modified); + if (!predecessor) { + throw new ReleaseConflictError('The only snapshot in a release track cannot be deleted', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + await dynamicRepo.deleteSnapshot(trackId, modified); await graphManifestService.discardSnapshot(trackId, snapshot.modified); await syncRegistryCounters(trackId); diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 621f68c4..f4c2870a 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -281,57 +281,32 @@ async function planLoadedSnapshot(trackId, snapshot, options) { async function commitPlan(plan) { if (plan.blockingError) throw plan.blockingError; - const reuseVirtualManifest = - plan.sourceSnapshot.type === 'virtual' && Boolean(plan.sourceSnapshot.graph_manifest_id); - const manifestId = reuseVirtualManifest - ? plan.sourceSnapshot.graph_manifest_id - : await graphManifestService.prepare(plan.plannedSnapshot); - - let tagged; - try { - tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { - version: plan.version, - versionHistoryEntry: plan.versionHistoryEntry, - additionalOps: { - ...plan.additionalOps, - graph_manifest_id: manifestId, - }, - }); - } catch (err) { - if (!reuseVirtualManifest) { - await graphManifestService.discard(manifestId); - } - throw err; - } + const obsoleteManifestId = plan.sourceSnapshot.graph_manifest_id; + const tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { + version: plan.version, + versionHistoryEntry: plan.versionHistoryEntry, + additionalOps: plan.additionalOps, + // Older deployments attached graphs to drafts. Releasing changes the + // member set, so that legacy draft graph cannot describe the release. + unsetOps: obsoleteManifestId ? { graph_manifest_id: '' } : undefined, + }); if (!tagged) { - if (!reuseVirtualManifest) { - await graphManifestService.discard(manifestId); - } await releaseHistoryService.reconcileTaggedReleases(plan.trackId); throw new AlreadyReleasedError('(concurrent release)'); } - // Link the complete pending manifest before activation. The snapshot link - // is the durable commit record, and replay can recover a linked pending - // manifest if the process stops in this narrow window. - if (!reuseVirtualManifest) { + if (obsoleteManifestId) { try { - await graphManifestService.activate(manifestId); + await graphManifestService.discard(obsoleteManifestId); } catch (err) { logger.warn( - `VersioningService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, + `VersioningService: Deferred cleanup for obsolete graph manifest ` + + `"${obsoleteManifestId}": ${err.message}`, ); } } - if ( - plan.sourceSnapshot.graph_manifest_id && - plan.sourceSnapshot.graph_manifest_id !== manifestId - ) { - await graphManifestService.discard(plan.sourceSnapshot.graph_manifest_id); - } - await releaseHistoryService.reconcileTaggedReleases(plan.trackId); const latest = await dynamicRepo.getLatestSnapshot(plan.trackId); await snapshotService.emitContentsChanged(plan.trackId, latest); diff --git a/app/services/stix/bundle-graph-resolver.js b/app/services/stix/bundle-graph-resolver.js index bde27257..086a487a 100644 --- a/app/services/stix/bundle-graph-resolver.js +++ b/app/services/stix/bundle-graph-resolver.js @@ -35,6 +35,7 @@ class BundleGraphResolver { options, relationships, onMissingDependency, + prefetchedDocuments = [], }) { this.attackObjectsRepository = attackObjectsRepository; this.detectionStrategiesRepository = detectionStrategiesRepository; @@ -46,6 +47,12 @@ class BundleGraphResolver { this.onMissingDependency = onMissingDependency; this.attackObjectCache = new Map(); + for (const document of prefetchedDocuments) { + this.attackObjectCache.set( + this.revisionKey(document.stix.id, document.stix.modified), + _.cloneDeep(document), + ); + } this.attackObjectByAttackIdCache = new Map(); this.domainCache = new Map(); this.dependencies = new Map(); diff --git a/app/services/system/notes-service.js b/app/services/system/notes-service.js index c2c74c00..6011fe31 100644 --- a/app/services/system/notes-service.js +++ b/app/services/system/notes-service.js @@ -1,53 +1,14 @@ 'use strict'; -const _ = require('lodash'); const notesRepository = require('../../repository/notes-repository'); const { BaseService } = require('../meta-classes'); const { Note: NoteType } = require('../../lib/types'); -const { - BadlyFormattedParameterError, - DuplicateIdError, - MissingParameterError, -} = require('../../exceptions'); +const { BadlyFormattedParameterError } = require('../../exceptions'); class NotesService extends BaseService { - async updateVersion(stixId, stixModified, data) { - if (!stixId) { - throw new MissingParameterError('stixId'); - } - - if (!stixModified) { - throw new MissingParameterError('stixModified'); - } - - try { - const document = await this.repository.retrieveOneByVersion(stixId, stixModified); - - if (!document) { - // document not found - return null; - } else { - // Copy data to found document and save - try { - _.merge(document, data); - const savedDocument = await document.save(); - return savedDocument; - } catch (err) { - if (err.name === 'MongoServerError' && err.code === 11000) { - throw new DuplicateIdError(); - } else { - throw err; - } - } - } - } catch (err) { - if (err.name === 'CastError') { - throw new BadlyFormattedParameterError(); - } else { - throw err; - } - } + async updateVersion(stixId, stixModified, data, options) { + return this.updateFull(stixId, stixModified, data, options); } /** diff --git a/app/tests/api/analytics/analytics.spec.js b/app/tests/api/analytics/analytics.spec.js index c0499970..469b3215 100644 --- a/app/tests/api/analytics/analytics.spec.js +++ b/app/tests/api/analytics/analytics.spec.js @@ -184,22 +184,18 @@ describe('Analytics API', function () { ); }); - it('PUT /api/analytics updates a analytic', async function () { - analytic1.stix.description = 'This is an updated analytic.'; - const body = analytic1; + it('PUT /api/analytics rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(analytic1); + body.stix.description = 'This is an updated analytic.'; const res = await request(app) .put('/api/analytics/' + analytic1.stix.id + '/modified/' + analytic1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated analytic - const analytic = res.body; - expect(analytic).toBeDefined(); - expect(analytic.stix.id).toBe(analytic1.stix.id); - expect(analytic.stix.modified).toBe(analytic1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/analytics does not create a analytic with the same id and modified date', async function () { diff --git a/app/tests/api/assets/assets.spec.js b/app/tests/api/assets/assets.spec.js index b2c742a4..22cf374a 100644 --- a/app/tests/api/assets/assets.spec.js +++ b/app/tests/api/assets/assets.spec.js @@ -193,22 +193,18 @@ describe('Assets API', function () { expect(asset.stix.x_mitre_related_assets.length).toBe(2); }); - it('PUT /api/assets updates an asset', async function () { - asset1.stix.description = 'This is an updated asset.'; - const body = asset1; + it('PUT /api/assets rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(asset1); + body.stix.description = 'This is an updated asset.'; const res = await request(app) .put('/api/assets/' + asset1.stix.id + '/modified/' + asset1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated asset - const asset = res.body; - expect(asset).toBeDefined(); - expect(asset.stix.id).toBe(asset1.stix.id); - expect(asset.stix.modified).toBe(asset1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/assets does not create an asset with the same id and modified date', async function () { diff --git a/app/tests/api/base-services/update-identity-guard.spec.js b/app/tests/api/base-services/update-identity-guard.spec.js index 03ea2437..f30edb29 100644 --- a/app/tests/api/base-services/update-identity-guard.spec.js +++ b/app/tests/api/base-services/update-identity-guard.spec.js @@ -34,12 +34,9 @@ function buildTechnique(name) { }; } -// Revision identity (stix.id + stix.modified) is immutable in place: a PUT -// whose body identity fields differ from the path parameters must be -// rejected. Release tracks pin revisions by (stix.id, stix.modified) — -// re-keying a document in place would strand those pins. Re-keying goes -// through POST (a new revision) instead. -describe('PUT revision identity guard', function () { +// Persisted STIX revisions are immutable. Identity mismatches remain malformed +// requests (400), while a same-identity STIX edit is a conflict (409). +describe('PUT revision immutability guard', function () { let app; let passportCookie; let technique; @@ -102,15 +99,21 @@ describe('PUT revision identity guard', function () { expect(res.body.stix.modified).toBe(technique.stix.modified); }); - it('accepts a PUT whose body identity matches the path parameters', async function () { + it('rejects a STIX-changing PUT whose body identity matches the path parameters', async function () { const update = buildTechnique('Identity Guard (updated)'); update.stix.id = technique.stix.id; update.stix.created = technique.stix.created; update.stix.modified = technique.stix.modified; - const res = await putTechnique(update).expect(200); - expect(res.body.stix.name).toBe('Identity Guard (updated)'); - expect(res.body.stix.modified).toBe(technique.stix.modified); + const res = await putTechnique(update).expect(409); + expect(res.body.message).toContain('immutable'); + + const stored = await request(app) + .get(`/api/techniques/${technique.stix.id}/modified/${technique.stix.modified}`) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + expect(stored.body.stix.name).toBe('Identity Guard'); }); after(async function () { diff --git a/app/tests/api/campaigns/campaigns.spec.js b/app/tests/api/campaigns/campaigns.spec.js index 2cca1846..a24fa178 100644 --- a/app/tests/api/campaigns/campaigns.spec.js +++ b/app/tests/api/campaigns/campaigns.spec.js @@ -217,22 +217,18 @@ describe('Campaigns API', function () { ); }); - it('PUT /api/campaigns updates a campaign', async function () { - campaign1.stix.description = 'This is an updated campaign. Blue.'; - const body = campaign1; + it('PUT /api/campaigns rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(campaign1); + body.stix.description = 'This is an updated campaign. Blue.'; const res = await request(app) .put('/api/campaigns/' + campaign1.stix.id + '/modified/' + campaign1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated campaign - const campaign = res.body; - expect(campaign).toBeDefined(); - expect(campaign.stix.id).toBe(campaign1.stix.id); - expect(campaign.stix.modified).toBe(campaign1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/campaigns does not create a campaign with the same id and modified date', async function () { diff --git a/app/tests/api/data-components/data-components.spec.js b/app/tests/api/data-components/data-components.spec.js index 592fb400..d3d7a5ff 100644 --- a/app/tests/api/data-components/data-components.spec.js +++ b/app/tests/api/data-components/data-components.spec.js @@ -276,9 +276,9 @@ describe('Data Components API', function () { ); }); - it('PUT /api/data-components updates a data component', async function () { - dataComponent1.stix.description = 'This is an updated data component.'; - const body = dataComponent1; + it('PUT /api/data-components rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(dataComponent1); + body.stix.description = 'This is an updated data component.'; const res = await request(app) .put( '/api/data-components/' + @@ -289,14 +289,10 @@ describe('Data Components API', function () { .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated data component - const dataComponent = res.body; - expect(dataComponent).toBeDefined(); - expect(dataComponent.stix.id).toBe(dataComponent1.stix.id); - expect(dataComponent.stix.modified).toBe(dataComponent1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/data-components does not create a data component with the same id and modified date', async function () { diff --git a/app/tests/api/data-sources/data-sources.spec.js b/app/tests/api/data-sources/data-sources.spec.js index f760ab4a..37437f4d 100644 --- a/app/tests/api/data-sources/data-sources.spec.js +++ b/app/tests/api/data-sources/data-sources.spec.js @@ -247,22 +247,18 @@ describe('Data Sources API', function () { expect(dataSource.dataComponents.length).toBe(5); }); - it('PUT /api/data-sources updates a data source', async function () { - dataSource1.stix.description = 'This is an updated data source.'; + it('PUT /api/data-sources rejects STIX changes to a persisted revision', async function () { const body = cloneForCreate(dataSource1); + body.stix.description = 'This is an updated data source.'; const res = await request(app) .put('/api/data-sources/' + dataSource1.stix.id + '/modified/' + dataSource1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated data source - const dataSource = res.body; - expect(dataSource).toBeDefined(); - expect(dataSource.stix.id).toBe(dataSource1.stix.id); - expect(dataSource.stix.modified).toBe(dataSource1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/data-sources does not create a data source with the same id and modified date', async function () { diff --git a/app/tests/api/detection-strategies/detection-strategies-spec.js b/app/tests/api/detection-strategies/detection-strategies-spec.js index 16d12a1d..4c7119dd 100644 --- a/app/tests/api/detection-strategies/detection-strategies-spec.js +++ b/app/tests/api/detection-strategies/detection-strategies-spec.js @@ -262,9 +262,9 @@ describe('Detection Strategies API', function () { ); }); - it('PUT /api/detection-strategies updates a detection strategy', async function () { - detectionStrategy1.stix.name = 'This is an updated detection strategy.'; - const body = detectionStrategy1; + it('PUT /api/detection-strategies rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(detectionStrategy1); + body.stix.name = 'This is an updated detection strategy.'; const res = await request(app) .put( '/api/detection-strategies/' + @@ -275,14 +275,10 @@ describe('Detection Strategies API', function () { .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated detection strategy - const detectionStrategy = res.body; - expect(detectionStrategy).toBeDefined(); - expect(detectionStrategy.stix.id).toBe(detectionStrategy1.stix.id); - expect(detectionStrategy.stix.modified).toBe(detectionStrategy1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/detection-strategies does not create a detection strategy with the same id and modified date', async function () { diff --git a/app/tests/api/groups/groups.spec.js b/app/tests/api/groups/groups.spec.js index cca8ebe8..f886ebb4 100644 --- a/app/tests/api/groups/groups.spec.js +++ b/app/tests/api/groups/groups.spec.js @@ -199,22 +199,18 @@ describe('Groups API', function () { expect(group.stix.x_mitre_attack_spec_version).toBe(group1.stix.x_mitre_attack_spec_version); }); - it('PUT /api/groups updates a group', async function () { - group1.stix.description = 'This is an updated group. Blue.'; - const body = group1; + it('PUT /api/groups rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(group1); + body.stix.description = 'This is an updated group. Blue.'; const res = await request(app) .put('/api/groups/' + group1.stix.id + '/modified/' + group1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated group - const group = res.body; - expect(group).toBeDefined(); - expect(group.stix.id).toBe(group1.stix.id); - expect(group.stix.modified).toBe(group1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/groups does not create a group with the same id and modified date', async function () { diff --git a/app/tests/api/identities/identities.spec.js b/app/tests/api/identities/identities.spec.js index 3efd1676..00ab0f42 100644 --- a/app/tests/api/identities/identities.spec.js +++ b/app/tests/api/identities/identities.spec.js @@ -325,22 +325,18 @@ describe('Identity API', function () { ); }); - it('PUT /api/identities updates an identity', async function () { - identity1.stix.description = 'This is an updated identity.'; - const body = identity1; + it('PUT /api/identities rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(identity1); + body.stix.description = 'This is an updated identity.'; const res = await request(app) .put('/api/identities/' + identity1.stix.id + '/modified/' + identity1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated identity - const identity = res.body; - expect(identity).toBeDefined(); - expect(identity.stix.id).toBe(identity1.stix.id); - expect(identity.stix.modified).toBe(identity1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/identities does not create an identity with the same id and modified date', async function () { diff --git a/app/tests/api/matrices/matrices.spec.js b/app/tests/api/matrices/matrices.spec.js index f61ac4b4..6985f6a8 100644 --- a/app/tests/api/matrices/matrices.spec.js +++ b/app/tests/api/matrices/matrices.spec.js @@ -171,23 +171,19 @@ describe('Matrices API', function () { expect(matrix.stix.x_mitre_attack_spec_version).toBe(matrix1.stix.x_mitre_attack_spec_version); }); - it('PUT /api/matrices updates a matrix', async function () { - matrix1.stix.description = 'This is an updated matrix.'; - const body = matrix1; + it('PUT /api/matrices rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(matrix1); + body.stix.description = 'This is an updated matrix.'; const res = await request(app) .put('/api/matrices/' + matrix1.stix.id + '/modified/' + matrix1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated matrix - const matrix = res.body; - expect(matrix).toBeDefined(); - expect(matrix.stix.id).toBe(matrix1.stix.id); - expect(matrix.stix.modified).toBe(matrix1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/matrices does not create a matrix with the same id and modified date', async function () { diff --git a/app/tests/api/mitigations/mitigations.spec.js b/app/tests/api/mitigations/mitigations.spec.js index d8775eb8..d2afe3f8 100644 --- a/app/tests/api/mitigations/mitigations.spec.js +++ b/app/tests/api/mitigations/mitigations.spec.js @@ -167,22 +167,18 @@ describe('Mitigations API', function () { expect(mitigation.stix.labels.length).toBe(mitigation1.stix.labels.length); }); - it('PUT /api/mitigations updates a mitigation', async function () { - mitigation1.stix.description = 'This is an updated mitigation.'; - const body = mitigation1; + it('PUT /api/mitigations rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(mitigation1); + body.stix.description = 'This is an updated mitigation.'; const res = await request(app) .put('/api/mitigations/' + mitigation1.stix.id + '/modified/' + mitigation1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated mitigation - const mitigation = res.body; - expect(mitigation).toBeDefined(); - expect(mitigation.stix.id).toBe(mitigation1.stix.id); - expect(mitigation.stix.modified).toBe(mitigation1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/mitigations does not create a mitigation with the same id and modified date', async function () { diff --git a/app/tests/api/notes/notes.spec.js b/app/tests/api/notes/notes.spec.js index eb61c527..f2361deb 100644 --- a/app/tests/api/notes/notes.spec.js +++ b/app/tests/api/notes/notes.spec.js @@ -184,22 +184,18 @@ describe('Notes API', function () { expect(note.stix.x_mitre_attack_spec_version).toBe(note1.stix.x_mitre_attack_spec_version); }); - it('PUT /api/notes should update a note', async function () { - note1.stix.description = 'This is an updated note.'; - const body = note1; + it('PUT /api/notes rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(note1); + body.stix.description = 'This is an updated note.'; const res = await request(app) .put('/api/notes/' + note1.stix.id + '/modified/' + note1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated note - const note = res.body; - expect(note).toBeDefined(); - expect(note.stix.id).toBe(note1.stix.id); - expect(note.stix.modified).toBe(note1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/notes should not create a note with the same id and modified date', async function () { diff --git a/app/tests/api/relationships/relationships.spec.js b/app/tests/api/relationships/relationships.spec.js index 982968e7..ed4b16b5 100644 --- a/app/tests/api/relationships/relationships.spec.js +++ b/app/tests/api/relationships/relationships.spec.js @@ -251,9 +251,9 @@ describe('Relationships API', function () { ); }); - it('PUT /api/relationships updates a relationship', async function () { - relationship1a.stix.description = 'This is an updated relationship.'; - const body = relationship1a; + it('PUT /api/relationships rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(relationship1a); + body.stix.description = 'This is an updated relationship.'; const res = await request(app) .put( '/api/relationships/' + @@ -264,17 +264,13 @@ describe('Relationships API', function () { .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated relationship - const relationship = res.body; - expect(relationship).toBeDefined(); - expect(relationship.stix.id).toBe(relationship1a.stix.id); - expect(relationship.stix.modified).toBe(relationship1a.stix.modified); + expect(res.body.message).toContain('immutable'); }); - it('PUT /api/relationships rejects in-place endpoint changes', async function () { + it('PUT /api/relationships rejects endpoint changes', async function () { const body = structuredClone(relationship1a); body.stix.source_ref = sourceRef2; diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 9e4689e2..2bd2ff5b 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -216,6 +216,15 @@ describe('Deterministic snapshot graph migration', function () { .exec(); expect(manifests.length).toBeGreaterThan(0); expect(manifests.every((manifest) => manifest.baseline_reconstruction === true)).toBe(true); + expect(manifests.every((manifest) => manifest.schema_version === 1)).toBe(true); + const legacyRelationshipEntry = await ReleaseTrackGraphManifestEntry.findOne({ + manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) }, + kind: 'relationship', + object_ref: relationship.stix.id, + }) + .lean() + .exec(); + expect(legacyRelationshipEntry.frozen_stix.description).toBe(relationship.stix.description); const countAfterFirstRun = manifests.length; await migration.up(mongoose.connection.db); diff --git a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js index 0b3b54bd..27b5f5a6 100644 --- a/app/tests/api/release-tracks/release-tracks-backrefs.spec.js +++ b/app/tests/api/release-tracks/release-tracks-backrefs.spec.js @@ -301,6 +301,11 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { it('deleting the latest draft reverts its candidate backrefs', async function () { const technique = await postObject('/api/techniques', buildTechnique('Backref Contents')); const trackId = await createTrack('Backref Contents Track'); + await postObject( + `/api/release-tracks/${trackId}/snapshots/latest/release`, + { version: '1.0' }, + 200, + ); const candidateSnapshot = await addCandidates(trackId, [technique]); @@ -312,8 +317,8 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { status: 'work-in-progress', }); - // Deleting the latest snapshot reverts contents to the previous - // (empty) snapshot — the backref disappears + // Deleting the latest rolling draft reverts contents to the preceding + // empty tagged snapshot — the backref disappears. await request(app) .delete(`/api/release-tracks/${trackId}/snapshots/${candidateSnapshot.modified}`) .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) @@ -688,10 +693,7 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { const trackId = await createTrack('Backref Injection Track'); await addCandidates(trackId, [technique]); - const update = buildTechnique('Backref Injection Put (updated)'); - update.stix.id = technique.stix.id; - update.stix.created = technique.stix.created; - update.stix.modified = technique.stix.modified; + const update = JSON.parse(JSON.stringify(technique)); update.workspace.release_tracks = [ { id: 'release-track--00000000-0000-4000-8000-000000000000', tier: 'members' }, ]; @@ -703,14 +705,13 @@ describe('Release Track Backrefs (workspace.release_tracks) API', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); - // The in-place PUT is captured by revision sync: the entry keeps its - // pin but is marked modified-in-place; the fake client-supplied entry - // is discarded + // Workspace-only PUT retains the real server-managed entry and discards + // the fake client-supplied one without creating a content revision. expect(entryForTrack(res.body, trackId)).toEqual({ id: trackId, type: 'standard', tier: 'candidates', - status: 'modified-in-place', + status: 'work-in-progress', }); expect(trackEntries(res.body)).toHaveLength(1); }); diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index 02017e9c..9fbf65ca 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -47,6 +47,7 @@ describe('Release Tracks Bundle Export API', function () { let organizationIdentityId; let trackId; let trackUuid; + let taggedModified; let snapshotModified; let memberObject; @@ -247,11 +248,17 @@ describe('Release Tracks Bundle Export API', function () { // Members enter through the supported candidate → staged → release // lifecycle. - await releaseExactMembers(app, passportCookie, trackId, [ + const tagged = await releaseExactMembers(app, passportCookie, trackId, [ memberObject, linkedMemberObject, relationshipSource, ]); + taggedModified = tagged.modified; + await postAction( + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent(taggedModified)}/graph`, + {}, + 201, + ); // Candidates (all start as work-in-progress) await postAction(`/api/release-tracks/${trackId}/candidates`, { @@ -353,9 +360,15 @@ describe('Release Tracks Bundle Export API', function () { ); }); - it('replays frozen relationship payloads and protects graph dependencies', async function () { + it('replays exact relationship pointers and protects graph dependencies', async function () { const relationshipUpdate = JSON.parse(JSON.stringify(secondaryRelationship)); - relationshipUpdate.stix.description = 'A later in-place typo correction.'; + delete relationshipUpdate._id; + delete relationshipUpdate.__v; + delete relationshipUpdate.__t; + relationshipUpdate.stix.modified = new Date( + new Date(secondaryRelationship.stix.modified).getTime() + 1000, + ).toISOString(); + relationshipUpdate.stix.description = 'A corrected relationship revision.'; relationshipUpdate.stix.external_references = [ { source_name: 'deterministic-bundle-test', @@ -364,22 +377,22 @@ describe('Release Tracks Bundle Export API', function () { ]; await request(app) - .put( - `/api/relationships/${secondaryRelationship.stix.id}/modified/` + - encodeURIComponent(secondaryRelationship.stix.modified), - ) + .post('/api/relationships') .send(relationshipUpdate) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200); + .expect(201); const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( + taggedModified, + )}?format=bundle&includeToc=false`, ); - const frozenRelationship = bundle.objects.find( + const pinnedRelationship = bundle.objects.find( (object) => object.id === secondaryRelationship.stix.id, ); - expect(frozenRelationship.description).toBe('Frozen relationship description.'); + expect(pinnedRelationship.modified).toBe(secondaryRelationship.stix.modified); + expect(pinnedRelationship.description).toBe('Frozen relationship description.'); await request(app) .delete( diff --git a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js index 246645e6..83436bbf 100644 --- a/app/tests/api/release-tracks/release-tracks-change-capture.spec.js +++ b/app/tests/api/release-tracks/release-tracks-change-capture.spec.js @@ -30,17 +30,16 @@ function buildTechnique(name) { object_marking_refs: [staticMarkingDefinitionId], kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], x_mitre_is_subtechnique: false, + x_mitre_domains: ['enterprise-attack'], x_mitre_platforms: ['Windows'], + x_mitre_version: '1.0', }, }; } -// Release tracks must never be blind to in-place mutations: -// - PUT/DELETE of a members-pinned revision is rejected (409) — released -// content is immutable in place; changes go through a new revision. -// - PUT of a candidate/staged-pinned revision resets the tier entry for -// re-review (staged entries demote back to candidates). -// - Revoking a tracked object enrolls/re-pins the revoked revision. +// Persisted STIX revisions are immutable regardless of tier. Workspace-only +// PUT remains available and must not masquerade as a content revision. +// Revoking a tracked object still creates and enrolls a new revision. describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { let app; let passportCookie; @@ -136,7 +135,7 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { technique, buildUpdateBody(technique, 'Capture Member (edited)'), ).expect(409); - expect(res.text).toContain('members tier'); + expect(res.text).toContain('Persisted STIX revisions are immutable'); const retrieved = await getTechniqueVersion(technique.stix.id, technique.stix.modified); expect(retrieved.stix.name).toBe('Capture Member'); @@ -179,8 +178,8 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { }); }); - describe('in-place edits of candidate/staged-pinned revisions', function () { - it('marks a reviewed candidate entry modified-in-place on in-place PUT', async function () { + describe('candidate/staged revision immutability', function () { + it('rejects a candidate STIX edit without changing its pin or review status', async function () { const technique = await postObject('/api/techniques', buildTechnique('Capture Candidate')); const trackId = await createTrack('Capture Candidate Track'); await addCandidate(trackId, technique); @@ -190,137 +189,25 @@ describe('Release Track Change Capture (PUT/DELETE/revoke) API', function () { 200, ); - const res = await putTechnique( + await putTechnique( technique, buildUpdateBody(technique, 'Capture Candidate (edited)'), - ).expect(200); - - // The PUT response reflects the marker (read-your-own-writes) - expect(entryForTrack(res.body, trackId)).toEqual({ - id: trackId, - type: 'standard', - tier: 'candidates', - status: 'modified-in-place', - }); + ).expect(409); const { candidates } = await getJson(`/api/release-tracks/${trackId}/candidates`); expect(candidates).toHaveLength(1); - expect(candidates[0].object_status).toBe('modified-in-place'); - expect(candidates[0].object_modified).toBe('latest'); - - // The marker is reviewable: modified-in-place → awaiting-review - await postObject( - `/api/release-tracks/${trackId}/candidates/review`, - { from: 'modified-in-place', to: 'awaiting-review' }, - 200, - ); - const after = await getJson(`/api/release-tracks/${trackId}/candidates`); - expect(after.candidates[0].object_status).toBe('awaiting-review'); + expect(candidates[0].object_status).toBe('awaiting-review'); + expect(new Date(candidates[0].object_modified).toISOString()).toBe(technique.stix.modified); }); - it('demotes a staged entry back to candidates on in-place PUT', async function () { - const technique = await postObject('/api/techniques', buildTechnique('Capture Staged')); - const trackId = await createTrack('Capture Staged Track'); + it('allows workspace-only PUT without cloning the rolling draft', async function () { + const technique = await postObject('/api/techniques', buildTechnique('Capture Workspace')); + const trackId = await createTrack('Capture Workspace Track'); await addCandidate(trackId, technique); - await postObject( - `/api/release-tracks/${trackId}/candidates/promote`, - { object_refs: [technique.stix.id] }, - 200, - ); - - const res = await putTechnique( - technique, - buildUpdateBody(technique, 'Capture Staged (edited)'), - ).expect(200); - - expect(entryForTrack(res.body, trackId)).toEqual({ - id: trackId, - type: 'standard', - tier: 'candidates', - status: 'modified-in-place', - }); - - const snapshot = await getJson(`/api/release-tracks/${trackId}/snapshots/latest`); - expect(snapshot.staged).toHaveLength(0); - expect(snapshot.candidates).toHaveLength(1); - }); - - it('keeps a staged entry staged in a permissive track (candidacy threshold codified)', async function () { - const technique = await postObject('/api/techniques', buildTechnique('Capture Permissive')); - const trackId = await createTrack('Capture Permissive Track'); - await request(app) - .put(`/api/release-tracks/${trackId}/config`) - .send({ candidacy_threshold: 'work-in-progress', auto_promote: true }) - .set('Accept', 'application/json') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200); - - // In a permissive track the fresh candidate auto-promotes immediately - await addCandidate(trackId, technique); - let snapshot = await getJson(`/api/release-tracks/${trackId}/snapshots/latest`); - expect(snapshot.staged).toHaveLength(1); - - // An in-place edit is marked, but the tier is decided by the workflow - // gate: modified-in-place meets the work-in-progress threshold, so the - // entry stays staged instead of being demoted - const res = await putTechnique( - technique, - buildUpdateBody(technique, 'Capture Permissive (edited)'), - ).expect(200); - - expect(entryForTrack(res.body, trackId)).toEqual({ - id: trackId, - type: 'standard', - tier: 'staged', - status: 'modified-in-place', - }); - snapshot = await getJson(`/api/release-tracks/${trackId}/snapshots/latest`); - expect(snapshot.staged).toHaveLength(1); - expect(snapshot.staged[0].object_status).toBe('modified-in-place'); - expect(snapshot.candidates).toHaveLength(0); - }); - - it('captures in-place deprecation of a reviewed candidate', async function () { - const technique = await postObject('/api/techniques', buildTechnique('Capture Deprecate')); - const trackId = await createTrack('Capture Deprecate Track'); - await addCandidate(trackId, technique); - await postObject( - `/api/release-tracks/${trackId}/candidates/review`, - { from: 'work-in-progress', to: 'awaiting-review' }, - 200, - ); - - const update = buildUpdateBody(technique, 'Capture Deprecate'); - update.stix.x_mitre_deprecated = true; - const res = await putTechnique(technique, update).expect(200); - - expect(res.body.stix.x_mitre_deprecated).toBe(true); - // The track saw the deprecation: the entry is marked for re-review - expect(entryForTrack(res.body, trackId)).toEqual({ - id: trackId, - type: 'standard', - tier: 'candidates', - status: 'modified-in-place', - }); - }); - - it('does not clone a snapshot when a repeat in-place PUT changes nothing track-visible', async function () { - const technique = await postObject('/api/techniques', buildTechnique('Capture Noop')); - const trackId = await createTrack('Capture Noop Track'); - await addCandidate(trackId, technique); - - // First in-place PUT marks the entry modified-in-place (new snapshot) - await putTechnique(technique, buildUpdateBody(technique, 'Capture Noop (edited)')).expect( - 200, - ); const before = await latestSnapshotModified(trackId); - - // Second in-place PUT: the entry is already modified-in-place in the - // same tier — no new snapshot should be created - await putTechnique( - technique, - buildUpdateBody(technique, 'Capture Noop (edited again)'), - ).expect(200); + const workspaceUpdate = JSON.parse(JSON.stringify(technique)); + workspaceUpdate.workspace.workflow.state = 'awaiting-review'; + await putTechnique(technique, workspaceUpdate).expect(200); const after = await latestSnapshotModified(trackId); expect(after).toBe(before); diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index a2d6686f..01139942 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -204,19 +204,15 @@ describe('Release-track release planning and commit API', function () { let responses; try { - responses = await Promise.all([release(track.modified), release(newerDraft.body.modified)]); + responses = await Promise.all([ + release(newerDraft.body.modified), + release(newerDraft.body.modified), + ]); } finally { historyStub.restore(); } expect(responses.map((response) => response.status).sort()).toEqual([200, 409]); - const conflict = responses.find((response) => response.status === 409); - expect(conflict.body).toEqual({ - message: `Release track ${track.id} already has tagged version 2.0`, - track_id: track.id, - version: '2.0', - }); - const tagged = await dynamicRepo.getAllSnapshots(track.id, { taggedOnly: true }); expect(tagged.pagination.total).toBe(1); expect(tagged.data[0].version).toBe('2.0'); @@ -304,7 +300,7 @@ describe('Release-track release planning and commit API', function () { expect(immutable.body.members[0].object_modified).toBe(revisionC.stix.modified); }); - it('resolves a historical draft dynamic selector when that draft is released', async function () { + it('carries a dynamic selector into the rolling replacement draft before release', async function () { const revisionA = (await post('/api/techniques', buildTechnique('Historical Dynamic A'), 201)) .body; const track = await createTrack('Historical Dynamic Release'); @@ -324,10 +320,13 @@ describe('Release-track release planning and commit API', function () { const revisionB = ( await post('/api/techniques', buildTechnique('Historical Dynamic B', revisionA), 201) ).body; - const releasePath = - `/api/release-tracks/${track.id}/snapshots/` + - `${encodeURIComponent(staged.body.modified)}/release`; - const released = await post(releasePath, { version: '4.0' }); + await get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(staged.body.modified)}`, + 404, + ); + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + version: '4.0', + }); expect(released.body.members).toEqual([ { @@ -508,20 +507,23 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version).toBe('1.0'); }); - it('previews and releases an explicitly selected historical snapshot', async function () { + it('prunes a replaced standard draft and releases the rolling draft', async function () { const track = await createTrack('Historical Release'); - await post(`/api/release-tracks/${track.id}/meta`, { description: 'new latest' }); - const preview = await get( + const replacement = await post(`/api/release-tracks/${track.id}/meta`, { + description: 'new latest', + }); + await get( `/api/release-tracks/${track.id}/snapshots/${track.modified}/release/preview?version=3.0`, + 404, ); - expect(preview.body.source_snapshot_modified).toBe(track.modified); - const released = await post( - `/api/release-tracks/${track.id}/snapshots/${track.modified}/release`, - { - version: '3.0', - }, + const preview = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?version=3.0`, ); - expect(released.body.modified).toBe(track.modified); + expect(preview.body.source_snapshot_modified).toBe(replacement.body.modified); + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + version: '3.0', + }); + expect(released.body.modified).toBe(replacement.body.modified); expect(released.body.version).toBe('3.0'); }); @@ -590,7 +592,7 @@ describe('Release-track release planning and commit API', function () { const unchanged = await get( `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(draftModified.toISOString())}`, ); - expect(unchanged.body.version).toBeNull(); + expect(unchanged.body.version ?? null).toBeNull(); }); it('compares a historical virtual draft with the tagged release that preceded it', async function () { diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 5aef3bc0..92819b3f 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -192,27 +192,11 @@ describe('Release Tracks API', function () { ); expectObjectInfo(quarantined, quarantinedObject); - const historicalRes = await request(app) + await request(app) .get(`/api/release-tracks/${trackId}/snapshots/${promoteRes.body.modified}`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) - .expect('Content-Type', /json/); - - const historicalMember = historicalRes.body.members.find( - (entry) => entry.object_ref === memberObject.stix.id, - ); - expectObjectInfo(historicalMember, memberObject); - - const historicalCandidate = historicalRes.body.candidates.find( - (entry) => entry.object_ref === candidateObject.stix.id, - ); - expectObjectInfo(historicalCandidate, candidateObject); - - const historicalStaged = historicalRes.body.staged.find( - (entry) => entry.object_ref === stagedObject.stix.id, - ); - expectObjectInfo(historicalStaged, stagedObject); + .expect(404); await request(app) .get(`/api/release-tracks/${trackId}/snapshots/latest?format=snapshot`) diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index 1c24567a..516e4870 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -67,7 +67,6 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const createdA = await createTrack('Releases By Object A'); trackA = createdA.id; - const initialSnapshotModified = createdA.modified; await setMembers(trackA, [objectRevisionA]); trackATaggedSnapshot = await releaseLatest(trackA); @@ -77,16 +76,6 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { await setMembers(trackA, [otherObject]); await releaseLatest(trackA); - // Retroactively tag the original empty draft. Its embedded history is - // stale, so the track-wide version ledger must produce 1.2 rather than 1.0. - await post( - `/api/release-tracks/${trackA}/snapshots/${initialSnapshotModified}/release`, - { - increment: 'minor', - }, - 200, - ); - const createdB = await createTrack('Releases By Object B'); trackB = createdB.id; await setMembers(trackB, [objectRevisionB]); @@ -206,24 +195,21 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { expect(response.body.data.every((entry) => entry.tagged_at && entry.tagged_by)).toBe(true); }); - it('maintains a reconciled registry catalogue during normal and retroactive tagging', async function () { + it('maintains a reconciled registry catalogue during normal tagging', async function () { const registry = await ReleaseTrackRegistry.findOne({ track_id: trackA }).lean().exec(); - expect(registry.tagged_release_count).toBe(3); - expect(registry.tagged_releases).toHaveLength(3); + expect(registry.tagged_release_count).toBe(2); + expect(registry.tagged_releases).toHaveLength(2); expect(registry.tagged_releases.map((release) => release.version).sort()).toEqual([ '1.0', '1.1', - '1.2', ]); - expect(registry.latest_tagged_version).toBe('1.2'); + expect(registry.latest_tagged_version).toBe('1.1'); }); it('previews the next version from the track-wide release ledger', async function () { - // Clone the latest snapshot after the retroactive 1.2 tag. The source - // snapshot predates that tag, so its embedded history does not contain it. await post( `/api/release-tracks/${trackA}/meta`, - { description: 'Draft created after a retroactive tag' }, + { description: 'Draft created after the current release' }, 200, ); @@ -233,7 +219,7 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { const major = await get( `/api/release-tracks/${trackA}/snapshots/latest/release/preview?increment=major`, ); - expect(minor.body.version).toBe('1.3'); + expect(minor.body.version).toBe('1.2'); expect(major.body.version).toBe('2.0'); }); @@ -289,9 +275,9 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { await backfillMigration.up(mongoose.connection.db); const registry = await ReleaseTrackRegistry.findOne({ track_id: trackA }).lean().exec(); - expect(registry.tagged_releases).toHaveLength(3); - expect(registry.tagged_release_count).toBe(3); - expect(registry.latest_tagged_version).toBe('1.2'); + expect(registry.tagged_releases).toHaveLength(2); + expect(registry.tagged_release_count).toBe(2); + expect(registry.latest_tagged_version).toBe('1.1'); const response = await get( `/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?type=standard`, diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js index adc7b717..04f29de9 100644 --- a/app/tests/api/release-tracks/snapshot-history.spec.js +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -6,6 +6,9 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const { + ReleaseTrackGraphManifestEntry, +} = require('../../../models/release-tracks/release-track-graph-manifest-model'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; const objectRevisions = []; @@ -74,6 +77,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(standardTrack), modified: standardTaggedModified, version: '1.0', + graph_manifest_id: 'release-track-graph-manifest--snapshot-history', members: [memberEntry(0), memberEntry(1)], staged: [stagedEntry(2, standardTaggedModified)], candidates: [ @@ -82,6 +86,35 @@ describe('GET /api/release-tracks/:id/snapshots', function () { candidateEntry(5, standardTaggedModified), ], }); + const manifestCommon = { + manifest_id: 'release-track-graph-manifest--snapshot-history', + track_id: standardTrack.id, + snapshot_modified: standardTaggedModified, + }; + const versionedManifestEntry = (index, kind, extra = {}) => ({ + ...manifestCommon, + revision_key: `${objectRevisions[index].id}::${new Date( + objectRevisions[index].modified, + ).getTime()}`, + kind, + object_ref: objectRevisions[index].id, + object_modified: objectRevisions[index].modified, + ...extra, + }); + await ReleaseTrackGraphManifestEntry.insertMany([ + versionedManifestEntry(0, 'root', { tier: 'members' }), + versionedManifestEntry(1, 'root', { tier: 'members' }), + versionedManifestEntry(2, 'secondary'), + versionedManifestEntry(3, 'secondary'), + versionedManifestEntry(4, 'relationship'), + { + ...manifestCommon, + revision_key: `${markingDefinitionId}::unversioned`, + kind: 'supporting', + object_ref: markingDefinitionId, + }, + versionedManifestEntry(5, 'link_target'), + ]); await dynamicRepo.saveSnapshot(standardTrack.id, { ...snapshotBase(standardTrack), modified: standardLatestModified, @@ -175,12 +208,22 @@ describe('GET /api/release-tracks/:id/snapshots', function () { candidates_count: 1, }); expect(response.body.data[0]).not.toHaveProperty('quarantine_count'); + expect(response.body.data[0]).not.toHaveProperty('graph_statistics'); expect(response.body.data[1]).toMatchObject({ modified: standardTaggedModified.toISOString(), version: '1.0', + graph_manifest_id: 'release-track-graph-manifest--snapshot-history', members_count: 2, staged_count: 1, candidates_count: 3, + graph_statistics: { + primary_count: 2, + secondary_count: 2, + relationship_count: 1, + supporting_count: 1, + link_target_count: 1, + total_count: 7, + }, }); }); diff --git a/app/tests/api/release-tracks/snapshot-immutability.spec.js b/app/tests/api/release-tracks/snapshot-immutability.spec.js index 3bd00ee5..19310faa 100644 --- a/app/tests/api/release-tracks/snapshot-immutability.spec.js +++ b/app/tests/api/release-tracks/snapshot-immutability.spec.js @@ -51,7 +51,7 @@ describe('Release-track snapshot immutability contract', function () { await api('post', `/api/release-tracks/${track.id}/snapshots/${modified}/contents`, {}, 404); }); - it('deletes only the latest untagged draft', async function () { + it('keeps one rolling standard draft and deletes it only when a tagged predecessor exists', async function () { const initial = await post( '/api/release-tracks/new', { name: 'Latest draft deletion boundary', type: 'standard' }, @@ -64,21 +64,37 @@ describe('Release-track snapshot immutability contract', function () { description: 'Latest draft', }); - const historicalDelete = await api( + await api( 'delete', `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(initial.modified)}`, undefined, + 404, + ); + await api( + 'get', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(middle.modified)}`, + undefined, + 404, + ); + + const onlyDraftDelete = await api( + 'delete', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(latest.modified)}`, + undefined, 409, ); - expect(historicalDelete.body).toEqual({ - message: 'Only the latest untagged snapshot can be deleted', - snapshot_modified: initial.modified, - latest_snapshot_modified: latest.modified, + expect(onlyDraftDelete.body.message).toContain('only snapshot'); + + const tagged = await post(`/api/release-tracks/${initial.id}/snapshots/latest/release`, { + version: '1.0', + }); + const replacement = await post(`/api/release-tracks/${initial.id}/meta`, { + description: 'Post-release rolling draft', }); await api( 'delete', - `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(latest.modified)}`, + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(replacement.modified)}`, undefined, 204, ); @@ -88,14 +104,11 @@ describe('Release-track snapshot immutability contract', function () { undefined, 200, ); - expect(reverted.body.modified).toBe(middle.modified); + expect(reverted.body.modified).toBe(tagged.modified); - await post(`/api/release-tracks/${initial.id}/snapshots/latest/release`, { - version: '1.0', - }); const taggedDelete = await api( 'delete', - `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(middle.modified)}`, + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(tagged.modified)}`, undefined, 409, ); diff --git a/app/tests/api/release-tracks/tagged-content-immutability.spec.js b/app/tests/api/release-tracks/tagged-content-immutability.spec.js index 3118c9ab..69810775 100644 --- a/app/tests/api/release-tracks/tagged-content-immutability.spec.js +++ b/app/tests/api/release-tracks/tagged-content-immutability.spec.js @@ -116,15 +116,7 @@ describe('Release-track authoritative tagged-content immutability', function () updated, 409, ); - expect(putResponse.body.release_tracks).toEqual([track.id]); - expect(putResponse.body.tagged_releases).toEqual([ - expect.objectContaining({ - track_id: track.id, - version: '1.0', - object_ref: technique.stix.id, - object_modified: technique.stix.modified, - }), - ]); + expect(putResponse.body.message).toMatch(/Persisted STIX revisions are immutable/); await api( 'delete', diff --git a/app/tests/api/release-tracks/virtual-graph-integrity.spec.js b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js index 025bf2cc..7c1e1c26 100644 --- a/app/tests/api/release-tracks/virtual-graph-integrity.spec.js +++ b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js @@ -10,9 +10,6 @@ const login = require('../../shared/login'); const AttackObject = require('../../../models/attack-object-model'); const linkById = require('../../../lib/linkById'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); -const { - ReleaseTrackGraphManifestEntry, -} = require('../../../models/release-tracks/release-track-graph-manifest-model'); const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -174,47 +171,62 @@ describe('Virtual release-track graph integrity', function () { expect(exportedRoot.description).toBe(`See [Active Link Target](${attackReference.url}).`); }); - it('reuses the materialized graph for virtual release preview and commit', async function () { + it('keeps virtual drafts and releases graphless until a tagged snapshot opts in', async function () { const root = await post('/api/techniques', technique('Frozen Virtual Root')); const virtual = await createVirtual('Virtual Frozen Release Graph', [root]); const draft = await dynamicRepo.getLatestSnapshot(virtual.id); - const frozenName = 'Canonical Source-Bundle Name'; - await ReleaseTrackGraphManifestEntry.updateOne( - { - manifest_id: draft.graph_manifest_id, - kind: 'root', - object_ref: root.stix.id, - }, - { - $set: { - frozen_stix: { - ...root.stix, - name: frozenName, - }, - }, - }, - ).exec(); + expect(draft.graph_manifest_id).toBeUndefined(); + await post( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(draft.modified)}/graph`, + {}, + 409, + ); const preview = await get( `/api/release-tracks/${virtual.id}/snapshots/latest/release/preview` + '?format=bundle&version=1.0', ); - expect(preview.objects.find((object) => object.id === root.stix.id).name).toBe(frozenName); + expect(preview.objects.find((object) => object.id === root.stix.id).name).toBe(root.stix.name); - await post( + const releasedResponse = await post( `/api/release-tracks/${virtual.id}/snapshots/latest/release`, { version: '1.0' }, 200, ); - const released = await dynamicRepo.getLatestSnapshot(virtual.id); + expect(releasedResponse.graph_manifest_id).toBeUndefined(); + + const deterministic = await post( + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent( + releasedResponse.modified, + )}/graph`, + {}, + 201, + ); + expect(deterministic.graph_manifest_id).toBeDefined(); + const releasedBundle = await get( `/api/release-tracks/${virtual.id}/snapshots/latest?format=bundle`, ); - - expect(released.graph_manifest_id).toBe(draft.graph_manifest_id); expect(releasedBundle.objects.find((object) => object.id === root.stix.id).name).toBe( - frozenName, + root.stix.name, ); + + const graphPath = `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent( + releasedResponse.modified, + )}/graph`; + await request(app) + .delete(graphPath) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(204); + await request(app) + .delete(graphPath) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(204); + + const graphlessRelease = await dynamicRepo.getLatestSnapshot(virtual.id); + expect(graphlessRelease.graph_manifest_id).toBeUndefined(); }); after(async function () { diff --git a/app/tests/api/software/software.spec.js b/app/tests/api/software/software.spec.js index c6552cae..2b5a9b88 100644 --- a/app/tests/api/software/software.spec.js +++ b/app/tests/api/software/software.spec.js @@ -212,22 +212,18 @@ describe('Software API', function () { ); }); - it('PUT /api/software updates a software', async function () { - software1.stix.description = 'This is an updated software.'; - const body = software1; + it('PUT /api/software rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(software1); + body.stix.description = 'This is an updated software.'; const res = await request(app) .put('/api/software/' + software1.stix.id + '/modified/' + software1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated software - const software = res.body; - expect(software).toBeDefined(); - expect(software.stix.id).toBe(software1.stix.id); - expect(software.stix.modified).toBe(software1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/software does not create a software with the same id and modified date', async function () { diff --git a/app/tests/api/tactics/tactics.spec.js b/app/tests/api/tactics/tactics.spec.js index c325a972..54f52d3f 100644 --- a/app/tests/api/tactics/tactics.spec.js +++ b/app/tests/api/tactics/tactics.spec.js @@ -159,22 +159,18 @@ describe('Tactics API', function () { expect(tactic.stix.x_mitre_deprecated).toBe(false); }); - it('PUT /api/tactics updates a tactic', async function () { - tactic1.stix.description = 'This is an updated tactic.'; - const body = tactic1; + it('PUT /api/tactics rejects STIX changes to a persisted revision', async function () { + const body = structuredClone(tactic1); + body.stix.description = 'This is an updated tactic.'; const res = await request(app) .put('/api/tactics/' + tactic1.stix.id + '/modified/' + tactic1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated tactic - const tactic = res.body; - expect(tactic).toBeDefined(); - expect(tactic.stix.id).toBe(tactic1.stix.id); - expect(tactic.stix.modified).toBe(tactic1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/tactics does not create a tactic with the same id and modified date', async function () { diff --git a/app/tests/api/techniques/techniques.convert.spec.js b/app/tests/api/techniques/techniques.convert.spec.js index f900b696..c9dc36d3 100644 --- a/app/tests/api/techniques/techniques.convert.spec.js +++ b/app/tests/api/techniques/techniques.convert.spec.js @@ -462,7 +462,7 @@ describe('Techniques Convert API', function () { expect(technique.stix.x_mitre_is_subtechnique).toBe(false); }); - it('update ignores attempt to change x_mitre_is_subtechnique', async function () { + it('rejects another STIX edit when an attempted subtechnique change is stripped', async function () { const updateBody = { ...technique, stix: { @@ -477,12 +477,9 @@ describe('Techniques Convert API', function () { .send(updateBody) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200); + .expect(409); - // The field should remain false - expect(res.body.stix.x_mitre_is_subtechnique).toBe(false); - // But the description should have been updated - expect(res.body.stix.description).toBe('Updated description'); + expect(res.body.message).toContain('immutable'); }); }); diff --git a/app/tests/api/techniques/techniques.revoke.spec.js b/app/tests/api/techniques/techniques.revoke.spec.js index 54cb4859..582d69d0 100644 --- a/app/tests/api/techniques/techniques.revoke.spec.js +++ b/app/tests/api/techniques/techniques.revoke.spec.js @@ -293,7 +293,7 @@ describe('Techniques Revoke API', function () { expect(res.body.stix.revoked).not.toBe(true); }); - it('PUT /api/techniques strips revoked from update requests', async function () { + it('PUT /api/techniques rejects STIX edits even when revoked is stripped', async function () { const updateData = cloneForCreate(techniqueB); updateData.stix.revoked = true; updateData.stix.description = 'Trying to sneak in revoked via update.'; @@ -303,12 +303,10 @@ describe('Techniques Revoke API', function () { .send(updateData) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // The revoked flag should have been stripped, description updated - expect(res.body.stix.revoked).not.toBe(true); - expect(res.body.stix.description).toBe('Trying to sneak in revoked via update.'); + expect(res.body.message).toContain('immutable'); }); it('POST /api/techniques/:stixId/revoke with preserveRelationships transfers relationships', async function () { diff --git a/app/tests/api/techniques/techniques.spec.js b/app/tests/api/techniques/techniques.spec.js index 35bd9023..c4908369 100644 --- a/app/tests/api/techniques/techniques.spec.js +++ b/app/tests/api/techniques/techniques.spec.js @@ -199,22 +199,18 @@ describe('Techniques Basic API', function () { expect(technique.created_by_user_account).toBeDefined(); }); - it('PUT /api/techniques updates a technique', async function () { - technique1.stix.description = 'This is an updated technique.'; + it('PUT /api/techniques rejects STIX changes to a persisted revision', async function () { const body = cloneForCreate(technique1); + body.stix.description = 'This is an updated technique.'; const res = await request(app) .put('/api/techniques/' + technique1.stix.id + '/modified/' + technique1.stix.modified) .send(body) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) + .expect(409) .expect('Content-Type', /json/); - // We expect to get the updated technique - const technique = res.body; - expect(technique).toBeDefined(); - expect(technique.stix.id).toBe(technique1.stix.id); - expect(technique.stix.modified).toBe(technique1.stix.modified); + expect(res.body.message).toContain('immutable'); }); it('POST /api/techniques does not create a technique with the same id and modified date', async function () { diff --git a/app/tests/middleware/adm-validation-middleware.spec.js b/app/tests/middleware/adm-validation-middleware.spec.js index f2fbe5a0..9a880675 100644 --- a/app/tests/middleware/adm-validation-middleware.spec.js +++ b/app/tests/middleware/adm-validation-middleware.spec.js @@ -17,13 +17,14 @@ const { cloneForCreate } = require('../shared/clone-for-create'); * Smoke tests for ATT&CK Data Model (ADM) validation middleware. * * These tests verify that the ADM validation middleware correctly validates - * POST and PUT requests using the Zod-based schemas from the ADM library. + * POST and metadata-only PUT requests using the Zod-based schemas from the ADM library. * * Test Coverage: * - POST operations with work-in-progress workflow state (partial validation) * - POST operations with reviewed workflow state (full validation) - * - PUT operations with work-in-progress workflow state (partial validation) - * - PUT operations with reviewed workflow state (full validation) + * - Metadata-only PUT operations with work-in-progress workflow state (partial validation) + * - Metadata-only PUT operations with reviewed workflow state (full validation) + * - STIX-changing PUT operations rejected before validation * - True positives: valid data should pass * - True negatives: invalid data should fail with proper errors * - Validation toggle (enabled/disabled) @@ -360,7 +361,7 @@ describe('ADM Validation Middleware', function () { }); }); - describe('PUT operations - work-in-progress (partial validation)', function () { + describe('metadata-only PUT operations - work-in-progress (partial validation)', function () { let createdObject; beforeEach(async function () { @@ -390,69 +391,42 @@ describe('ADM Validation Middleware', function () { createdObject = createRes.body; }); - it('should accept valid updates in work-in-progress state', async function () { + it('should accept valid workspace updates in work-in-progress state', async function () { let updateBody = { - type: 'attack-pattern', - status: 'work-in-progress', workspace: { workflow: { state: 'work-in-progress', }, }, - stix: { - ...createdObject.stix, - name: 'Updated Technique Name', - description: 'Updated description', - }, + stix: createdObject.stix, }; updateBody = cloneForCreate(updateBody); // Remove server-managed field (server adds this automatically) delete updateBody.stix.x_mitre_attack_spec_version; - // Note: We keep id, created, modified because ADM schemas validate the full STIX structure - const res = await request(app) .put(`${endpoint}/${createdObject.stix.id}/modified/${createdObject.stix.modified}`) .send(updateBody) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); - if (res.status !== 200) { - logger.debug('=== REQUEST FAILED ==='); - logger.debug('Status:', res.status); - logger.debug('Errors:', JSON.stringify(res.body, null, 2)); - } - expect(res.status).toBe(200); - expect(res.body.stix.name).toBe('Updated Technique Name'); + expect(res.body.stix.name).toBe(createdObject.stix.name); }); - it('should accept updates with missing optional fields in work-in-progress state', async function () { + it('should accept workspace-only bodies without optional wrapper fields', async function () { let updateBody = { - type: 'attack-pattern', - status: 'work-in-progress', workspace: { workflow: { state: 'work-in-progress', }, }, - stix: { - ...createdObject.stix, - name: 'Updated Name', - }, + stix: createdObject.stix, }; updateBody = cloneForCreate(updateBody); - // Remove optional fields to test partial validation - delete updateBody.stix.description; - delete updateBody.stix.x_mitre_platforms; - - // Remove server-managed field - delete updateBody.stix.x_mitre_attack_spec_version; - // Note: We keep id, created, modified because ADM schemas validate the full STIX structure - const res = await request(app) .put(`${endpoint}/${createdObject.stix.id}/modified/${createdObject.stix.modified}`) .send(updateBody) @@ -462,7 +436,7 @@ describe('ADM Validation Middleware', function () { expect(res.status).toBe(200); }); - it('should reject updates with invalid field values in work-in-progress state', async function () { + it('should reject STIX changes before ADM validation', async function () { const updateBody = { workspace: { workflow: { @@ -471,26 +445,24 @@ describe('ADM Validation Middleware', function () { }, stix: { ...createdObject.stix, - description: true, // <-- should trigger validation error (should be string) + description: true, }, }; // Remove server-managed field delete updateBody.stix.x_mitre_attack_spec_version; - // Note: We keep id, created, modified because ADM schemas validate the full STIX structure - const res = await request(app) .put(`${endpoint}/${createdObject.stix.id}/modified/${createdObject.stix.modified}`) .send(updateBody) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); - expect(res.status).toBe(400); - expect(res.body.message).toBeDefined(); + expect(res.status).toBe(409); + expect(res.body.message).toContain('immutable'); }); }); - describe('PUT operations - reviewed (full validation)', function () { + describe('metadata-only PUT operations - reviewed (full validation)', function () { let createdObject; beforeEach(async function () { @@ -518,17 +490,14 @@ describe('ADM Validation Middleware', function () { createdObject = createRes.body; }); - it('should accept valid complete updates in reviewed state', async function () { + it('should accept a reviewed workflow transition for valid complete STIX', async function () { let updateBody = { workspace: { workflow: { state: 'reviewed', }, }, - stix: { - ...createdObject.stix, - name: 'Reviewed Technique Name', - }, + stix: createdObject.stix, }; updateBody = cloneForCreate(updateBody); @@ -544,35 +513,47 @@ describe('ADM Validation Middleware', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); expect(res.status).toBe(200); - expect(res.body.stix.name).toBe('Reviewed Technique Name'); + expect(res.body.stix.name).toBe(createdObject.stix.name); + expect(res.body.workspace.workflow.state).toBe('reviewed'); }); - it('should reject updates missing required fields in reviewed state', async function () { - const updateBody = { + it('should reject a reviewed transition when persisted STIX is incomplete', async function () { + const partialStix = createSyntheticStix(stixType); + // Domains are required by the full ATT&CK technique schema but remain + // optional in the Mongoose document shape and the WIP partial schema. + delete partialStix.x_mitre_domains; + let partialCreateBody = { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: partialStix, + }; + partialCreateBody = cloneForCreate(partialCreateBody); + const partialCreateRes = await request(app) + .post(endpoint) + .send(partialCreateBody) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(201); + + let updateBody = { workspace: { workflow: { state: 'reviewed', }, }, - stix: { - ...createdObject.stix, - }, + stix: partialCreateRes.body.stix, }; - - // Remove required field - delete updateBody.stix.name; - // Remove server-managed field - delete updateBody.stix.x_mitre_attack_spec_version; - // Note: We keep id, created, modified because ADM schemas validate the full STIX structure + updateBody = cloneForCreate(updateBody); const res = await request(app) - .put(`${endpoint}/${createdObject.stix.id}/modified/${createdObject.stix.modified}`) + .put( + `${endpoint}/${partialCreateRes.body.stix.id}/modified/${partialCreateRes.body.stix.modified}`, + ) .send(updateBody) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); expect(res.status).toBe(400); - expect(res.body.message).toBeDefined(); + expect(res.body.message).toBe('ADM validation failed'); }); }); @@ -775,7 +756,7 @@ describe('ADM Validation Middleware', function () { expect(res.body.message).toBe('ADM validation failed'); }); - it('should return composed object without persisting on PUT with dryRun=true', async function () { + it('should return workspace metadata without persisting on PUT with dryRun=true', async function () { // First, create an object to update const syntheticStix = createSyntheticStix(stixType); @@ -799,17 +780,14 @@ describe('ADM Validation Middleware', function () { const createdObject = createRes.body; - // Now do a dry-run update + // Now do a dry-run metadata update let updateBody = { workspace: { workflow: { - state: 'work-in-progress', + state: 'awaiting-review', }, }, - stix: { - ...createdObject.stix, - name: 'Dry Run Updated Name', - }, + stix: createdObject.stix, }; updateBody = cloneForCreate(updateBody); @@ -824,7 +802,8 @@ describe('ADM Validation Middleware', function () { expect(res.status).toBe(200); expect(res.body.stix).toBeDefined(); - expect(res.body.stix.name).toBe('Dry Run Updated Name'); + expect(res.body.stix.name).toBe(createdObject.stix.name); + expect(res.body.workspace.workflow.state).toBe('awaiting-review'); // Mongoose internals should not be exposed expect(res.body._id).toBeUndefined(); expect(res.body.__v).toBeUndefined(); @@ -837,8 +816,7 @@ describe('ADM Validation Middleware', function () { .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); expect(getRes.status).toBe(200); - // Original name should be unchanged - expect(getRes.body.stix.name).not.toBe('Dry Run Updated Name'); + expect(getRes.body.workspace.workflow.state).toBe('work-in-progress'); }); }); diff --git a/docs/developer/FRONTEND_TODO.md b/docs/developer/FRONTEND_TODO.md index e891c5e3..d78a481c 100644 --- a/docs/developer/FRONTEND_TODO.md +++ b/docs/developer/FRONTEND_TODO.md @@ -155,14 +155,13 @@ Done when: - Tests cover multiple missing references and prove no partial snapshot or bundle is rendered. -### [ ] Explain snapshot-graph protection conflicts on object edits and deletes +### [ ] Explain immutable-revision and snapshot-graph deletion conflicts -Release-track snapshots now freeze the exact relationships and secondary -objects needed to reproduce their bundle graph. If an object revision is a -protected dependency of any active or linked-pending snapshot manifest, an -in-place `PUT`, exact-revision `DELETE`, or full-lineage `DELETE` that would -invalidate that graph returns -`409 Conflict`: +Persisted STIX revisions are globally immutable. Any STIX-changing PUT returns +`409 Conflict` and should direct the operator to POST a new revision. Separately, +a tagged snapshot's opt-in graph manifest holds exact revision pointers. If an +object revision is a protected dependency of an active or linked-pending +manifest, exact-revision or full-lineage DELETE returns `409 Conflict`: ```ts { @@ -183,12 +182,11 @@ create a new object revision, or remove the draft snapshots that no longer need the old revision. Do not offer a force-delete path; administrator authorization does not bypass graph integrity. -A standalone standard-track candidate or staged root remains editable through -the existing in-place review workflow. It becomes graph-protected only when -the same revision is also needed as a frozen dependency. Description-only -relationship corrections are allowed because older snapshots retain the -relationship payload captured in their manifests; relationship source, -target, and type changes are rejected as graph changes. +Candidate and staged workspace metadata remains editable, but STIX content does +not. Relationship corrections are always new POST revisions; schema-v2 graph +manifests point to the exact older relationship revision and never rely on a +frozen clone. Marking definitions remain the narrow frozen-payload exception +because they are unversioned. Done when: @@ -1001,19 +999,46 @@ Minimum regression coverage: - The frontend never offers standard-only contents/candidate/staged mutations on a virtual track. +## Deterministic bundle cache controls + +`GET /api/release-tracks/:id/snapshots` includes the opaque +`graph_manifest_id` on summaries whose tagged snapshot has a materialized +member graph. The History tab should translate that technical state into a +user-oriented bundle cache: + +- Show **Bundle cached** with a success indicator when `graph_manifest_id` is + present. +- Show **Not cached** with a warning icon otherwise. Explain that member-only + bundle exports are not guaranteed to be deterministic until cached. +- Offer **Cache bundle** only for uncached tagged snapshots and editors. It + calls `POST /api/release-tracks/:id/snapshots/:modified/graph`. +- Use the existing indeterminate Material spinner while a cache operation is + in progress. Cached snapshots offer editors a confirmed **Delete cache** + action backed by `DELETE /api/release-tracks/:id/snapshots/:modified/graph`. +- For cached snapshots, render the accompanying `graph_statistics` in a + compact **Graph cache** panel: Primary, Secondary, Relationships, and + Dependencies. Dependencies is the sum of supporting and LinkById entries; + show `total_count` as the overall cached-item count. +- Refresh snapshot history after cache creation or deletion so server-derived + state and statistics are visible immediately. +- Drafts remain uncached and must be tagged first. Candidate and staged + exports remain live even when the member graph is cached. +- Treat `graph_manifest_id` as an opaque read-only signal; never send or + persist a client-authored value. + +“Cache” is deliberately presentation language, not an implementation claim +about HTTP response caching. Tooltips should retain the member-only and +determinism qualifiers so users do not infer broader guarantees. + ## Backend changes that do not require Angular API changes The following changes are useful context but should not create extra connector work: -- Snapshot bundle exports now include bounded secondary objects and their - relationships from a frozen graph manifest. Existing bundle download code - receives a more complete and reproducible bundle without changing its - request. A standard draft tier explicitly stored as `"latest"` remains - dynamic until release. -- Snapshot responses include an opaque, server-controlled - `graph_manifest_id`. The SPA does not need to send, interpret, or persist - this field; tolerate it in response models and omit it from request bodies. +- Snapshot bundle exports include bounded secondary objects and their + relationships. Materialized schema-v2 graphs retain exact revision pointers + while graphless exports resolve live. A standard draft tier explicitly + stored as `"latest"` remains dynamic until release. - Release-track object back-references are reconciled when snapshots change. Frontend object refreshes will see the updated membership metadata without a new endpoint. diff --git a/docs/developer/data-model.md b/docs/developer/data-model.md index 29768fef..8eab910d 100644 --- a/docs/developer/data-model.md +++ b/docs/developer/data-model.md @@ -22,7 +22,12 @@ The ATT&CK Workbench database supports the following ATT&CK object types (with t ## Object Versioning and Updates -Most ATT&CK object types should be updated by creating a new object with a new `modified` timestamp (POST request). The Collection Index is different and should be updated by modifying (overwriting) the current object (PUT request). +Persisted STIX revisions are immutable. Change STIX content by creating a new +revision with the same `stix.id` and a newer `stix.modified` timestamp through +POST. PUT on a versioned STIX endpoint is limited to non-exported `workspace` +metadata and returns 409 if the submitted `stix` payload differs from the +stored revision. The Collection Index is not a versioned STIX document and +continues to use overwrite-style PUT. ## Canonical Domain Membership diff --git a/docs/developer/event-bus-architecture.md b/docs/developer/event-bus-architecture.md index 8112ce2f..34c21b91 100644 --- a/docs/developer/event-bus-architecture.md +++ b/docs/developer/event-bus-architecture.md @@ -16,7 +16,7 @@ beforeX → X → afterX → emitXEvent For example: - `beforeCreate` → `create` → `afterCreate` → `emitCreatedEvent` -- `beforeUpdate` → `update` → `afterUpdate` → `emitUpdatedEvent` +- `beforeUpdate` → immutable-STIX check → metadata update → `afterUpdate` - `beforeDelete` → `delete` → `afterDelete` → `emitDeletedEvent` **Execution Order:** @@ -172,7 +172,7 @@ Where: | Event | When Emitted | Payload | Use Cases | |-------|--------------|---------|-----------| | `{type}::created` | After `afterCreate` hook | `{ stixId, document, type, options }` | Audit logging, notifications | -| `{type}::updated` | After `afterUpdate` hook | `{ stixId, stixModified, document, previousDocument, type }` | Track changes, propagate updates | +| `{type}::updated` | Legacy/custom service update paths only | `{ stixId, stixModified, document, previousDocument, type }` | Propagate a service-defined STIX update; generic metadata-only PUT does not emit this event | | `{type}::deleted` | After `afterDelete` hook | `{ stixId, document, options }` | Cleanup, cascade deletes | Where `{type}` is the STIX type (e.g., `attack-pattern`, `x-mitre-analytic`, `x-mitre-detection-strategy`). @@ -234,45 +234,47 @@ Where `{type}` is the STIX type (e.g., `attack-pattern`, `x-mitre-analytic`, `x- - Update analytic's `external_references` with URL: `https://attack.mitre.org/detectionstrategies/DS0001#DA-0001` - Save the analytic -### Workflow 2: Update Detection Strategy - Add Analytic +### Workflow 2: Revise Detection Strategy - Add Analytic -**User Action:** `PUT /api/detection-strategies/{id}/{modified}` -- Change `x_mitre_analytic_refs` from `[]` to `['x-mitre-analytic--123']` +**User Action:** `POST /api/detection-strategies` +- Create a later revision whose `x_mitre_analytic_refs` changes from `[]` to + `['x-mitre-analytic--123']` **Execution Flow:** -1. **DetectionStrategiesService.beforeUpdate(stixId, stixModified, data, existingDocument)** +1. **DetectionStrategiesService.beforeCreate(data)** - Detect change: `oldRefs = []`, `newRefs = ['x-mitre-analytic--123']` - Store: `this._addedAnalyticRefs = ['x-mitre-analytic--123']` - Rebuild outbound embedded_relationships for new refs - Update `data.workspace.embedded_relationships` -2. **BaseService.updateFull()** - Persist document to database +2. **BaseService.create()** - Persist the new revision -3. **DetectionStrategiesService.afterUpdate(updatedDocument, previousDocument)** +3. **DetectionStrategiesService.afterCreate(createdDocument)** - If `_addedAnalyticRefs` not empty: - Emit `x-mitre-detection-strategy::analytics-referenced` - Clean up: `delete this._addedAnalyticRefs` -4. **BaseService.emitUpdatedEvent()** - Emit `x-mitre-detection-strategy::updated` +4. **BaseService.emitCreatedEvent()** - Emit `x-mitre-detection-strategy::created` 5. **AnalyticsService** listener receives event and updates analytics -### Workflow 3: Update Detection Strategy - Remove Analytic +### Workflow 3: Revise Detection Strategy - Remove Analytic -**User Action:** `PUT /api/detection-strategies/{id}/{modified}` -- Change `x_mitre_analytic_refs` from `['x-mitre-analytic--123']` to `[]` +**User Action:** `POST /api/detection-strategies` +- Create a later revision whose `x_mitre_analytic_refs` changes from + `['x-mitre-analytic--123']` to `[]` **Execution Flow:** -1. **DetectionStrategiesService.beforeUpdate(...)** +1. **DetectionStrategiesService.beforeCreate(...)** - Detect change: `removedRefs = ['x-mitre-analytic--123']` - Store: `this._removedAnalyticRefs = ['x-mitre-analytic--123']` - Rebuild outbound embedded_relationships (now empty) -2. **BaseService.updateFull()** - Persist document +2. **BaseService.create()** - Persist the new revision -3. **DetectionStrategiesService.afterUpdate(...)** +3. **DetectionStrategiesService.afterCreate(...)** - If `_removedAnalyticRefs` not empty: - Emit `x-mitre-detection-strategy::analytics-removed` ```javascript diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index d5a98790..a30b0ce7 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -125,10 +125,11 @@ into a *new* revision must not carry the field forward; this is handled in: (Clones routed through `create()` — e.g. the relationship *transfer* during revoke — are already covered by `stripServerControlledFields`.) -Relatedly, revision identity is immutable in place: `BaseService.updateFull` -rejects (400) a PUT whose body `stix.id`/`stix.modified` differ from the path -parameters, so a pinned revision can never be re-keyed out from under a -track's pin (which would strand the pin and orphan the backref). +Relatedly, persisted STIX revisions are immutable. `BaseService.updateFull` +rejects any PUT that changes `stix` content with `409`; corrections use POST to +create a new revision. A PUT may still update non-exported `workspace` +metadata. This global rule prevents a revision from being re-keyed or changed +out from under a track pointer. New revisions created through `create()` are covered by the strip; if any track references the object (members, candidates, or staged), member sync @@ -145,11 +146,12 @@ backref to the newly latest revision without rewriting the stored selector revision is later re-created, its backref is restored on the next contents-changed event for that track, not immediately. - **Historical snapshots.** Backrefs describe only the *latest* snapshot per - track. Object mutation guards do not trust that derived view: they query - every registered track's tagged snapshots for the exact revision before an - in-place update or delete. Historical tagged membership therefore remains - immutable even after the latest draft removes the object or the registry's - tagged-release catalogue is stale. + track. Delete guards do not trust that derived view: they query every + registered track's tagged snapshots and graph manifests for the exact + revision. Historical tagged membership and deterministic graph pointers + therefore remain valid even after the latest draft removes the object or the + registry's tagged-release catalogue is stale. STIX-changing PUTs are already + rejected globally. - **Crash window before record creation.** Snapshot persistence and the central reconciliation record are not in one MongoDB transaction. A hard process failure in that narrow interval can leave no pending record. diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index e0cde744..612d92c9 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -102,19 +102,20 @@ STIX version serialization. The pipeline: filter, mirroring the fact that members are inherently reviewed. `state` never affects members. `reviewed` is intentionally not a valid `state` value for this reason. -2. **Manifest replay** — the snapshot identifies an active graph manifest, or - a complete linked pending manifest recovering from an interrupted - activation, created at the same persistence boundary. The manifest records exact - primary, relationship, secondary, supporting, and LinkById dependency - revisions. Export hydrates those entries and performs no live graph - expansion. -3. **Bounded secondary selection** — replay starts from the requested primary - tiers, follows only dependency edges frozen in the manifest, and emits a - relationship only when both exact endpoint revisions are selected. -4. **Supporting objects** — only identities and marking definitions frozen in - the manifest and referenced by the selected graph are appended. -5. **LinkById conversion** — conversion uses only exact render targets frozen - in the manifest and never falls back to a current database lookup. +2. **Graph selection** — a member-only export replays the schema-v2 graph when + the tagged snapshot has explicitly opted in. Graphless snapshots resolve a + live bounded graph. Any request that includes `staged` or `candidates` is + also live; determinism is promised for `members` only. +3. **Bounded secondary selection** — graph resolution starts from the selected + roots and emits a relationship only when both exact endpoint revisions are + selected. Persisted schema-v2 manifests store exact-revision pointers, not + cloned STIX payloads. +4. **Supporting objects** — referenced identities and marking definitions are + appended. Versioned supporting objects use pointers; unversioned marking + definitions retain a frozen payload in persisted graphs. +5. **LinkById conversion** — deterministic replay uses the exact render target + pointer captured in the graph. Live resolution uses the current eligible + target. 6. **Assembly** (Zod transform) — notes are dropped, objects are conformed to `stixVersion` via the shared `lib/stix-conformance.js` helpers, and the bundle envelope is emitted (with `spec_version: "2.0"` only when @@ -152,8 +153,8 @@ snapshot export, and ephemeral export observe the same membership. Because snapshot contents are explicitly curated, primary entries do **not** receive the legacy attack-id / deprecated / revoked filters. Secondary graph -capture retains the established bounded ATT&CK expansion rules and freezes -the resulting graph at snapshot creation. +resolution retains the established bounded ATT&CK expansion rules. It is +frozen only when a tagged snapshot opts into a graph. #### Relationship and secondary-object consistency boundary @@ -164,7 +165,7 @@ Release-track snapshots distinguish **primary** and **secondary** content: and staged entries may instead store `"latest"` and are resolved just in time when a draft export includes those tiers. - Secondary objects are not snapshot members. They are discovered when the - snapshot is created because an exact-pinned SRO connects them to a primary, + graph is resolved because an exact-pinned SRO connects them to a primary, the bounded ATT&CK rules identify a detection strategy, or the bundle needs a supporting identity, marking definition, or LinkById render target. @@ -185,43 +186,43 @@ SRO. They are not emitted because bundle output includes only the `stix` object. When an endpoint advances, Workbench creates a new SRO revision with updated pins rather than rewriting the older SRO. -Each persisted snapshot references a tier-aware manifest. A pending manifest -and all of its entries are written before the snapshot is linked to it, then -activated after persistence succeeds. The snapshot link is the durable commit -record: replay can use and self-activate a complete linked pending manifest -after a process interruption. -A standard release replaces the draft manifest with one built from the -resolved release plan, so dynamic staged selectors become exact members. -Materialized virtual snapshots contain exact roots from the outset. Releasing -a virtual draft does not change those roots, so bundle preview and commit -reuse its existing manifest. This makes the preview the literal graph that -will be tagged rather than a second resolution against newer database state. - -Active and pending manifests protect their exact dependencies. In-place -updates and hard deletes that would invalidate a primary or secondary -revision return `409`; lineage deletion is rejected when any version is -protected. Relationship source, target, and type changes are rejected. -Description-only relationship corrections remain allowed because the -relationship STIX payload used by older snapshots is frozen in the manifest. -Manifest entries may also freeze complete source payloads for an audited -operational baseline. The exact database revision pin remains mandatory and -protected; the frozen payload preserves the reviewed publication -representation for deterministic replay. -Deleting a draft snapshot or track removes its manifest and releases -protection that no other snapshot needs. +Snapshots are graphless by default. After tagging, an editor may call +`POST /api/release-tracks/:id/snapshots/:modified/graph`. The service builds a +schema-v2 member graph, writes a pending manifest and decoupled entry rows, +rehydrates every pointer while those pending rows already protect deletion, +then atomically attaches the manifest ID to the still-tagged snapshot. Replay +can self-activate a complete linked pending manifest after an interrupted +activation. `DELETE` on the same graph resource detaches and removes it. + + +Graph creation uses an indexed relationship frontier rather than scanning all +relationships. It starts with member IDs, queries only current relationship +lineages touching the frontier, batch-hydrates exact endpoints by STIX type, +and repeats only when bounded resolution discovers another relevant object +ID. This retains secondary-to-secondary edges without rebuilding unrelated +database state. Incremental reuse from a previous snapshot is deliberately +deferred: an unchanged member set does not prove an unchanged graph because a +new relationship can connect to an old member. + +Active and pending manifests protect every exact versioned dependency from +hard deletion. Persisted STIX content is globally immutable through PUT, +whether or not it is graph-pinned; corrections are new POSTed revisions. +Schema-v2 relationships therefore need no frozen payload or mutation +exemption. Legacy schema-v1 manifests still replay their frozen relationship +payloads. Deleting a graph or track releases protection that no other graph or +tagged membership needs. Existing data is upgraded by an idempotent migration. Only the latest revision of each legacy relationship can be endpoint-pinned truthfully. Pre-existing snapshot manifests are labeled `baseline_reconstruction` because they describe the graph visible during migration rather than an -unknowable historical graph. - -The deliberate exception is a standard draft export that explicitly includes -a candidate or staged entry stored as `"latest"`. That selector is defined to -move until release, so the selected draft graph is resolved for that request. -Release preview and commit resolve it again; a successful commit stores an -exact manifest. Members, tagged releases, materialized virtual snapshots, and -exact-selector draft tiers replay deterministically. +unknowable historical graph. They must not be represented as historical truth. + +Drafts and tagged snapshots without graphs resolve live. Candidate/staged +exports also resolve live even when the snapshot has a graph, because those +tiers are expected to move. Release preview is live and release commit does +not create a graph. Determinism begins only with the explicit tagged-snapshot +graph operation and applies only to member exports. The graph and object payload are reproducible, but the bundle is not promised to be byte-for-byte identical: the bundle envelope receives a newly generated diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 13fc10b3..e352cf25 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -445,13 +445,15 @@ copies the exact member revisions from the selected tagged component snapshots, and later component activity cannot change the persisted virtual snapshot. -Each snapshot also references an internal, tier-aware graph manifest. It -freezes the exact relationship endpoint revisions, bounded secondary objects, -supporting objects, and LinkById render targets needed by `format=bundle`. -Tagged standard snapshots, materialized virtual snapshots, and draft tiers -that use exact selectors therefore replay the same graph. A standard draft -tier explicitly stored as `"latest"` remains intentionally dynamic until the -release boundary. +A tagged snapshot may optionally reference an internal schema-v2 member graph +manifest. `POST /api/release-tracks/:id/snapshots/:modified/graph` resolves the +bounded graph from `members` and stores exact-revision pointers for primary, +relationship, secondary, versioned supporting, and LinkById objects. Only +unversioned supporting objects such as marking definitions retain a frozen +payload. Drafts are always graphless. A tagged snapshot without a manifest is +exportable, but graph relationships and secondary objects are resolved live. +Exports that include `candidates` or `staged` are also live even when the +tagged snapshot has a member manifest. The three valid `snapshot_schedule` shapes are: diff --git a/docs/developer/release-tracks/error-handling.md b/docs/developer/release-tracks/error-handling.md index ebbd6cd8..435b07a3 100644 --- a/docs/developer/release-tracks/error-handling.md +++ b/docs/developer/release-tracks/error-handling.md @@ -62,13 +62,15 @@ latest snapshot. The response identifies both `snapshot_modified` and `latest_snapshot_modified`. Refresh the track and continue from the latest -draft; historical drafts cannot be removed. +draft. Standard tracks normally return 404 for a replaced draft because only +their newest untagged snapshot is retained; this exception remains relevant +to retained virtual drafts. ### SnapshotGraphPinnedRevisionError -**Thrown when:** An in-place update or hard delete would change an exact -primary, relationship, secondary, supporting, or LinkById dependency frozen -in a release-track snapshot graph. Full-lineage and collection +**Thrown when:** A hard delete would remove an exact primary, relationship, +secondary, supporting, or LinkById dependency referenced by an opt-in +release-track snapshot graph. Full-lineage and collection `deleteAllContents` operations are preflighted against the same invariant. **HTTP Status:** 409 Conflict @@ -76,9 +78,8 @@ in a release-track snapshot graph. Full-lineage and collection The response includes `snapshot_graph_pins` entries identifying the track, snapshot timestamp, manifest entry kind, and tier where applicable. Create a new STIX revision instead. Administrator authorization is not a force-delete -override. Description-only relationship corrections remain allowed because -the older relationship payload is frozen inside each existing manifest; -source, target, and relationship-type changes return 400. +override. STIX-changing PUTs are rejected globally by +`ImmutableStixRevisionError`; schema-v2 relationships have no exemption. ### NotFoundError diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 3385c7af..678bea53 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -94,10 +94,12 @@ could each clone a stale snapshot and lose one candidate update. Before changing data, the migration resolves the complete candidate set from persisted canonical collection provenance. A latest domainless target object -without recognized provenance defaults to `["enterprise-attack"]` so legacy -or custom content does not block startup. Each fallback is identified as -`domain_source: "enterprise-default"` in its automation item and counted by -the run's `enterprise_defaults` counter. +without recognized provenance is left unchanged; lack of a TOC match cannot +justify Enterprise membership. The run records a bounded warning sample and +an `unmapped_skipped` count. Persisted domain-validation bypasses remain while +any such object exists, so startup can complete without enforcing an +unsatisfied contract. Failures while repairing mapped objects still fail the +migration. The migration deletes database copies of retired `x_mitre_domains` bypass rules only after every target object has been repaired. Removing the rules @@ -229,11 +231,11 @@ a standard component track. Snapshot retrieval never re-runs composition, so there is no `resolve` query parameter or `resolved_content` response wrapper. Workbench retrieval returns -the persisted primary membership. Bundle export replays a graph manifest -captured with the snapshot. Relationship revisions carry server-controlled -exact endpoint pins in `workspace.relationship_endpoints`, and the manifest -freezes the bounded secondary/supporting graph without emitting those internal -fields in STIX output. +the persisted primary membership. Bundle export replays a graph only after a +tagged snapshot explicitly opts in; otherwise it resolves the current bounded +graph. Relationship revisions carry server-controlled exact endpoint pins in +`workspace.relationship_endpoints`, and schema-v2 manifests reference those +exact revisions without emitting the internal fields in STIX output. Snapshot schedules use the same strict, mode-discriminated Zod schema at the controller and service boundaries. `manual` has no selector field, `cron` @@ -361,6 +363,14 @@ ambiguous. An omitted `tagged` parameter adds no version predicate; `tagged=true` matches string versions and `tagged=false` matches null draft versions. +For summaries with `graph_manifest_id`, the snapshot service collects all +manifest IDs from the paginated result and performs one aggregation against +`releaseTrackGraphManifestEntries`, grouped by `manifest_id` and `kind`. The +existing `{ manifest_id: 1, kind: 1, tier: 1 }` index supports the match. The +service fills zero-valued categories for empty graphs and attaches +`graph_statistics` only to cached snapshots. This keeps history latency to one +additional bounded query rather than one query per snapshot. + ## Integrating with the Event-Driven Architecture ### Events Published diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index 023a4f32..ed601a37 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -109,31 +109,21 @@ sync strategy determines what workflow action (if any) to take: > Both workflows save the new revision directly via the repository (no > `::created`/`::updated` fires), so without these subscriptions a track > silently kept exporting the pre-revoke / pre-conversion revision. -> - In-place `PUT`s of a pinned revision arrive as `::updated` with an -> unchanged `(stix.id, modified)` key. The entry is marked with the -> server-assigned **`modified-in-place`** status — the content changed, -> but with no revision history to diff the track can only signal that a -> re-review is required. The marker ranks with `work-in-progress` in the -> candidacy-threshold order and is cleared through the normal review -> endpoint (`from: "modified-in-place"`). > - The gate codifies the candidacy threshold into placement itself: an > entry whose resulting status meets `candidacy_threshold` (with > `auto_promote`) is placed directly in `staged` — one snapshot instead of -> bouncing through candidates and a post-hoc auto-promotion pass. In a -> permissive track (threshold `work-in-progress`) an in-place edit of a -> staged entry therefore keeps its staged tier; in a strict track it -> demotes to candidates. +> bouncing through candidates and a post-hoc auto-promotion pass. > - Repeat no-op changes (entry already in the gate-decided tier/status) and > enrollment of already-pinned revisions (e.g. a re-import announcing an > already-released revision) skip snapshot creation. -> - `members`-pinned revisions never reach the in-place path at all: -> `BaseService` rejects `PUT`/`DELETE` of a members-pinned revision with -> 409 (`MemberPinnedRevisionError`) — released content is immutable in -> place. -> - A candidate or staged revision remains editable unless it is also a -> secondary/supporting dependency frozen in a snapshot graph manifest. In -> that case graph integrity takes precedence and the operation returns 409 -> (`SnapshotGraphPinnedRevisionError`). +> +> **Behavior evolution (2026-08-03):** every persisted STIX revision is now +> immutable, not only released content. A PUT that changes `stix` returns 409; +> authors POST a new revision and member sync handles the resulting `::created` +> event. Metadata-only PUTs may update `workspace` but do not emit a STIX +> update event or create a track snapshot. The legacy `modified-in-place` +> transition remains readable for stored data but is no longer produced by +> the object update path. ### Relationship to Existing Features @@ -141,7 +131,7 @@ Member sync strategies integrate with several existing release track features: - **Candidacy Threshold:** When a new revision is auto-enrolled as a candidate, it may be immediately promoted to `staged` if its status meets the candidacy threshold. - **Conflict Resolution Policies:** Member sync resolves overlaps with existing `candidates`/`staged` entries through its own `supplant` config (below). *Manual* candidate adds and demotions instead go through `config.promotion_conflicts.into_candidates` (default `prefer_latest`) — see `release-workflow.md`. The two are deliberately separate: supplant expresses sync intent (replace/queue/ignore), while `into_candidates` uses the same policy vocabulary as the other tier transitions. -- **Snapshot Creation:** Any change to a release track's object lists (`candidates`, `staged`, `members`) results in a new draft snapshot being created. Member sync follows this convention. +- **Snapshot Creation:** Any change to a release track's object lists (`candidates`, `staged`, `members`) creates a replacement draft snapshot. Standard tracks retain only the newest untagged draft after it is durably saved; tagged snapshots remain historical. Member sync follows this convention. --- diff --git a/docs/developer/stix-versioning-and-embedded-relationships.md b/docs/developer/stix-versioning-and-embedded-relationships.md index 15541ddf..82c9645d 100644 --- a/docs/developer/stix-versioning-and-embedded-relationships.md +++ b/docs/developer/stix-versioning-and-embedded-relationships.md @@ -8,7 +8,9 @@ This document explains how STIX versioning works in the ATT&CK Workbench REST AP ## STIX Versioning: POST vs PUT -The ATT&CK Workbench implements STIX 2.1 versioning semantics with two distinct update mechanisms: +The ATT&CK Workbench implements STIX 2.1 versioning semantics with immutable +persisted revisions. POST creates STIX content; PUT is limited to non-exported +workspace metadata. ### POST - Creating New Versions (Versioned History) @@ -61,68 +63,37 @@ POST /api/data-components --- -### PUT - Editing Existing Snapshots (In-Place Modification) +### PUT - Updating Workspace Metadata **Endpoint:** `PUT /api/{type}/{id}/modified/{modified}` **Behavior:** -- **Updates an existing Mongoose document** in-place -- Targets a specific version by `stix.id` AND `stix.modified` -- Uses `_.merge()` to apply changes to the document -- Increments Mongoose `__v` field (optimistic locking counter) -- No new document created - modifies the snapshot directly +- Targets a specific revision by `stix.id` and `stix.modified` +- Allows changes only to non-exported `workspace` metadata +- Returns `409 Conflict` if the resulting `stix` differs from the persisted + revision +- Does not emit a STIX updated event or create a release-track snapshot **Example:** ```javascript -// Update the 2024-01-01 version in-place +// Update review metadata without changing the STIX revision PUT /api/data-components/x-mitre-data-component--123/modified/2024-01-01T00:00:00.000Z { - stix: { - description: "Updated description" + workspace: { + workflow: { "state": "reviewed" } } } ``` -**Result:** The existing document is modified: -- Same `_id` in MongoDB -- Same `stix.modified` timestamp -- `__v` incremented from 0 to 1 -- Content updated via `_.merge(document, data)` +To correct a description, name, relationship endpoint, or any other STIX field, +POST a complete new revision with the same `stix.id` and a later +`stix.modified` timestamp. -**Use Case:** -- **Rarely used** in practice -- Useful for fixing typos in historical snapshots -- Administrative corrections without creating new versions +**Use Case:** review and other workspace-only state that is not exported. **Lifecycle Hooks Triggered:** - `beforeUpdate` - `afterUpdate` -- `emitUpdatedEvent` - -**Important Note on `_.merge()` Behavior:** -- Lodash `_.merge()` performs a **deep merge** -- Properties present in the target but **omitted** from the source are **NOT deleted** -- To remove a property, you must **explicitly set it to `null`** - -```javascript -// This does NOT remove x_mitre_data_source_ref: -PUT /api/data-components/{id}/modified/{modified} -{ - stix: { - name: "New Name" - // x_mitre_data_source_ref omitted - } -} - -// This DOES remove x_mitre_data_source_ref: -PUT /api/data-components/{id}/modified/{modified} -{ - stix: { - name: "New Name", - x_mitre_data_source_ref: null // Explicitly set to null - } -} -``` --- @@ -184,7 +155,7 @@ Embedded relationships are stored **directly on the STIX documents** under `work - ❌ `name` - NOT stored (mutable, must be fetched on read) **Why Not Store Names:** -- Names are **mutable** - users can change them via PUT/POST operations +- Names change by creating a new POST revision - Storing them would create **data staleness** issues - Would require **event propagation** to keep in sync across all references - MongoDB warns against **unbounded arrays** with duplicated mutable data @@ -550,10 +521,10 @@ Only create DS1 snapshots when its `embedded_relationships` actually change. - Enables rollback - Triggers correct lifecycle hooks -2. **Use PUT sparingly** - - Only for administrative corrections - - Be aware of `_.merge()` behavior - - Explicitly set fields to `null` to remove them +2. **Use PUT only for workspace metadata** + - STIX-changing requests return `409 Conflict` + - POST corrections as new revisions + - Treat the selected `stix.id` plus `stix.modified` as immutable 3. **Query latest versions by default** - `GET /api/data-components/{id}?versions=latest` @@ -565,9 +536,10 @@ Only create DS1 snapshots when its `embedded_relationships` actually change. ### For Service Developers -1. **Implement both lifecycle hooks** +1. **Implement the applicable lifecycle hooks** - `beforeCreate` / `afterCreate` for POST operations (versioning) - - `beforeUpdate` / `afterUpdate` for PUT operations (in-place edits) + - `beforeUpdate` / `afterUpdate` only for workspace-metadata PUT behavior; + metadata PUTs do not emit STIX updated events 2. **Detect version changes in `beforeCreate`** - Fetch previous latest version diff --git a/docs/developer/workspace-validation.md b/docs/developer/workspace-validation.md index 6abd378d..ce90e173 100644 --- a/docs/developer/workspace-validation.md +++ b/docs/developer/workspace-validation.md @@ -19,8 +19,8 @@ write or clear it. ## Why state-track validation at all? -ADM validation is the gate at the write boundary: every POST and PUT -runs the composed STIX object through the ADM schemas before +ADM validation is the gate at the write boundary: every POST and metadata-only +PUT runs the composed STIX object through the ADM schemas before persistence (see [`base.service.js`](../../app/services/meta-classes/base.service.js) pipeline stage 5, "VALIDATE WITH ADM"). If validation fails on a write, the request throws and nothing is persisted. @@ -86,7 +86,7 @@ document was either never validated or last passed validation." 1. `workspace.validation` is **server-controlled.** Clients cannot set, modify, or carry forward this field through any write path. 2. The field is **recomputed (or omitted) on every successful write.** - A POST or PUT that passes ADM validation produces a document with + A POST or metadata-only PUT that passes ADM validation produces a document with no `workspace.validation`. A POST or PUT that fails ADM validation throws — nothing is persisted, and the prior document (if any) is untouched until a future write or scheduler tick revisits it. @@ -112,6 +112,8 @@ document was either never validated or last passed validation." - `stripServerControlledFields()` removes any client-supplied `workspace.validation`. - ADM validation runs against the composed object. +- If the submitted body changes persisted `stix` content, the request returns + `409 Conflict`; a content correction must be a new POST revision. - If validation fails, the request throws — the existing document is untouched. - If validation passes: diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index c092c3b2..5f2a202d 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -413,8 +413,20 @@ Filtering occurs before pagination, so `pagination.total` is the total number of snapshots matching `tagged`, not the total number in the track. Every summary contains `id`, `type`, `modified`, `version`, `name`, -`description` (when set), and `members_count`. Count keys then reflect the -track type: +`description` (when set), and `members_count`. A tagged snapshot whose +deterministic member graph has been materialized also contains the opaque +`graph_manifest_id` and `graph_statistics`; graphless snapshots omit both. +Graph statistics describe the cached graph at a glance: + +- `primary_count`: member objects deliberately selected for the snapshot. +- `secondary_count`: related objects reached by graph resolution. +- `relationship_count`: relationships connecting cached graph objects. +- `supporting_count`: supporting identities and marking definitions. +- `link_target_count`: objects pinned for deterministic LinkById expansion. +- `total_count`: all entries across those manifest roles. + +The UI groups supporting and LinkById targets together as **Dependencies**. +Snapshot tier count keys continue to reflect the track type: - `type: "standard"` adds `staged_count` and `candidates_count`. - `type: "virtual"` adds `quarantine_count`. @@ -429,9 +441,18 @@ Inapplicable count keys are omitted rather than returned as zero. "type": "standard", "modified": "2024-01-15T16:20:00.000Z", "version": "14.1", + "graph_manifest_id": "release-track-graph-manifest--01234567-89ab-4cde-8f01-23456789abcd", "name": "Enterprise ATT&CK", "description": "Enterprise domain release track", "members_count": 3247, + "graph_statistics": { + "primary_count": 3247, + "secondary_count": 812, + "relationship_count": 6841, + "supporting_count": 5, + "link_target_count": 17, + "total_count": 10922 + }, "staged_count": 18, "candidates_count": 5 } @@ -630,6 +651,27 @@ Bootstraps a new release track from the specified snapshot. POST /api/release-tracks/:id/snapshots/:modified/clone ``` +### Create or Delete a Deterministic Member Graph + +``` +POST /api/release-tracks/:id/snapshots/:modified/graph +DELETE /api/release-tracks/:id/snapshots/:modified/graph +``` + +Only tagged snapshots may have graphs. POST resolves the snapshot's `members` +into a pointer-only exact-revision manifest and returns `201`; repeating it is +idempotent and returns `200`. DELETE removes the manifest and returns `204` +even when no graph exists. Graphless bundles resolve relationships and +secondary objects live. Requests that include candidates or staged objects +remain live even if the tagged snapshot has a graph. + +User interfaces may present this operation as **caching the bundle**: a cached +indicator means member-only bundle exports reuse the exact object and +relationship revisions selected when the cache was created. This is not a +general response cache and does not make candidate or staged exports +deterministic. + + ### Delete Specific Snapshot ``` @@ -637,9 +679,10 @@ DELETE /api/release-tracks/:id/snapshots/:modified ``` Deletes the selected snapshot only when it is both the latest snapshot and an -untagged draft. Deletion reverts the track to the immediately preceding -snapshot. Tagged releases and older drafts return `409 Conflict`; they cannot -be removed or rewritten. +untagged draft with a predecessor. Deletion reverts the track to that +predecessor. Standard tracks retain only one rolling draft, so replaced +untagged timestamps return `404`. Tagged releases and a track's sole snapshot +return `409 Conflict`. --- @@ -1348,14 +1391,14 @@ retrieval never recomputes virtual composition. As long as the track does not acquire a newer snapshot, `/snapshots/latest` selects the same primary revision set, and `/snapshots/:modified` addresses that set explicitly. -The server freezes the bounded `format=bundle` graph when it persists the -snapshot. Repeated exports reuse exact relationship, secondary, supporting, -and LinkById dependency revisions rather than discovering the current graph. -Hard deletes and unsafe in-place edits to those protected revisions return -`409 Conflict`. +Virtual snapshot persistence freezes primary membership, not the bounded +bundle graph. A tagged snapshot may opt into the graph separately through the +graph endpoint above. Until then, relationships and secondary objects resolve +live. Hard deletes of graph-pinned revisions return `409 Conflict`; every +STIX-changing PUT returns `409` regardless of graph state. -A standard draft remains intentionally dynamic only when the request includes -a candidate or staged entry stored with `object_modified: "latest"`. That +Candidate and staged exports are intentionally live, including exact-selector +entries, because determinism is guaranteed only for `members`. A `"latest"` selector is resolved at request time until release. Tagged standard members and all materialized virtual members are exact. diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 4fff6916..83213514 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -29,7 +29,7 @@ scanning tracks. | `id` | `release-track--` | The referencing release track | | `type` | `standard`, `virtual` | The type of the referencing release track | | `tier` | `members`, `staged`, `candidates`, `quarantine` | Which tier of the track references this revision; values match the snapshot tier array names | -| `status` | `modified-in-place`, `work-in-progress`, `awaiting-review`, `reviewed` | Track-scoped workflow status (`modified-in-place` is server-assigned when the pinned revision is edited via an in-place PUT) | +| `status` | `modified-in-place`, `work-in-progress`, `awaiting-review`, `reviewed` | Track-scoped workflow status (`modified-in-place` is retained for legacy data but is no longer produced because STIX revisions are immutable) | An object referenced by multiple tracks carries one entry per track. @@ -67,52 +67,33 @@ An object referenced by multiple tracks carries one entry per track. `workspace.validation`, the field is maintained by the server. Values supplied in `POST`/`PUT` bodies are silently ignored, and updates through the standard object endpoints cannot remove or alter existing entries. -- **Read-your-own-writes.** `POST`/`PUT` responses include backrefs produced - by the request's own side effects — e.g. when revision sync re-pins a - track to the newly created revision, the response body already carries the - resulting `workspace.release_tracks` entry. +- **Read-your-own-writes.** POST responses include backrefs produced when + revision sync re-pins a track to the newly created revision. Metadata-only + PUT responses retain the existing server-managed backrefs. ## In-place edits, deletes, and revocations Release tracks are never blind to changes in the objects they pin: -- **Released and graph-frozen revisions are immutable in place.** `PUT` and - `DELETE` against a revision that any track pins in its `members` tier, or - that a snapshot needs as a secondary/supporting graph dependency, return - `409 Conflict` — released content cannot be changed or destroyed under the - track. Make changes by creating a new revision (`POST`); retire an object - by creating a new revision with `x_mitre_deprecated: true`. Revision sync - captures either one. This guard checks tagged snapshots authoritatively, not - only the current `workspace.release_tracks` value. A revision remains - protected when it belongs only to a historical tagged release, when a newer - draft has removed it, or when a reconciliation failure temporarily omitted - its backref. -- **Standalone candidate/staged roots remain editable.** Merely appearing in - a draft workflow tier does not create a graph-protection conflict, so the - existing in-place review workflow below still applies. If that same revision - is also a frozen secondary dependency of another selected root, graph - protection takes precedence and the edit returns `409`. -- **Candidate/staged-pinned revisions can be edited in place, but the track - sees it.** An in-place `PUT` (including one that only sets - `x_mitre_deprecated`) marks the pinned entry `modified-in-place`: the - content changed, but because in-place edits carry no revision history the - track cannot say *what* changed — only that a re-review is required. The - entry's tier is decided by the workflow gate against the track's candidacy - threshold: in a strict track (threshold `reviewed`, the default) a staged - entry demotes back to `candidates`; in a permissive track (threshold - `work-in-progress` with `auto_promote`) the entry stays staged, since - `modified-in-place` ranks with `work-in-progress`. `manual`-strategy - tracks opt out entirely. Repeat edits of an entry already marked - `modified-in-place` do not create additional snapshots. Reviewers clear - the marker through the normal review endpoint - (`from: "modified-in-place"`). +- **Every persisted STIX revision is immutable.** A PUT whose `stix` payload + differs from the stored revision returns `409 Conflict`, regardless of + whether the revision is a member, candidate, staged object, or unrelated to + a track. Create corrections and deprecations as new POST revisions. PUT is + limited to non-exported `workspace` metadata and does not trigger revision + sync. +- **Graph and membership pins protect deletion.** Exact revisions in tagged + membership or an active/pending opt-in graph cannot be hard-deleted. The + guard checks authoritative tagged snapshots and graph entries rather than + relying only on `workspace.release_tracks`. A revision remains protected + even if a derived backref is temporarily absent. - **Revoking a tracked object queues the revoked revision.** The revoke workflow creates one new revision of the revoked object (`revoked: true`); revision sync enrolls it as a candidate in tracks where the object is a member and moves candidate/staged pins to it. The revoking object and the `revoked-by` relationship are not direct track members. Snapshot creation captures them as bounded secondary graph dependencies - when applicable; later bundle export replays that frozen graph. + when applicable; later member-only bundle export replays its exact revision + pointers. Unversioned marking definitions are the frozen-payload exception. ## Lifecycle example diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index 4a27bb59..463f93a3 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -99,9 +99,12 @@ Standard STIX bundle format: - Self-contained: identities and marking definitions referenced by the exported objects are included automatically - `LinkById` tags in descriptions are converted to markdown citations -- If a draft export explicitly includes candidate or staged tiers, dynamic - `"latest"` selectors are resolved for that export request. Tagged member - contents remain exact. +- Drafts, graphless tagged snapshots, and every export that includes candidate + or staged tiers resolve the bounded graph live. A tagged member-only export + is deterministic only after its snapshot opts into a graph manifest. +- Frontends may describe manifest creation as **caching the bundle**. The + cache pins the exact member graph for repeatable export; it is not a general + performance cache, and candidate or staged additions remain live. - Bundle export is fail-closed for primary content. If any selected exact revision no longer exists, the server returns HTTP `409` with every missing `(object_ref, object_modified)` pair in `missing_references`; it never emits diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index bbae7b7d..66fbe35a 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -97,7 +97,7 @@ Typical release tracks will use the default candidacy threshold setting of `revi However, smaller teams operating in purely development or research capacities may prefer a more permissive model. Perhaps they simply want all objects to be included in the release irrespective of object status. In such situations, the candidacy threshold can be lowered to `awaiting-review` or `work-in-progress`. -The threshold is enforced by the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`), the single decision point that places tracked objects into tiers whenever revision sync reacts to a change (new revision, in-place edit, revocation). The server-assigned `modified-in-place` status ranks with `work-in-progress` in the threshold order — so in a permissive track, an in-place edit of a staged object keeps it staged (marked for re-review), while in a strict track it demotes back to candidates. +The threshold is enforced by the **workflow gate** (`app/lib/release-tracks/workflow-gate.js`), the single decision point that places tracked objects into tiers whenever revision sync reacts to a new revision, revocation, or conversion. Persisted STIX revisions cannot be edited in place; content changes arrive as new POSTed revisions. ### Option 1: Include Only Reviewed (Default) ```javascript diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index e887a010..e7c1ad27 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -82,6 +82,8 @@ GET /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/release +POST /api/release-tracks/:id/snapshots/:modified/graph +DELETE /api/release-tracks/:id/snapshots/:modified/graph ``` ### 2. Git-Inspired Versioning @@ -89,10 +91,10 @@ POST /api/release-tracks/:id/snapshots/:modified/release We borrow heavily concepts from git. Snapshots are sort of like commits and tagged releases are like git tags. A release track contains snapshots: delta permutations that can be linearly tracked to deduce how the release track has evolved over time. A snapshot is generated every time a supported draft operation changes state, such as adding or promoting candidates, updating release-track configuration, or renaming the release track. **Snapshots** (like Git commits) -- Every modification creates a new snapshot +- Every supported modification creates a replacement draft snapshot - Identified by `stix.modified` timestamp - Immutable once created -- Complete audit trail +- Standard tracks retain one rolling untagged draft; tagged releases remain historical - May be a **draft release** (untagged) or **tagged release** (has version number) **Tagged Releases** (like Git tags) @@ -129,12 +131,13 @@ and committing are separate operations, so a newer object revision created between them can legitimately produce a different plan; the committed release records the revision resolved by the commit itself. -At snapshot persistence, the server also freezes the bounded bundle graph: -exact relationship endpoint revisions, secondary objects, supporting objects, -and LinkById render targets. Tagged releases and materialized virtual -snapshots therefore reproduce the same STIX object graph on later -`format=bundle` retrievals. The generated bundle-envelope ID itself is not -stable. +Snapshots are graphless by default. After tagging, callers may opt into a +deterministic member graph with `POST .../snapshots/:modified/graph`. The graph +stores exact-revision pointers for relationships, secondary objects, +versioned supporting objects, and LinkById targets; unversioned marking +definitions are frozen by value. `DELETE` on the graph resource returns the +snapshot to live graph resolution. Candidate/staged bundle additions are +always live. The generated bundle-envelope ID itself is not stable. Virtual snapshots are stricter still: they copy only exact member revisions from tagged standard component snapshots. They never inherit `track_latest`, diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 44e6962f..ca25180c 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -4,8 +4,8 @@ The Release Tracks API uses a Git-inspired versioning strategy that separates two distinct concerns: -1. **Snapshot History** - Every modification creates a new timestamped snapshot for complete audit trail -2. **Release Versioning** - Specific snapshots can be "tagged" as releases using semantic versioning +1. **Draft State** - Standard tracks keep one rolling, untagged snapshot +2. **Release History** - Tagged snapshots are retained as immutable releases using semantic versioning This approach allows continuous development while providing stable, versioned releases for publication. @@ -19,7 +19,9 @@ A **snapshot** is an immutable state of a release track at a specific point in t - `id` - The release track's STIX identifier (constant across all snapshots) - `modified` - ISO 8601 timestamp when the snapshot was created (unique per snapshot) -Every modification operation creates a new snapshot with a new `modified` timestamp. +Every content-changing operation creates a replacement snapshot with a new +`modified` timestamp. For a standard track, the replacement is saved first and +then the older untagged draft is removed. Tagged snapshots are never pruned. A snapshot may be either a **draft release** (untagged) or a **tagged release** (has version number). @@ -34,13 +36,13 @@ A **tagged release** is a snapshot that has been marked as production-ready for Not all snapshots are tagged releases. Only snapshots explicitly tagged via the **release** operation become tagged releases. -**Example Timeline with Tagged Releases:** +**Example Timeline with Tagged Releases (standard track):** ``` id: "release-track--123", modified: "2024-01-01T10:00:00.000Z" - version: null ← DRAFT RELEASE (work in progress) + version: null ← FIRST ROLLING DRAFT id: "release-track--123", modified: "2024-01-02T14:30:00.000Z" - version: null ← DRAFT RELEASE (work in progress) + version: null ← REPLACEMENT DRAFT; THE 2024-01-01 DRAFT IS PRUNED id: "release-track--123", modified: "2024-01-05T09:15:00.000Z" version: "1.0" ← TAGGED RELEASE (via release operation) @@ -52,7 +54,7 @@ id: "release-track--123", modified: "2024-01-05T09:15:00.000Z" }] id: "release-track--123", modified: "2024-01-10T11:00:00.000Z" - version: null ← DRAFT RELEASE (more development) + version: null ← NEW ROLLING DRAFT AFTER RELEASE 1.0 id: "release-track--123", modified: "2024-01-15T16:20:00.000Z" version: "1.1" ← TAGGED RELEASE (via release operation) @@ -62,6 +64,9 @@ id: "release-track--123", modified: "2024-01-15T16:20:00.000Z" ] ``` +The timeline lists the first draft only to illustrate its replacement. Once the +second draft is durably stored, the first draft is no longer retrievable. + ## The Release Operation ### What Does Releasing Do? @@ -83,6 +88,17 @@ the preview or commit request is handled. Only exact revisions are promoted into `members`, so the tagged release never contains a dynamic member reference. +Releasing does not automatically create a graph manifest. A tagged snapshot +may subsequently opt into deterministic member-graph retrieval with: + +```http +POST /api/release-tracks/:id/snapshots/:modified/graph +``` + +Deleting that manifest with the corresponding `DELETE` operation returns the +snapshot to live graph resolution. Candidate and staged export additions are +always resolved live; the determinism guarantee applies only to `members`. + ### In-Place Tagging Strategy When you release a snapshot: diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index dc18d3de..34f121c3 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -962,9 +962,9 @@ quarantined object counts. Use `format=workbench` or `format=bundle` to inspect the literal snapshot or publication artifact that would be tagged. The draft must have a non-null `composition_resolution`, proving that its members and quarantine tiers were materialized from its current composition. -Bundle preview replays the materialized draft's graph manifest, and release -retains that same manifest because tagging a virtual snapshot does not alter -its contents. +Bundle preview resolves the live graph. Tagging does not implicitly create a +manifest; determinism is a separate opt-in operation on the tagged snapshot: +`POST /api/release-tracks/:id/snapshots/:modified/graph`. ### Retrieve a Materialized Virtual Snapshot @@ -1000,7 +1000,8 @@ object-revision selector. This guarantee also covers the bounded `format=bundle` object graph. Relationship endpoint revisions, secondary objects, supporting objects, and -LinkById render targets are frozen in the snapshot's graph manifest. +LinkById render targets are frozen only after a tagged snapshot opts into a +graph manifest; graphless snapshots resolve them live. Repeated exports may use a different bundle-envelope UUID, but replay the same snapshot object graph. See [Bundle Export](../../developer/release-tracks/bundle-export.md#relationship-and-secondary-object-consistency-boundary). diff --git a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js index ff5ba792..dc969a24 100644 --- a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js +++ b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js @@ -214,6 +214,9 @@ async function backfillSnapshotManifests(db, options) { const manifestId = await graphManifestService.prepare(snapshot, { baselineReconstruction: true, + // Preserve the historical migration's schema-v1 frozen relationship + // contract. New opt-in graphs use pointer-only schema v2. + schemaVersion: 1, }); try { await db From 0a913292e3368055785e3df2c658855fdd7f4185 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:39:57 -0400 Subject: [PATCH 48/55] fix(release-tracks): attest historical snapshot graphs Reconstruct v19.1 manifests from exact persisted entity and relationship revisions, including deterministic LinkById dependencies and source serialization hints. Fail closed when canonical-domain provenance is unavailable and provide a corrective migration for previously inferred revisions. --- .../definitions/components/release-tracks.yml | 82 ++++ app/api/definitions/openapi.yml | 3 + .../paths/release-tracks-paths.yml | 45 ++ app/controllers/release-tracks-controller.js | 26 ++ .../release-tracks/release-track-schemas.js | 39 ++ .../release-track-graph-manifest-model.js | 6 + app/routes/release-tracks-routes.js | 8 + app/services/release-tracks/export-service.js | 16 +- .../release-tracks/graph-manifest-service.js | 213 +++++++++ .../release-tracks/release-tracks-service.js | 4 + .../release-tracks/snapshot-service.js | 10 + .../canonical-domain-migration.spec.js | 124 +++-- .../api/release-tracks/opt-in-graphs.spec.js | 427 ++++++++++++++++++ docs/admin/canonical-domain-migration.md | 68 ++- docs/developer/TODO.md | 291 ++++++++++++ docs/developer/data-model.md | 14 +- .../developer/release-tracks/bundle-export.md | 46 +- docs/user/release-tracks/api-reference.md | 17 + docs/user/release-tracks/summary.md | 1 + ...0000-backfill-canonical-x-mitre-domains.js | 251 ++++++++-- ...90000-correct-canonical-x-mitre-domains.js | 34 ++ 21 files changed, 1617 insertions(+), 108 deletions(-) create mode 100644 app/tests/api/release-tracks/opt-in-graphs.spec.js create mode 100644 migrations/20260803190000-correct-canonical-x-mitre-domains.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 307f671a..cbacc16d 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -775,6 +775,88 @@ components: items: $ref: '#/components/schemas/object-revision-reference' + source-graph-reconstruction: + type: object + additionalProperties: false + required: + - source_attestation + - entries + properties: + source_attestation: + type: object + additionalProperties: false + required: + - kind + - bundle_sha256 + - collection_id + - release + - domain + properties: + kind: + type: string + enum: + - source-bundle + bundle_sha256: + type: string + pattern: '^[a-f0-9]{64}$' + collection_id: + type: string + release: + type: string + domain: + type: string + enum: + - enterprise-attack + - ics-attack + - mobile-attack + entries: + type: array + minItems: 1 + items: + $ref: '#/components/schemas/source-graph-entry' + + source-graph-entry: + type: object + additionalProperties: false + required: + - kind + - object_ref + - object_modified + properties: + kind: + type: string + enum: + - root + - relationship + - secondary + - supporting + - link_target + object_ref: + type: string + object_modified: + type: string + format: date-time + nullable: true + source: + $ref: '#/components/schemas/object-revision-reference' + target: + $ref: '#/components/schemas/object-revision-reference' + omitted_optional_defaults: + type: array + maxItems: 2 + uniqueItems: true + description: >- + Optional false-valued fields omitted by the attested source + publication; valid only as deterministic serialization hints. + items: + type: string + enum: + - revoked + - x_mitre_remote_support + frozen_stix: + type: object + description: 'Allowed only for unversioned marking definitions' + release-track-reconciliation-error: type: object required: diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index e1e67b6e..502346c8 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -412,6 +412,9 @@ paths: /api/release-tracks/{id}/snapshots/{modified}/graph: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1graph' + /api/release-tracks/{id}/snapshots/{modified}/graph/reconstruct: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1graph~1reconstruct' + /api/release-tracks/{id}/snapshots/{modified}/release: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1release' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index da2740b9..5d311459 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -1322,6 +1322,51 @@ paths: '409': description: 'The snapshot is untagged or its graph changed concurrently' + /api/release-tracks/{id}/snapshots/{modified}/graph/reconstruct: + post: + summary: 'Reconstruct a historical deterministic graph from source-bundle pointers' + operationId: 'release-tracks-snapshot-graph-reconstruct' + description: | + Administrative recovery operation for a tagged historical snapshot. + The caller derives exact revision pointers from an externally verified + source bundle; the bundle itself is not imported. The server verifies + that every pointer exists, roots exactly match snapshot members, + relationship endpoint pins match the stored relationship, and all + referenced endpoint and supporting objects are present. Versioned + objects remain pointer-only; only unversioned marking definitions may + carry a frozen payload. Repeating the same attestation is idempotent; + an ordinary graph or a different attestation is rejected. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: modified + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/source-graph-reconstruction' + responses: + '200': + description: 'The tagged snapshot already had a deterministic graph' + '201': + description: 'Source-attested deterministic graph created successfully' + '400': + description: 'Malformed reconstruction plan' + '404': + description: 'Snapshot or exact object revision not found' + '409': + description: 'The snapshot is untagged or the source plan violates graph integrity' + /api/release-tracks/{id}/snapshots/{modified}/release: post: summary: 'Release a specific snapshot' diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index c90c8d74..fbf51faf 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -50,6 +50,7 @@ const { updateCompositionBodySchema, createVirtualSnapshotBodySchema, promoteQuarantinedObjectBodySchema, + reconstructSnapshotGraphBodySchema, xMitreVersionSchema, } = require('../lib/release-tracks/release-track-schemas'); @@ -629,6 +630,31 @@ exports.createSnapshotGraph = async function createSnapshotGraph(req, res, next) } }; +/** POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct */ +exports.reconstructSnapshotGraph = async function reconstructSnapshotGraph(req, res, next) { + try { + const bodyResult = reconstructSnapshotGraphBodySchema.safeParse(req.body); + if (!bodyResult.success) { + return next( + new BadRequestError({ + message: 'Invalid source graph reconstruction request', + details: bodyResult.error.errors, + }), + ); + } + const result = await releaseTracksService.reconstructSnapshotGraph( + req.params.id, + req.params.modified, + bodyResult.data, + ); + logger.debug(`Success: Reconstructed graph for snapshot ${req.params.modified}`); + return res.status(result.created ? 201 : 200).send(result.snapshot); + } catch (err) { + logger.error('Failed to reconstruct snapshot graph: ' + err); + return next(err); + } +}; + /** DELETE /api/release-tracks/:id/snapshots/:modified/graph */ exports.deleteSnapshotGraph = async function deleteSnapshotGraph(req, res, next) { try { diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 62077d35..ee114513 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -530,6 +530,44 @@ const promoteQuarantinedObjectBodySchema = z }) .strict(); +const exactGraphRevisionSchema = z + .object({ + object_ref: stixIdentifierSchema, + object_modified: z.iso.datetime(), + }) + .strict(); + +const sourceGraphEntrySchema = z + .object({ + kind: z.enum(['root', 'relationship', 'secondary', 'supporting', 'link_target']), + object_ref: stixIdentifierSchema, + object_modified: z.iso.datetime().nullable(), + source: exactGraphRevisionSchema.optional(), + target: exactGraphRevisionSchema.optional(), + omitted_optional_defaults: z + .array(z.enum(['revoked', 'x_mitre_remote_support'])) + .max(2) + .optional(), + frozen_stix: z.object({}).passthrough().optional(), + }) + .strict(); + +/** Administrative recovery of a historical graph from an external source bundle. */ +const reconstructSnapshotGraphBodySchema = z + .object({ + source_attestation: z + .object({ + kind: z.literal('source-bundle'), + bundle_sha256: z.string().regex(/^[a-f0-9]{64}$/), + collection_id: createStixIdValidator('x-mitre-collection'), + release: xMitreVersionSchema, + domain: z.enum(['enterprise-attack', 'ics-attack', 'mobile-attack']), + }) + .strict(), + entries: z.array(sourceGraphEntrySchema).min(1), + }) + .strict(); + // ============================================================================= // Exports // ============================================================================= @@ -593,6 +631,7 @@ module.exports = { updateCompositionBodySchema, createVirtualSnapshotBodySchema, promoteQuarantinedObjectBodySchema, + reconstructSnapshotGraphBodySchema, // Reusable sub-schemas componentTrackSchema, diff --git a/app/models/release-tracks/release-track-graph-manifest-model.js b/app/models/release-tracks/release-track-graph-manifest-model.js index c1d37c77..26b2fdec 100644 --- a/app/models/release-tracks/release-track-graph-manifest-model.js +++ b/app/models/release-tracks/release-track-graph-manifest-model.js @@ -24,6 +24,7 @@ const manifestSchema = new mongoose.Schema( schema_version: { type: Number, required: true, default: 1 }, resolver_version: { type: String, required: true }, baseline_reconstruction: { type: Boolean, required: true, default: false }, + source_attestation: { type: mongoose.Schema.Types.Mixed }, created_at: { type: Date, required: true, default: Date.now }, }, { collection: 'releaseTrackGraphManifests' }, @@ -54,6 +55,11 @@ const entrySchema = new mongoose.Schema( object_modified: { type: Date }, source: { type: exactRevisionSchema }, target: { type: exactRevisionSchema }, + omitted_optional_defaults: { + type: [String], + enum: ['revoked', 'x_mitre_remote_support'], + default: undefined, + }, discovered_from: { type: [exactRevisionSchema], default: undefined }, // Schema-v2 relationships are exact-revision pointers. Marking // definitions are not STIX-versioned, so their complete payload is frozen diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 594985b7..51515dd0 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -265,6 +265,14 @@ router releaseTracksController.cloneByModified, ); +router + .route('/release-tracks/:id/snapshots/:modified/graph/reconstruct') + .post( + authn.authenticate, + authz.requireRole(authz.admin), + releaseTracksController.reconstructSnapshotGraph, + ); + router .route('/release-tracks/:id/snapshots/:modified/graph') .post( diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index 3e15ea80..bec58eb5 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -79,6 +79,20 @@ function requiresLiveGraph(snapshot, options) { ); } +function normalizeSourceBundleDefaults(documents, graph) { + if (graph.manifest?.resolver_version !== 'source-bundle-pointer-v2') return documents; + + return documents.map((document) => { + const normalized = { ...document, stix: { ...document.stix } }; + // Apply only source-attested shape hints. Most v19.1 objects explicitly + // emitted false and must retain it; a small minority omitted the default. + for (const field of graph.sourceOmittedDefaults?.get(document.stix.id) || []) { + if (normalized.stix[field] === false) delete normalized.stix[field]; + } + return normalized; + }); +} + // ============================================================================= // Format helpers (delegating to Zod transform schemas) // ============================================================================= @@ -157,7 +171,7 @@ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options const graph = requiresLiveGraph(snapshot, options) ? await graphManifestService.replayPlannedSnapshot(snapshot, options) : await graphManifestService.replay(snapshot, options); - const allObjects = graph.documents; + const allObjects = normalizeSourceBundleDefaults(graph.documents, graph); await convertLinkByIdTags(allObjects, graph.linkTargetDocuments); return exports.formatAsBundle(snapshot, allObjects, { diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js index 407054aa..9720cc46 100644 --- a/app/services/release-tracks/graph-manifest-service.js +++ b/app/services/release-tracks/graph-manifest-service.js @@ -1,5 +1,6 @@ 'use strict'; +const { isDeepStrictEqual } = require('node:util'); const { v4: uuidv4 } = require('uuid'); const linkById = require('../../lib/linkById'); const bundleRelationships = require('../../lib/stix-bundle-relationships'); @@ -16,6 +17,7 @@ const primaryRevisionService = require('./primary-revision-service'); const MANIFEST_SCHEMA_VERSION = 2; const RESOLVER_VERSION = 'bounded-member-graph-v2'; +const SOURCE_BUNDLE_RESOLVER_VERSION = 'source-bundle-pointer-v2'; const TIERS = ['members', 'staged', 'candidates', 'quarantine']; const STATISTIC_FIELDS_BY_KIND = { root: 'primary_count', @@ -326,6 +328,208 @@ async function prepare(snapshot, options = {}) { return manifestId; } +function sourcePlanIntegrityError(details, references = []) { + return new ReleaseContentIntegrityError(references, { details }); +} + +async function buildSourceManifestEntries(snapshot, plan) { + const seenObjectRefs = new Set(); + const planned = []; + + for (const input of plan.entries) { + if (seenObjectRefs.has(input.object_ref)) { + throw sourcePlanIntegrityError( + `Source bundle contains more than one revision for '${input.object_ref}'.`, + [{ object_ref: input.object_ref, dependency: 'unique_source_revision' }], + ); + } + seenObjectRefs.add(input.object_ref); + + const isVersioned = input.object_modified != null; + if (isVersioned && input.frozen_stix) { + throw sourcePlanIntegrityError( + 'Versioned source-bundle entries must be exact database pointers, not frozen payloads.', + [{ object_ref: input.object_ref, dependency: 'pointer_only_manifest' }], + ); + } + if (!isVersioned) { + if ( + input.kind !== 'supporting' || + input.frozen_stix?.type !== 'marking-definition' || + input.frozen_stix?.id !== input.object_ref || + input.frozen_stix?.modified != null + ) { + throw sourcePlanIntegrityError( + 'Only unversioned marking definitions may be frozen in a schema-v2 manifest.', + [{ object_ref: input.object_ref, dependency: 'unversioned_supporting_object' }], + ); + } + } + if (input.kind === 'relationship') { + if (!input.source || !input.target || !isVersioned) { + throw sourcePlanIntegrityError( + 'Relationship entries require an exact relationship pointer and exact endpoint pins.', + [{ object_ref: input.object_ref, dependency: 'relationship_endpoints' }], + ); + } + } else if (input.source || input.target) { + throw sourcePlanIntegrityError( + 'Only relationship entries may declare source and target endpoint pins.', + [{ object_ref: input.object_ref, dependency: 'relationship_endpoints' }], + ); + } + + planned.push({ + ...input, + object_modified: isVersioned ? new Date(input.object_modified) : undefined, + source: input.source + ? { ...input.source, object_modified: new Date(input.source.object_modified) } + : undefined, + target: input.target + ? { ...input.target, object_modified: new Date(input.target.object_modified) } + : undefined, + revision_key: isVersioned + ? revisionKey(input.object_ref, input.object_modified) + : `${input.object_ref}::unversioned`, + }); + } + + const expectedRoots = new Map( + (snapshot.members || []).map((entry) => [ + revisionKey(entry.object_ref, entry.object_modified), + entry, + ]), + ); + const suppliedRoots = planned.filter((entry) => entry.kind === 'root'); + const suppliedRootKeys = new Set(suppliedRoots.map((entry) => entry.revision_key)); + if ( + suppliedRoots.length !== expectedRoots.size || + [...expectedRoots.keys()].some((key) => !suppliedRootKeys.has(key)) + ) { + throw sourcePlanIntegrityError( + 'Source bundle root pointers must exactly equal the tagged snapshot members.', + [{ track_id: snapshot.id, dependency: 'snapshot_members' }], + ); + } + + const versioned = planned.filter((entry) => entry.object_modified); + const hydrated = await primaryRevisionService.assertStoredEntries(versioned); + const documentsByRevision = new Map( + hydrated.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + const selectableKeys = new Set( + planned + .filter((entry) => ['root', 'secondary'].includes(entry.kind)) + .map((entry) => entry.revision_key), + ); + + for (const entry of planned) { + if (!entry.object_modified) continue; + const document = documentsByRevision.get(entry.revision_key); + if (entry.kind === 'relationship') { + if (document.stix.type !== 'relationship') { + throw sourcePlanIntegrityError(`'${entry.object_ref}' is not a relationship revision.`, [ + { object_ref: entry.object_ref, dependency: 'relationship_type' }, + ]); + } + for (const side of ['source', 'target']) { + const endpoint = entry[side]; + if (document.stix[`${side}_ref`] !== endpoint.object_ref) { + throw sourcePlanIntegrityError( + `Relationship '${entry.object_ref}' has a mismatched ${side} pointer.`, + [{ object_ref: entry.object_ref, dependency: `${side}_ref` }], + ); + } + if (!selectableKeys.has(revisionKey(endpoint.object_ref, endpoint.object_modified))) { + throw sourcePlanIntegrityError( + `Relationship '${entry.object_ref}' references an endpoint revision absent from the source graph.`, + [{ ...endpoint, dependency: `${side}_revision` }], + ); + } + } + } else if (document.stix.type === 'relationship') { + throw sourcePlanIntegrityError( + `Relationship revision '${entry.object_ref}' must use kind 'relationship'.`, + [{ object_ref: entry.object_ref, dependency: 'entry_kind' }], + ); + } + } + + const includedObjectRefs = new Set(planned.map((entry) => entry.object_ref)); + for (const document of hydrated.documents) { + const supportingRefs = [ + document.stix.created_by_ref, + ...(document.stix.object_marking_refs || []), + ].filter(Boolean); + const missingRef = supportingRefs.find((objectRef) => !includedObjectRefs.has(objectRef)); + if (missingRef) { + throw sourcePlanIntegrityError(`Source graph omits supporting object '${missingRef}'.`, [ + { object_ref: missingRef, dependency: 'supporting_object' }, + ]); + } + } + + return planned.map((entry) => { + if (entry.kind !== 'root') return entry; + const root = expectedRoots.get(entry.revision_key); + return { ...entry, tier: 'members', object_status: root.object_status }; + }); +} + +async function prepareSourceReconstruction(snapshot, plan) { + const manifestId = `release-track-graph-manifest--${uuidv4()}`; + const entries = await buildSourceManifestEntries(snapshot, plan); + const common = { + manifest_id: manifestId, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + }; + const manifest = { + ...common, + state: 'pending', + schema_version: MANIFEST_SCHEMA_VERSION, + resolver_version: SOURCE_BUNDLE_RESOLVER_VERSION, + baseline_reconstruction: true, + source_attestation: plan.source_attestation, + }; + + await ReleaseTrackGraphManifest.create(manifest); + try { + await ReleaseTrackGraphManifestEntry.insertMany( + entries.map((entry) => ({ ...common, ...entry })), + ); + await replayEntries(entries, manifest, {}); + } catch (err) { + await discard(manifestId); + throw err; + } + return manifestId; +} + +async function assertSourceReconstruction(snapshot, sourceAttestation) { + const manifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: snapshot.graph_manifest_id, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + state: { $in: ['pending', 'active'] }, + }) + .lean() + .exec(); + if ( + !manifest || + manifest.resolver_version !== SOURCE_BUNDLE_RESOLVER_VERSION || + !isDeepStrictEqual(manifest.source_attestation, sourceAttestation) + ) { + throw sourcePlanIntegrityError( + 'Snapshot already has a graph that was not reconstructed from the same source bundle.', + [{ manifest_id: snapshot.graph_manifest_id, dependency: 'source_attestation' }], + ); + } +} + async function activate(manifestId) { await ReleaseTrackGraphManifest.updateOne( { manifest_id: manifestId, state: 'pending' }, @@ -532,6 +736,11 @@ async function replayEntries(entries, manifest, options) { .filter((entry) => entry.kind === 'link_target') .map((entry) => documentsByRevision.get(entry.revision_key)) .filter(Boolean); + const sourceOmittedDefaults = new Map( + entries + .filter((entry) => entry.omitted_optional_defaults?.length) + .map((entry) => [entry.object_ref, entry.omitted_optional_defaults]), + ); const emittedByRevision = new Map(); for (const document of [...selectedDocuments, ...supportingDocuments]) { @@ -544,6 +753,7 @@ async function replayEntries(entries, manifest, options) { return { documents: [...emittedByRevision.values()], linkTargetDocuments, + sourceOmittedDefaults, manifest, }; } @@ -676,6 +886,8 @@ async function findPinsForObject(objectRef) { module.exports = { prepare, + prepareSourceReconstruction, + assertSourceReconstruction, activate, discard, discardSnapshot, @@ -688,4 +900,5 @@ module.exports = { buildManifestEntries, MANIFEST_SCHEMA_VERSION, RESOLVER_VERSION, + SOURCE_BUNDLE_RESOLVER_VERSION, }; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 3945d576..103819b5 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -357,6 +357,10 @@ exports.createSnapshotGraph = function createSnapshotGraph(trackId, modified) { return snapshotService.createGraph(trackId, modified); }; +exports.reconstructSnapshotGraph = function reconstructSnapshotGraph(trackId, modified, plan) { + return snapshotService.reconstructGraph(trackId, modified, plan); +}; + exports.deleteSnapshotGraph = function deleteSnapshotGraph(trackId, modified) { return snapshotService.deleteGraph(trackId, modified); }; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index c796caaf..9faf4c44 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -561,6 +561,16 @@ exports.createGraph = function createLiveGraph(trackId, modified) { return createGraph(trackId, modified, (snapshot) => graphManifestService.prepare(snapshot)); }; +exports.reconstructGraph = function reconstructGraph(trackId, modified, plan) { + return createGraph( + trackId, + modified, + (snapshot) => graphManifestService.prepareSourceReconstruction(snapshot, plan), + (snapshot) => + graphManifestService.assertSourceReconstruction(snapshot, plan.source_attestation), + ); +}; + exports.deleteGraph = async function deleteGraph(trackId, modified) { const snapshot = await dynamicRepo.getSnapshotByModified(trackId, modified); if (!snapshot) { diff --git a/app/tests/api/release-tracks/canonical-domain-migration.spec.js b/app/tests/api/release-tracks/canonical-domain-migration.spec.js index 20e75c7b..c5ed262c 100644 --- a/app/tests/api/release-tracks/canonical-domain-migration.spec.js +++ b/app/tests/api/release-tracks/canonical-domain-migration.spec.js @@ -43,8 +43,11 @@ const objectFixtures = [ type: 'campaign', name: 'Active migration campaign', lifecycle: 'active', + // Legacy collection appearance says ICS because the campaign was pulled + // into an ICS graph as secondary content. Only exact TOC membership is + // authoritative, and this campaign is an Enterprise primary. collectionRefs: [collectionIds.ics], - expectedDomains: ['ics-attack'], + expectedDomains: ['enterprise-attack'], }, { path: '/api/mitigations', @@ -214,6 +217,36 @@ describe('Canonical ATT&CK domain migration', function () { expect(provenanceResult.matchedCount).toBe(1); } + const fixturesByDomain = new Map([ + ['enterprise-attack', []], + ['ics-attack', []], + ['mobile-attack', []], + ]); + for (const fixture of objectFixtures) { + const document = created.get(fixture.id); + for (const domain of fixture.expectedDomains) { + fixturesByDomain.get(domain).push({ + object_ref: document.stix.id, + object_modified: new Date(document.stix.modified), + }); + } + } + await mongoose.connection.db.collection('attackObjects').insertMany( + Object.entries(collectionIds).map(([domainName, collectionId]) => ({ + __t: 'Collection', + workspace: { workflow: { state: 'reviewed' } }, + stix: { + id: collectionId, + type: 'x-mitre-collection', + spec_version: '2.1', + created: new Date('2026-01-01T00:00:00.000Z'), + modified: new Date('2026-01-01T00:00:00.000Z'), + name: `${domainName} canonical collection`, + x_mitre_contents: fixturesByDomain.get(`${domainName}-attack`), + }, + })), + ); + const revokedFixture = objectFixtures.find((fixture) => fixture.lifecycle === 'revoked'); await mongoose.connection.db.collection('attackObjects').updateOne( { 'stix.id': revokedFixture.id }, @@ -325,7 +358,7 @@ describe('Canonical ATT&CK domain migration', function () { expect(retiredRules).toEqual([]); }); - it('covers every domain-bearing ATT&CK object type and infers domain unions from provenance', function () { + it('covers every domain-bearing type and ignores secondary collection appearances', async function () { expect(migration._private.TARGET_TYPES).toEqual([ 'attack-pattern', 'campaign', @@ -342,17 +375,22 @@ describe('Canonical ATT&CK domain migration', function () { 'x-mitre-tactic', ]); + const domainsByRevision = await migration._private.buildCanonicalTocDomainIndex(migrationDb); + const campaign = created.get(campaignFixture.id); expect( - migration._private.domainsFromCollectionProvenance({ - workspace: { - collections: [ - { collection_ref: collectionIds.mobile }, - { collection_ref: collectionIds.enterprise }, - { collection_ref: 'x-mitre-collection--ffffffff-ffff-4fff-8fff-ffffffffffff' }, - ], + migration._private.domainsFromCanonicalToc( + { + stix: { + id: campaign.stix.id, + modified: campaign.stix.modified, + }, + workspace: { + collections: [{ collection_ref: collectionIds.ics }], + }, }, - }), - ).toEqual(['enterprise-attack', 'mobile-attack']); + domainsByRevision, + ), + ).toEqual(['enterprise-attack']); }); it('leaves inactive clone ids to the native database driver', async function () { @@ -419,6 +457,7 @@ describe('Canonical ATT&CK domain migration', function () { }); expect(report.verification).toEqual({ remaining_latest_domainless_target_objects: 0, + remaining_latest_incorrect_domain_objects: 0, remaining_domain_validation_bypasses: 0, }); @@ -487,6 +526,35 @@ describe('Canonical ATT&CK domain migration', function () { ); }); + it('corrects a previously generated domain-only successor from its exact TOC predecessor', async function () { + const latest = await mongoose.connection.db + .collection('attackObjects') + .findOne({ 'stix.id': campaignFixture.id }, { sort: { 'stix.modified': -1 } }); + const incorrect = structuredClone(latest); + delete incorrect._id; + incorrect.stix.modified = new Date(new Date(latest.stix.modified).getTime() + 1); + incorrect.stix.x_mitre_domains = ['enterprise-attack', 'ics-attack']; + incorrect.stix.x_mitre_modified_by_ref = 'identity--ffffffff-ffff-4fff-8fff-ffffffffffff'; + await mongoose.connection.db.collection('attackObjects').insertOne(incorrect); + + const report = await migration._private.run(migrationDb, migrationClient, { + migrationName: 'test-correct-canonical-x-mitre-domains', + correctIncorrect: true, + }); + expect(report.counts).toMatchObject({ + scanned_candidates: 1, + active_reposts: 1, + updated: 1, + failed: 0, + }); + expect(report.verification.remaining_latest_incorrect_domain_objects).toBe(0); + + const corrected = await mongoose.connection.db + .collection('attackObjects') + .findOne({ 'stix.id': campaignFixture.id }, { sort: { 'stix.modified': -1 } }); + expect(corrected.stix.x_mitre_domains).toEqual(['enterprise-attack']); + }); + it('is idempotent after canonical revisions and bypass removal are complete', async function () { const report = await migration._private.run(migrationDb, migrationClient); expect(report.counts.scanned_candidates).toBe(0); @@ -495,7 +563,7 @@ describe('Canonical ATT&CK domain migration', function () { expect(await migration._private.countRemainingDomainlessTargets(migrationDb)).toBe(0); }); - it('defaults unmapped active and inactive domainless objects to Enterprise', async function () { + it('leaves unmapped domainless objects unchanged and retains enforcement bypasses', async function () { const unknownActiveId = 'intrusion-set--ffffffff-ffff-4fff-8fff-ffffffffffff'; const unknownInactiveId = 'campaign--eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee'; const now = new Date(); @@ -538,14 +606,13 @@ describe('Canonical ATT&CK domain migration', function () { const report = await migration._private.run(migrationDb, migrationClient); expect(report.counts).toMatchObject({ - scanned_candidates: 2, - active_reposts: 1, - inactive_clones: 1, - enterprise_defaults: 2, - deprecated: 1, - updated: 2, + scanned_candidates: 0, + unmapped_skipped: 2, + active_reposts: 0, + inactive_clones: 0, + updated: 0, failed: 0, - bypasses_removed: 1, + bypasses_removed: 0, }); for (const stixId of [unknownActiveId, unknownInactiveId]) { @@ -554,11 +621,10 @@ describe('Canonical ATT&CK domain migration', function () { .find({ 'stix.id': stixId }) .sort({ 'stix.modified': -1 }) .toArray(); - expect(revisions).toHaveLength(2); - expect(revisions[0].stix.x_mitre_domains).toEqual(['enterprise-attack']); - expect(revisions[1].stix.x_mitre_domains).toBeUndefined(); + expect(revisions).toHaveLength(1); + expect(revisions[0].stix.x_mitre_domains).toBeUndefined(); } - expect(await migration._private.countStaleDomainBypasses(migrationDb)).toBe(0); + expect(await migration._private.countStaleDomainBypasses(migrationDb)).toBe(1); const completedRun = await mongoose.connection.db .collection('automationRuns') @@ -567,17 +633,17 @@ describe('Canonical ATT&CK domain migration', function () { { sort: { started_at: -1 } }, ); expect(completedRun.status).toBe('completed'); - expect(completedRun.counts.enterprise_defaults).toBe(2); + expect(completedRun.counts.unmapped_skipped).toBe(2); + expect(completedRun.warnings.unmapped_domainless_objects.count).toBe(2); + expect(completedRun.warnings.unmapped_domainless_objects.sample).toEqual( + expect.arrayContaining([unknownActiveId, unknownInactiveId]), + ); const fallbackItems = await mongoose.connection.db .collection('automationRunItems') .find({ run_id: completedRun.run_id }) .toArray(); - expect(fallbackItems).toHaveLength(2); - expect(fallbackItems.map((item) => item.details.domain_source)).toEqual([ - 'enterprise-default', - 'enterprise-default', - ]); + expect(fallbackItems).toHaveLength(0); }); after(async function () { diff --git a/app/tests/api/release-tracks/opt-in-graphs.spec.js b/app/tests/api/release-tracks/opt-in-graphs.spec.js new file mode 100644 index 00000000..01258437 --- /dev/null +++ b/app/tests/api/release-tracks/opt-in-graphs.spec.js @@ -0,0 +1,427 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const { + ReleaseTrackGraphManifest, + ReleaseTrackGraphManifestEntry, +} = require('../../../models/release-tracks/release-track-graph-manifest-model'); +const relationshipsRepository = require('../../../repository/relationships-repository'); +const AttackObject = require('../../../models/attack-object-model'); +const { releaseExactMembers } = require('./release-track-test-helpers'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +describe('Opt-in deterministic release-track graphs', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + function authenticated(builder) { + return builder + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + function technique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_domains: ['enterprise-attack'], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + x_mitre_version: '1.0', + }, + }; + } + + function relationship(source, target, previous) { + const modified = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || modified, + modified, + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: source.stix.id, + target_ref: target.stix.id, + description: previous ? 'New relationship revision' : 'Original relationship revision', + object_marking_refs: [markingDefinitionId], + }, + }; + } + + async function post(path, body, status = 201) { + return (await authenticated(request(app).post(path).send(body)).expect(status)).body; + } + + async function createTrack(name) { + return post('/api/release-tracks/new', { name, type: 'standard' }); + } + + async function sourcePlan(primary, secondary, relationshipRevision) { + const supporting = await AttackObject.find({ + 'stix.id': { + $in: [primary.stix.created_by_ref, markingDefinitionId], + }, + }) + .lean() + .exec(); + return { + source_attestation: { + kind: 'source-bundle', + bundle_sha256: '0'.repeat(64), + collection_id: 'x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019', + release: '19.1', + domain: 'enterprise-attack', + }, + entries: [ + { + kind: 'root', + object_ref: primary.stix.id, + object_modified: primary.stix.modified, + omitted_optional_defaults: ['revoked'], + }, + { + kind: 'secondary', + object_ref: secondary.stix.id, + object_modified: secondary.stix.modified, + }, + { + kind: 'relationship', + object_ref: relationshipRevision.stix.id, + object_modified: relationshipRevision.stix.modified, + source: { + object_ref: primary.stix.id, + object_modified: primary.stix.modified, + }, + target: { + object_ref: secondary.stix.id, + object_modified: secondary.stix.modified, + }, + }, + ...supporting.map((document) => ({ + kind: 'supporting', + object_ref: document.stix.id, + object_modified: document.stix.modified + ? new Date(document.stix.modified).toISOString() + : null, + ...(document.stix.modified ? {} : { frozen_stix: document.stix }), + })), + ], + }; + } + + it('creates pointer-only member graphs only when a tagged snapshot opts in', async function () { + const primary = await post('/api/techniques', technique('Opt-in Graph Primary')); + const secondary = await post('/api/techniques', technique('Opt-in Graph Secondary')); + const originalRelationship = await post('/api/relationships', relationship(primary, secondary)); + const track = await createTrack('Opt in Graph Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [primary]); + + expect(released).not.toHaveProperty('graph_manifest_id'); + expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: track.id })).toBe(0); + + const globalRelationshipScan = relationshipsRepository.retrieveAllForBundle; + relationshipsRepository.retrieveAllForBundle = async () => { + throw new Error('graph capture must not scan every relationship'); + }; + let graphSnapshot; + try { + graphSnapshot = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + {}, + ); + } finally { + relationshipsRepository.retrieveAllForBundle = globalRelationshipScan; + } + expect(graphSnapshot.graph_manifest_id).toBeDefined(); + + const manifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: graphSnapshot.graph_manifest_id, + }) + .lean() + .exec(); + expect(manifest.schema_version).toBe(2); + + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: graphSnapshot.graph_manifest_id, + }) + .lean() + .exec(); + expect(entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'root', + object_ref: primary.stix.id, + object_modified: expect.any(Date), + }), + expect.objectContaining({ + kind: 'secondary', + object_ref: secondary.stix.id, + object_modified: expect.any(Date), + }), + expect.objectContaining({ + kind: 'relationship', + object_ref: originalRelationship.stix.id, + object_modified: expect.any(Date), + }), + ]), + ); + const relationshipEntry = entries.find((entry) => entry.kind === 'relationship'); + expect(relationshipEntry).not.toHaveProperty('frozen_stix'); + for (const entry of entries.filter((item) => ['root', 'secondary'].includes(item.kind))) { + expect(entry.discovered_from).toBeUndefined(); + } + const markingEntry = entries.find((entry) => entry.object_ref === markingDefinitionId); + expect(markingEntry.frozen_stix).toBeDefined(); + + const correctedRelationship = await post( + '/api/relationships', + relationship(primary, secondary, originalRelationship), + ); + expect(correctedRelationship.stix.id).toBe(originalRelationship.stix.id); + + const bundle = ( + await authenticated( + request(app).get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle`, + ), + ).expect(200) + ).body; + const exportedRelationship = bundle.objects.find( + (object) => object.id === originalRelationship.stix.id, + ); + expect(exportedRelationship.modified).toBe(originalRelationship.stix.modified); + expect(exportedRelationship.description).toBe('Original relationship revision'); + + const idempotent = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + {}, + 200, + ); + expect(idempotent.graph_manifest_id).toBe(graphSnapshot.graph_manifest_id); + + await authenticated( + request(app).delete( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + ), + ).expect(204); + await authenticated( + request(app).delete( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + ), + ).expect(204); + + const liveBundle = ( + await authenticated( + request(app).get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle`, + ), + ).expect(200) + ).body; + const liveRelationship = liveBundle.objects.find( + (object) => object.id === originalRelationship.stix.id, + ); + expect(liveRelationship.modified).toBe(correctedRelationship.stix.modified); + expect(liveRelationship.description).toBe('New relationship revision'); + }); + + it('rejects graph creation for an untagged snapshot', async function () { + const track = await createTrack('Draft Graph Rejection'); + await authenticated( + request(app) + .post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(track.modified)}/graph`, + ) + .send({}), + ).expect(409); + }); + + it('reconstructs a historical graph from exact source-bundle pointers', async function () { + const primary = await post('/api/techniques', technique('Source Graph Primary')); + const secondary = await post('/api/techniques', technique('Source Graph Secondary')); + const linkTarget = await post('/api/techniques', technique('Source Graph Link Target')); + const originalRelationship = await post('/api/relationships', relationship(primary, secondary)); + const track = await createTrack('Source Attested Graph Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [primary]); + + const revisedSecondaryPayload = structuredClone(secondary); + revisedSecondaryPayload.stix.modified = new Date( + new Date(secondary.stix.modified).getTime() + 1000, + ).toISOString(); + revisedSecondaryPayload.stix.description = 'Post-release secondary revision'; + const revisedSecondary = await post('/api/techniques', revisedSecondaryPayload); + const revisedRelationship = await post( + '/api/relationships', + relationship(primary, revisedSecondary, originalRelationship), + ); + + const plan = await sourcePlan(primary, secondary, originalRelationship); + plan.entries.push({ + kind: 'link_target', + object_ref: linkTarget.stix.id, + object_modified: linkTarget.stix.modified, + }); + const invalidPlan = structuredClone(plan); + invalidPlan.entries.find((entry) => entry.kind === 'relationship').target.object_modified = + revisedSecondary.stix.modified; + await authenticated( + request(app) + .post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}/graph/reconstruct`, + ) + .send(invalidPlan), + ).expect(409); + + const reconstructed = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}/graph/reconstruct`, + plan, + ); + const manifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: reconstructed.graph_manifest_id, + }) + .lean() + .exec(); + expect(manifest).toMatchObject({ + schema_version: 2, + resolver_version: 'source-bundle-pointer-v2', + baseline_reconstruction: true, + source_attestation: plan.source_attestation, + }); + + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: reconstructed.graph_manifest_id, + }) + .lean() + .exec(); + expect( + entries.find((entry) => entry.object_ref === originalRelationship.stix.id), + ).not.toHaveProperty('frozen_stix'); + expect(entries.find((entry) => entry.object_ref === linkTarget.stix.id)).toMatchObject({ + kind: 'link_target', + }); + + const bundle = ( + await authenticated( + request(app).get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle`, + ), + ).expect(200) + ).body; + expect(bundle.objects.find((object) => object.id === secondary.stix.id).modified).toBe( + secondary.stix.modified, + ); + expect( + bundle.objects.find((object) => object.id === originalRelationship.stix.id).modified, + ).toBe(originalRelationship.stix.modified); + expect( + bundle.objects.some((object) => object.modified === revisedRelationship.stix.modified), + ).toBe(false); + expect(bundle.objects.some((object) => object.id === linkTarget.stix.id)).toBe(false); + expect(bundle.objects.find((object) => object.id === primary.stix.id)).not.toHaveProperty( + 'revoked', + ); + + await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}/graph/reconstruct`, + plan, + 200, + ); + const conflictingAttestation = structuredClone(plan); + conflictingAttestation.source_attestation.bundle_sha256 = '1'.repeat(64); + await authenticated( + request(app) + .post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}/graph/reconstruct`, + ) + .send(conflictingAttestation), + ).expect(409); + }); + + it('keeps one rolling draft per standard track', async function () { + const track = await createTrack('Rolling Standard Draft'); + await post(`/api/release-tracks/${track.id}/meta`, { description: 'first replacement' }, 200); + const latest = await post( + `/api/release-tracks/${track.id}/meta`, + { description: 'second replacement' }, + 200, + ); + + const snapshots = await dynamicRepo.getAllSnapshots(track.id); + expect(snapshots.data.filter((snapshot) => snapshot.version == null)).toHaveLength(1); + expect(new Date(snapshots.data[0].modified).getTime()).toBe( + new Date(latest.modified).getTime(), + ); + expect(latest).not.toHaveProperty('graph_manifest_id'); + }); + + it('treats versioned STIX payloads as immutable while allowing workspace-only PUTs', async function () { + const object = await post('/api/techniques', technique('Immutable STIX Revision')); + const changed = structuredClone(object); + changed.stix.description = 'An illegal in-place STIX correction'; + + const rejected = await authenticated( + request(app) + .put(`/api/techniques/${object.stix.id}/modified/${object.stix.modified}`) + .send(changed), + ).expect(409); + expect(rejected.body.message).toMatch(/immutable/i); + + const workspaceOnly = structuredClone(object); + workspaceOnly.workspace.workflow.state = 'awaiting-review'; + const accepted = await authenticated( + request(app) + .put(`/api/techniques/${object.stix.id}/modified/${object.stix.modified}`) + .send(workspaceOnly), + ).expect(200); + expect(accepted.body.stix.description).toBe(object.stix.description); + expect(accepted.body.workspace.workflow.state).toBe('awaiting-review'); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/docs/admin/canonical-domain-migration.md b/docs/admin/canonical-domain-migration.md index f3890316..9c7c1989 100644 --- a/docs/admin/canonical-domain-migration.md +++ b/docs/admin/canonical-domain-migration.md @@ -22,16 +22,21 @@ are enabled. ## Domain source Startup does not access GitHub or another network service, and the migration -is not coupled to a particular ATT&CK release manifest. Workbench records the -canonical ATT&CK collections containing each exact object revision in -`workspace.collections`. The migration maps those persisted Enterprise, ICS, -and Mobile collection references back to their domains. +is not coupled to a particular ATT&CK release manifest. It reads the persisted +Enterprise, ICS, and Mobile `x-mitre-collection` revisions and indexes their +exact `x_mitre_contents` pins. -Domain membership is inferred from an exact revision's collection provenance: +Domain membership is inferred from exact collection TOC membership: -- one canonical collection reference produces one domain; -- multiple canonical collection references produce the complete domain union; -- unrelated collection references are ignored. +- one canonical collection TOC containing an exact revision produces one domain; +- multiple canonical TOCs containing the exact revision produce the complete union; +- bundle appearance and `workspace.collections` backrefs are ignored. + +That distinction is essential. Legacy imports recorded `workspace.collections` +for every imported bundle object, including campaigns and groups discovered as +secondary relationship content. Legacy bundle rendering could also project a +primary target's domains onto those secondary payload copies. Neither signal +proves that the secondary object was a primary member of that domain. The migration examines the latest revision of every domain-bearing ATT&CK lineage: techniques, campaigns, mitigations, groups, malware, tools, @@ -73,21 +78,38 @@ automation audit records are inserted together with stable sequence numbers. The old revision is never updated or deleted in either path. -## Unmapped-object fallback +### Forward correction for earlier deployments + +Migration `20260803190000-correct-canonical-x-mitre-domains.js` repairs +deployments that already ran the earlier collection-appearance inference. It +only selects a latest revision when: + +- an exact historical predecessor is present in a canonical collection TOC; +- the latest revision is substantively identical to that predecessor after + ignoring the fields controlled by a domain repair; and +- the latest domain array differs from the predecessor's exact TOC union. + +This recognizes migration/bootstrap-generated domain-only successors without +overwriting a later operator-authored revision that changed substantive STIX +content. The correction creates another immutable revision through the same +active/inactive paths described above. + +## Unmapped-object handling Before creating any object revision, the migration resolves the complete -latest domainless candidate set from persisted collection provenance. If an -object has no recognized canonical collection reference, the migration assigns -`["enterprise-attack"]`. This permits legacy custom content to satisfy the -stricter contract without blocking startup. The fallback is explicit in the -per-object audit record as `domain_source: "enterprise-default"` and increments -the run's `enterprise_defaults` counter. +latest domainless candidate set from exact collection TOC membership. If an +object has no recognized canonical TOC pin, the migration leaves it unchanged. +Neither legacy `workspace.collections` appearances nor the absence of a TOC +match proves Enterprise membership. A completed run records these objects in +`warnings.unmapped_domainless_objects`, increments `unmapped_skipped`, and +does not create per-object repair audit items for them. Persisted missing-domain validation bypasses are deleted only after all object -repairs succeed and verification finds no remaining latest domainless target. -A partial repair therefore leaves enforcement unchanged and fails startup. On -restart, already repaired lineages are skipped and only the remaining work is -retried. +repairs succeed **and** verification finds no remaining latest domainless +target. When unmapped objects remain, the bypasses are retained so startup can +complete without activating a contract the database does not yet satisfy. A +failed mapped repair still fails startup. On restart, already repaired +lineages are skipped and only the remaining mapped work is retried. ## Verification and audit @@ -106,17 +128,21 @@ Important counters are: - `inactive_clones` - `active_batches` - `inactive_batches` -- `enterprise_defaults` +- `unmapped_skipped` - `revoked` - `deprecated` - `bypasses_removed` - `failed` -A completed run reports both verification values as zero: +A completed run with complete canonical provenance reports all verification +values as zero. A completed run with unmapped objects may instead report +nonzero domainless-object and bypass counts alongside the warning described +above: ```javascript { remaining_latest_domainless_target_objects: 0, + remaining_latest_incorrect_domain_objects: 0, remaining_domain_validation_bypasses: 0 } ``` diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index fe540c0d..6db226ed 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,296 @@ # Release Track TODOs +## Frontend graph cache lifecycle controls + +- [x] Replace the static cache-materialization hourglass with the existing + Angular Material indeterminate spinner. +- [x] Add an editor-only, confirmed delete action for cached snapshot graphs, + including progress and success/error feedback. +- [x] Add connector/component regressions, update frontend behavior notes, and + run focused plus complete frontend verification. + +Verification (2026-08-03): + +- Focused Angular connector/component regressions pass: 68 tests. The complete + frontend suite passes: 162 files and 366 tests. +- Targeted ESLint and Prettier checks pass. The production Angular build passes + with the local persistent cache temporarily disabled to avoid the documented + environment-specific native crash; `angular.json` was restored afterward. +- Proposed frontend commit: `feat(release-tracks): manage snapshot bundle + caches`. + +## Source-attested v19.1 graph reconstruction + +- [x] Add fail-closed regressions for canonical-domain migration when exact + collection TOC provenance is unavailable; never infer Enterprise from + missing evidence. +- [x] Add an administrator-only schema-v2 source reconstruction endpoint that + validates a closed pointer plan against tagged snapshot members and + persisted exact revisions. +- [x] Build v19.1 source plans without importing bundles, inferring legacy SRO + endpoint revisions from the unique objects emitted in each source bundle. +- [x] Validate the complete source plan against MongoDB before tagging, then + attach it atomically and require a final bundle comparison. +- [x] Update OpenAPI, Bruno, operator/developer documentation, and bootstrap + recovery guidance. +- [x] Run focused migration, reconstruction, and bootstrap regressions; then + lint and the complete `npm test` suite. + +Verification (2026-08-03): + +- Canonical-domain migration regressions pass: 9 cases. Unmapped domainless + objects remain unchanged, are reported, and keep the legacy bypasses active. +- Source reconstruction regressions pass: 5 cases. They prove exact historical + relationship and endpoint revisions replay after their live lineages advance, + reject incomplete pointer plans, and enforce source-attestation idempotence. +- Bootstrap regressions pass: 33 cases. Source bundles are never imported; + 30,649 emitted-object pointers and 54 non-emitted LinkById dependency + pointers are hydrated from MongoDB before tagging. The largest + reconstruction request remains below the 50 MiB API request limit and + outside the 16 MiB per-document BSON limit. +- The clean complete REST suite passes: OpenAPI 2, config 21, API 995, + middleware 29, and scheduler 10. Four roaming harness failures in the first + run passed independently (13, 24, 25, and 4 cases) before the clean rerun. +- Repository ESLint, Python Ruff, Python bytecode compilation, and diff + whitespace validation pass. +- Proposed commit: `fix(release-tracks): attest v19.1 snapshot graphs`. + +### Historical relationship hydration follow-up + +- [x] Reproduce the 21,025 missing Enterprise revisions against the restored + production-shaped database and classify payload differences. +- [x] Hydrate relationship pointers from the dedicated MongoDB collection and + render LinkById fields through the same deterministic export semantics. +- [x] Add regressions that fail on relationship content drift while accepting + exact persisted timestamps and export-only LinkById rendering. +- [x] Update bootstrap documentation and run focused plus complete verification. + +Verification (2026-08-03): + +- The 21,025 failures are exactly the Enterprise relationship count. Every + sampled exact timestamp exists in MongoDB's dedicated `relationships` + collection; the bootstrap had incorrectly queried `attackObjects` for all + pointer kinds. +- All 24,552 v19.1 relationships were audited read-only. Of those, 5,624 raw + payloads already match exactly and 18,928 differ only because exports render + persisted `(LinkById: ...)` tags as Markdown links. +- A production-shaped preflight reconstructs Enterprise (25,851 graph entries), + ICS (2,201), and Mobile (2,651) with zero missing, changed, or additional + emitted objects. The 54 entries beyond the 30,649 emitted objects are exact, + non-emitted cross-domain LinkById dependencies. +- Bootstrap regressions pass: 33 cases. Focused source-graph regressions pass: + 5 cases. The clean complete REST suite passes: OpenAPI 2, config 21, API 995, + middleware 29, and scheduler 10. +- A separate test-only correction serializes a Mongoose snapshot date before + placing it in a graph URL; its isolated virtual-graph-integrity spec passes: + 3 cases. +- Proposed implementation commit: `fix(release-tracks): hydrate historical + relationship graphs`. Proposed test-only commit: `test(release-tracks): + serialize snapshot timestamps in graph URLs`. + +## Snapshot-history graph cache statistics + +- [x] Add regression coverage for exact manifest-kind counts on cached + snapshot summaries and omission on uncached snapshots. +- [x] Aggregate graph cache statistics for every manifest on a history page in + one indexed query and expose the typed summary through OpenAPI. +- [x] Show CTI-oriented Primary, Secondary, Relationships, and Dependencies + statistics for cached snapshots in the frontend History tab. +- [x] Update release-track user/developer documentation and run focused plus + complete backend/frontend verification. + +Verification (2026-08-03): + +- Focused snapshot-history regressions pass: 7 REST cases and 65 Angular + connector/component cases. OpenAPI validation passes. +- The complete frontend suite passes: 162 files and 363 tests. The Angular + build passes with the local persistent cache temporarily disabled to avoid + the environment-specific native cache crash; `angular.json` was restored. +- The complete REST suite passes: OpenAPI 2, config 21, API 994, middleware 29, + and scheduler 10. A documented roaming backref setup flake passed all 24 + cases in isolation before the clean complete rerun. +- REST lint and targeted frontend ESLint/Prettier checks pass. The performance + audit is `PERFORMANT`: one indexed aggregate covers every manifest on the + bounded history page, with no per-snapshot query. +- Proposed REST commit: `feat(release-tracks): expose graph cache statistics`. +- Proposed frontend commit: `feat(release-tracks): show graph cache statistics`. + +## v19.1 bootstrap graph lifecycle and canonical-domain correction + +- [x] Add regressions proving canonical domains come from exact collection TOC + membership, not secondary bundle appearance or projected payload fields. +- [x] Correct the startup canonical-domain backfill and add a forward migration + for already-created domain-only successor revisions. +- [x] Replace the bootstrap's custom schema-v1 draft manifest writes with the + supported tagged-snapshot schema-v2 graph endpoint. +- [x] Make bootstrap resume and final verification require a persisted graph + and a post-graph v19.1 bundle comparison with no drift override. +- [x] Update the bootstrap runbook and canonical-domain documentation with the + corrected provenance contract and recovery behavior. +- [x] Run focused migration/bootstrap/release-track regressions, then lint and + the complete `npm test` suite. + +Verification (2026-08-03): + +- The local official v19.1 source audit finds exactly nine payload/TOC domain + mismatches, all campaigns; corrected virtual membership is Enterprise 4,815, + ICS 503, and Mobile 743. +- Canonical-domain migration regressions pass: 9 cases. Bootstrap regressions + pass: 29 cases, including pointer-only graph validation and semantic drift + rejection for relationships advanced by domain repairs. +- The complete `npm test` suite, repository ESLint, targeted migration ESLint, + Python Ruff, Python bytecode compilation, and diff whitespace checks pass. +- Proposed commit: `fix(release-tracks): correct v19.1 bootstrap provenance`. + +## Opt-in deterministic member graphs and rolling drafts + +- [x] Add regressions for immutable versioned STIX payloads, pointer-only + relationship manifests, and legacy frozen-manifest replay. +- [x] Add tagged-snapshot graph create/delete endpoints and make graphless + bundle exports resolve live while persisted graphs cover members only. +- [x] Stop automatic graph generation during snapshot cloning and release; + retain only the latest standard-track draft after a durable replacement. +- [x] Bound graph construction to relationship lineages that touch the + selected member frontier and batch exact-revision hydration. +- [x] Update OpenAPI, user/developer documentation, and Bruno requests for the + opt-in determinism and immutable-revision contracts. +- [x] Run focused regression specs, then the complete `npm test` suite and + review the final performance/architecture diff. + +Verification (2026-08-03): + +- Opt-in graph regressions pass: graphless release, tagged-only graph + creation/deletion, schema-v2 pointers, frozen marking definitions, + correction by POST, live graphless replay, and rolling-draft truncation. +- Release-track regressions pass: 176 cases. Immutable CRUD regressions pass: + 378 cases. The migration regression preserves schema-v1 frozen replay. +- The complete `npm test` suite passes: OpenAPI 2, config 21, API 993, + middleware 29, and scheduler 10. +- Repository lint and diff whitespace validation pass. +- Proposed commit: `feat(release-tracks): make deterministic graphs opt in`. + +### Frontend deterministic bundle cache controls + +- [x] Expose `graph_manifest_id` in lightweight snapshot-history summaries so + the UI can render cache state without per-snapshot requests. +- [x] Add the frontend connector and History-tab cache state, warning + tooltips, editor action, progress state, and success/error feedback. +- [x] Add focused REST and Angular regressions for summary propagation, + connector routing, cache-state mapping, and materialization. +- [x] Run formatting, lint, builds, and the complete frontend/backend suites; + record the final verification and proposed commits. + +Verification (2026-08-03): + +- Focused Angular connector/component regressions pass: 65 tests. The focused + REST snapshot-history regression passes: 7 tests. +- The complete frontend suite passes: 162 files and 363 tests. The Angular + build passes with the local persistent cache temporarily disabled to avoid + an environment-specific native `lmdb` crash; no cache setting was committed. +- The complete REST suite passes: OpenAPI 2, config 21, API 993, middleware 29, + and scheduler 10. REST lint passes. +- Changed frontend sources pass Prettier, targeted ESLint, and diff whitespace + checks. Repository-wide frontend lint remains red on 254 pre-existing + errors; the shared release-track API type retains one pre-existing + index-signature violation. +- Proposed frontend commit: `feat(release-tracks): add deterministic bundle + cache controls`. + +## Frontend canonical-domain preservation + +- [x] Inventory every frontend model and object view corresponding to the + canonical-domain migration's `TARGET_TYPES`. +- [x] Add a regression contract proving every target type preserves + `x_mitre_domains` through deserialize/serialize. +- [x] Add domain model support and editable domain fields to the missing + campaign, intrusion-set, detection-strategy, and matrix views. +- [x] Document the frontend domain-editing contract and run focused tests, + lint/format checks, and the complete frontend test suite. + +Verification (2026-08-03): + +- Canonical-domain model and view contracts pass: 25 cases covering every + migration target type. +- Complete frontend suite passes: 158 files and 323 tests. +- Angular build and targeted ESLint/Prettier checks for every changed source + file pass. +- Repository-wide lint remains red on 256 pre-existing errors outside this + change; no new lint errors remain in the hotfix files. +- Proposed frontend commit: `fix(stix): preserve canonical domains in + editors`. + +## C0028 campaign revision / released virtual-snapshot investigation + +- [x] Trace the submitted campaign payload through REST create handling and ADM + citation validation against the authoritative ADM source. +- [x] Reproduce the reported 400 response and isolate whether the defect is in + the payload, frontend transformation, REST API, or ADM. +- [x] Document the supported repair path and, as a fallback, enumerate every + database invariant/provenance record a manual repair would have to keep + consistent. +- [x] Record evidence, recommended regressions/fix scope, and a proposed + conventional commit message without mutating production data. + +Investigation (2026-08-03): + +- The reported request cites `Booz Allen Hamilton` in both campaign temporal + citation fields but sends only the `mitre-attack` external reference. ADM + 4.11.7 correctly reports both missing-reference refinements. Adding the + released Booz Allen reference makes the composed campaign pass the WIP ADM + schema. +- The Angular `Campaign` model does not deserialize or serialize + `x_mitre_domains`, and the campaign view exposes no domain editor. The + reported request consequently also omits the intended canonical-domain + correction. This is a frontend payload defect, not an ADM defect. +- The supplied database record is a `releaseTrackGraphManifestEntries` root + with an operationally frozen payload, not the authoritative campaign entity + in `attackObjects`. Editing it would rewrite an immutable released artifact + while retaining the old revision key and object timestamp. +- Supported hotfix: dry-run and then POST a new C0028 revision containing the + Booz Allen external reference and + `x_mitre_domains: ["enterprise-attack", "ics-attack"]`; let created-event + relationship advancement and standard-track member sync create the next + candidate/draft, then release the component track and materialize/tag a new + virtual snapshot. Do not alter the already-tagged virtual snapshot. +- Recommended regressions: frontend campaign/group round-trip coverage for + canonical domains; campaign save coverage that retains temporal citation + references; backend campaign regression proving a missing cited source is + rejected and the corrected revision succeeds with ADM validation enabled. +- Proposed implementation commit: `fix(campaigns): preserve domains and cited + references in revisions`. + +Verification (2026-08-03): + +- Direct ADM schema reproduction returns the two production error paths for + the reported composed STIX object and succeeds after adding the cited + reference and canonical domains in work-in-progress, awaiting-review, and + reviewed states. +- Existing campaign API regression passes: 21 cases. Its placeholder + organization identity causes an earlier suppressible ADM issue, so it does + not currently exercise the citation refinement and needs the targeted + regression above. + +Follow-up frontend citation-loss investigation (2026-08-03): + +- REST `BaseService.create()` removes only ATT&CK-owned external references, + preserves every submitted user reference, regenerates the canonical ATT&CK + reference, and validates that composed object. It deliberately does not + merge omitted user references from the previous revision. +- Angular initially retains the C0028 `Booz Allen Hamilton` reference when it + deserializes the GET response. `StixObject.base_validate()` first sends a + valid dry-run payload, then calls the mutating `ExternalReferences.validate()` + with only `description` and `aliases` as campaign citation fields. +- That incomplete field list treats the temporal citation reference as unused + and removes it before the real save POST. Commit `4f04ac70` added server + dry-run validation and explicitly removed `first_seen_citation` and + `last_seen_citation` from this field list, creating a time-of-check/time-of-use + mismatch. `ExternalReferences.parseObjectCitations()` still has the correct + campaign field list. +- Minimum repair: restore both temporal citation fields to campaign reference + validation. Durable repair: centralize the field list and complete all + reference synchronization before the server dry run so validation and save + serialize the same object state. + ## STIX 2.0 virtual snapshot bundles - [x] Add a virtual-track regression proving materialized snapshots emit STIX diff --git a/docs/developer/data-model.md b/docs/developer/data-model.md index 8eab910d..23b6b060 100644 --- a/docs/developer/data-model.md +++ b/docs/developer/data-model.md @@ -37,13 +37,21 @@ union, such as `["enterprise-attack", "mobile-attack"]`; Workbench does not store separate domain-narrowed copies of that revision. ADM validation requires the property before a domain-bearing object leaves the -partial `work-in-progress` workflow. Workbench does not suppress the -missing-domain error for any domain-bearing ATT&CK type. +partial `work-in-progress` workflow. New installations do not seed a +missing-domain bypass. A legacy persisted bypass may remain temporarily when +the migration finds domainless content with no authoritative TOC provenance. Migration `20260730230000-backfill-canonical-x-mitre-domains.js` creates replacement latest revisions for all domainless lineages without rewriting historical revisions. Domain unions come from persisted canonical collection provenance; -unmappable content defaults to Enterprise. See the +specifically, exact `(object_ref, object_modified)` membership in canonical +collection `x_mitre_contents` TOCs. Broad `workspace.collections` appearance +backrefs are not authoritative because legacy imports also attached them to +secondary graph objects. Unmappable content is left unchanged and reported; +the migration retains legacy validation bypasses rather than fabricate +Enterprise membership. Forward +migration `20260803190000-correct-canonical-x-mitre-domains.js` corrects +domain-only successors created by the older inference. See the [operator guide](../admin/canonical-domain-migration.md). ## Database Structure diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 612d92c9..c13efb49 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -146,10 +146,12 @@ The legacy and ephemeral graph renderer now preserves every nonempty compatibility fallback for exact historical domainless revisions pinned before canonical-domain enforcement, including historical matrix revisions. The fallback affects the rendered copy and does not update the stored -revision. The release-agnostic startup migration creates canonical replacement -revisions for the latest domainless object in every domain-bearing chain; all -subsequent content must persist canonical domains so virtual composition, -snapshot export, and ephemeral export observe the same membership. +revision. The release-agnostic startup migration creates a canonical +replacement only when an exact collection TOC entry proves the object's +domain. Unmapped legacy objects remain unchanged, are reported for follow-up, +and keep the temporary validation bypasses active. All subsequent content must +persist canonical domains so virtual composition, snapshot export, and +ephemeral export observe the same membership. Because snapshot contents are explicitly curated, primary entries do **not** receive the legacy attack-id / deprecated / revoked filters. Secondary graph @@ -194,6 +196,39 @@ then atomically attaches the manifest ID to the still-tagged snapshot. Replay can self-activate a complete linked pending manifest after an interrupted activation. `DELETE` on the same graph resource detaches and removes it. +Historical baselines whose relationships predate endpoint-pin capture require +a different, admin-only path: +`POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct`. Its body +contains a source-bundle attestation and a decoupled pointer plan, not the +bundle payload. The caller must independently verify the named bundle and its +SHA-256 digest. The server then verifies that roots exactly equal tagged +members, every exact revision exists, each relationship's STIX refs agree with +the supplied endpoint IDs, the endpoint revisions are included, and required +supporting objects are present. Versioned entries are always pointers; only an +unversioned marking definition may be frozen by value. The resulting manifest +uses resolver version `source-bundle-pointer-v2`, records the attestation, and +sets `baseline_reconstruction: true`. + +Source plans may contain `link_target` pointers for objects outside the emitted +domain bundle. They are hydrated for LinkById conversion but are not emitted. +Active ATT&CK-ID targets are preferred; a unique inactive historical target is +accepted only when no active v19.1 target exists. + +The v19.1 production bootstrap uses this path without importing the published +bundles. Because each official domain bundle contains one revision per STIX +ID, it can infer legacy SRO endpoint revisions by joining `source_ref` and +`target_ref` to those unique objects. Before tagging, the script batch-hydrates +the entire pointer plan from Workbench and compares its STIX object set with +the source bundle. This is the missing provenance that live database traversal +cannot recover after endpoint lineages have advanced. The bootstrap routes +entity pointers to `attackObjects` and relationship pointers to the dedicated +`relationships` collection. Its pre-tag comparison mirrors export-time +LinkById rendering. A pointer may carry a narrow serialization hint when the +attested source omitted a persisted optional `revoked: false` or +`x_mitre_remote_support: false` default. Most source objects explicitly emit +those false values and retain them. True values and every other payload +difference remain significant. Ordinary release-track exports retain their +existing serialization. Graph creation uses an indexed relationship frontier rather than scanning all relationships. It starts with member IDs, queries only current relationship @@ -217,6 +252,9 @@ revision of each legacy relationship can be endpoint-pinned truthfully. Pre-existing snapshot manifests are labeled `baseline_reconstruction` because they describe the graph visible during migration rather than an unknowable historical graph. They must not be represented as historical truth. +A verified external bundle can reconstruct a historical graph through the +admin operation above; without such an artifact, exact legacy endpoint +selection remains unknowable. Drafts and tagged snapshots without graphs resolve live. Candidate/staged exports also resolve live even when the snapshot has a graph, because those diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 5f2a202d..b5c70612 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -655,6 +655,7 @@ POST /api/release-tracks/:id/snapshots/:modified/clone ``` POST /api/release-tracks/:id/snapshots/:modified/graph +POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct DELETE /api/release-tracks/:id/snapshots/:modified/graph ``` @@ -671,6 +672,22 @@ relationship revisions selected when the cache was created. This is not a general response cache and does not make candidate or staged exports deterministic. +Administrators may use the separate `/graph/reconstruct` POST for a historical +baseline backed by an independently verified source bundle. The request sends +the bundle's SHA-256/collection/release/domain attestation plus exact graph +pointers; it does not import source STIX payloads. The server rejects plans +whose roots differ from `members`, whose revisions are missing, or whose +relationship endpoints are inconsistent. This recovery endpoint exists for +controlled bootstrap tooling and is not a replacement for ordinary graph +creation. A retry is idempotent only when the attached graph has the same +source attestation. + +Pointer roles may also include `link_target`: an exact, non-emitted dependency +used only to render historical `(LinkById: ...)` fields deterministically. +An entry may carry `omitted_optional_defaults` containing `revoked` and/or +`x_mitre_remote_support` when the attested publication omitted those +false-valued defaults. This is a serialization-shape hint, not frozen STIX +content; all other fields still come from the exact persisted revision. ### Delete Specific Snapshot diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index e7c1ad27..8547fe1a 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -83,6 +83,7 @@ POST /api/release-tracks/:id/snapshots/:modified/clone DELETE /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/release POST /api/release-tracks/:id/snapshots/:modified/graph +POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct # admin recovery DELETE /api/release-tracks/:id/snapshots/:modified/graph ``` diff --git a/migrations/20260730230000-backfill-canonical-x-mitre-domains.js b/migrations/20260730230000-backfill-canonical-x-mitre-domains.js index 4ccc69d0..e2831c35 100644 --- a/migrations/20260730230000-backfill-canonical-x-mitre-domains.js +++ b/migrations/20260730230000-backfill-canonical-x-mitre-domains.js @@ -1,14 +1,17 @@ 'use strict'; /** - * Backfill canonical x_mitre_domains values for every latest domain-bearing - * ATT&CK object, then retire the validation bypasses that historically - * allowed domainless objects. + * Backfill canonical x_mitre_domains values where exact collection TOC + * provenance exists, then retire the historical validation bypasses only when + * no domainless objects remain. * - * Domain membership is inferred from the canonical ATT&CK collection - * provenance already persisted on each object revision. This keeps the - * migration release-agnostic while preserving multi-domain unions. Objects - * without mappable provenance default to Enterprise. + * Domain membership is inferred from exact object pins in canonical ATT&CK + * collection TOCs. `workspace.collections` is deliberately not authoritative: + * legacy imports attached it to secondary graph objects as well as primary + * collection members. This keeps the migration release-agnostic while + * preserving real multi-domain unions. Objects without mappable TOC + * provenance are left unchanged: absence of evidence is not evidence of + * Enterprise membership. * * Active latest revisions use the ordinary POST/create service pipeline so * validation, lifecycle hooks, events, release-track member sync, and audit @@ -24,6 +27,7 @@ */ const mongoose = require('mongoose'); +const _ = require('lodash'); const config = require('../app/config/config'); const { createAutomationRunRecorder, @@ -50,7 +54,6 @@ const TARGET_TYPES = [ 'x-mitre-tactic', ]; const TARGET_TYPE_SET = new Set(TARGET_TYPES); -const DEFAULT_DOMAINS = ['enterprise-attack']; const BATCH_SIZE = 50; const ACTIVE_CONCURRENCY = 4; const SERIAL_ACTIVE_TYPES = new Set([ @@ -111,13 +114,39 @@ function hasCanonicalDomains(document) { return Array.isArray(document?.stix?.x_mitre_domains) && document.stix.x_mitre_domains.length > 0; } -function domainsFromCollectionProvenance(document) { - const collectionRefs = new Set( - (document?.workspace?.collections || []).map((collection) => collection?.collection_ref), +function canonicalRevisionKey(stixId, modified) { + return `${stixId}\0${new Date(modified).toISOString()}`; +} + +async function buildCanonicalTocDomainIndex(db) { + const collectionDocuments = await db + .collection('attackObjects') + .find({ + 'stix.id': { $in: [...CANONICAL_COLLECTION_DOMAINS.keys()] }, + 'stix.type': 'x-mitre-collection', + }) + .project({ 'stix.id': 1, 'stix.x_mitre_contents': 1 }) + .toArray(); + const domainsByRevision = new Map(); + + for (const collection of collectionDocuments) { + const domain = CANONICAL_COLLECTION_DOMAINS.get(collection.stix.id); + for (const entry of collection.stix.x_mitre_contents || []) { + if (!entry?.object_ref || !entry?.object_modified) continue; + const key = canonicalRevisionKey(entry.object_ref, entry.object_modified); + const domains = domainsByRevision.get(key) || new Set(); + domains.add(domain); + domainsByRevision.set(key, domains); + } + } + + return new Map([...domainsByRevision].map(([key, domains]) => [key, [...domains].sort()])); +} + +function domainsFromCanonicalToc(document, domainsByRevision) { + return ( + domainsByRevision.get(canonicalRevisionKey(document.stix.id, document.stix.modified)) || [] ); - return [...CANONICAL_COLLECTION_DOMAINS] - .filter(([collectionRef]) => collectionRefs.has(collectionRef)) - .map(([, domain]) => domain); } function isInactive(document) { @@ -148,7 +177,7 @@ async function latestDomainlessTargetDocuments(db) { return documents.filter((document) => !hasCanonicalDomains(document)); } -function resolveCandidates(documents) { +function resolveCandidates(documents, domainsByRevision = new Map()) { const candidates = []; for (const document of documents) { @@ -158,12 +187,12 @@ function resolveCandidates(documents) { throw new Error(`Unsupported canonical-domain migration type: ${stixType}`); } - const provenanceDomains = domainsFromCollectionProvenance(document); - const hasProvenanceMapping = provenanceDomains.length > 0; + const provenanceDomains = domainsFromCanonicalToc(document, domainsByRevision); + if (provenanceDomains.length === 0) continue; candidates.push({ document, - domains: hasProvenanceMapping ? provenanceDomains : [...DEFAULT_DOMAINS], - domainSource: hasProvenanceMapping ? 'collection-provenance' : 'enterprise-default', + domains: provenanceDomains, + domainSource: 'canonical-collection-toc', lifecycle: isInactive(document) ? 'inactive' : 'active', }); } @@ -171,6 +200,70 @@ function resolveCandidates(documents) { return candidates; } +function normalizedRepairStix(stix) { + const normalized = JSON.parse(JSON.stringify(stix)); + delete normalized.modified; + delete normalized.x_mitre_domains; + delete normalized.x_mitre_attack_spec_version; + delete normalized.x_mitre_modified_by_ref; + if (normalized.revoked === false) delete normalized.revoked; + return normalized; +} + +function isDomainOnlySuccessor(document, predecessor) { + return _.isEqual(normalizedRepairStix(document.stix), normalizedRepairStix(predecessor.stix)); +} + +function normalizedDomains(value) { + return Array.isArray(value) ? [...new Set(value)].sort() : []; +} + +async function latestIncorrectTargetDocuments(db, domainsByRevision) { + const latestDocuments = await db + .collection('attackObjects') + .aggregate(latestTargetDocumentsPipeline()) + .toArray(); + const revisions = await db + .collection('attackObjects') + .find({ 'stix.id': { $in: latestDocuments.map((document) => document.stix.id) } }) + .sort({ 'stix.id': 1, 'stix.modified': -1 }) + .toArray(); + const revisionsById = new Map(); + for (const revision of revisions) { + const lineage = revisionsById.get(revision.stix.id) || []; + lineage.push(revision); + revisionsById.set(revision.stix.id, lineage); + } + + const candidates = []; + for (const document of latestDocuments) { + let domains = domainsFromCanonicalToc(document, domainsByRevision); + let domainSource = 'canonical-collection-toc'; + + if (domains.length === 0) { + const predecessor = (revisionsById.get(document.stix.id) || []) + .slice(1) + .find( + (revision) => + domainsFromCanonicalToc(revision, domainsByRevision).length > 0 && + isDomainOnlySuccessor(document, revision), + ); + if (!predecessor) continue; + domains = domainsFromCanonicalToc(predecessor, domainsByRevision); + domainSource = 'canonical-collection-toc-predecessor'; + } + + if (_.isEqual(normalizedDomains(document.stix.x_mitre_domains), domains)) continue; + candidates.push({ + document, + domains, + domainSource, + lifecycle: isInactive(document) ? 'inactive' : 'active', + }); + } + return candidates; +} + function ensureMongooseUsesClient(client) { if (client && mongoose.connection.readyState === 0) { mongoose.connection.setClient(client); @@ -249,7 +342,7 @@ function removeResolvedDomainValidation(workspace) { return replacement; } -async function repostActive(candidate, recorder) { +async function repostActive(candidate, recorder, migrationName = MIGRATION_NAME) { const { document, domains } = candidate; const service = serviceFor(document.stix.type); const modified = nextModifiedTimestamp(document.stix.modified); @@ -257,7 +350,7 @@ async function repostActive(candidate, recorder) { const created = await service.create(repost, { import: false, automationContext: { - automationName: MIGRATION_NAME, + automationName: migrationName, runId: recorder.runId, }, }); @@ -291,7 +384,7 @@ function prepareInactiveClone(candidate) { }; } -async function syncInactiveClone(candidate, result, recorder) { +async function syncInactiveClone(candidate, result, recorder, migrationName = MIGRATION_NAME) { const { document } = candidate; // The direct clone is intentionally not presented as a generic create. It // still advances any standard track that references this object, matching @@ -302,18 +395,23 @@ async function syncInactiveClone(candidate, result, recorder) { modifiedBy: 'system', trigger: document.stix.revoked === true ? 'revocation' : 'new-revision', automationContext: { - automationName: MIGRATION_NAME, + automationName: migrationName, runId: recorder.runId, }, }); } -async function processActiveBatch(candidates, recorder, concurrency) { +async function processActiveBatch( + candidates, + recorder, + concurrency, + migrationName = MIGRATION_NAME, +) { return mapWithConcurrency(candidates, concurrency, async (candidate) => { try { return { candidate, - result: await repostActive(candidate, recorder), + result: await repostActive(candidate, recorder, migrationName), }; } catch (error) { return { candidate, error }; @@ -321,7 +419,7 @@ async function processActiveBatch(candidates, recorder, concurrency) { }); } -async function processInactiveBatch(db, candidates, recorder) { +async function processInactiveBatch(db, candidates, recorder, migrationName = MIGRATION_NAME) { return mapWithConcurrency(candidates, ACTIVE_CONCURRENCY, async (candidate) => { try { const result = prepareInactiveClone(candidate); @@ -330,7 +428,7 @@ async function processInactiveBatch(db, candidates, recorder) { // the native driver performing the insert create its own ObjectId. const insertResult = await db.collection('attackObjects').insertOne(result.document); result.document._id = insertResult.insertedId; - await syncInactiveClone(candidate, result, recorder); + await syncInactiveClone(candidate, result, recorder, migrationName); return { candidate, result }; } catch (error) { return { candidate, error }; @@ -494,7 +592,6 @@ async function finalizeBatch(db, processed, recorder, counts, failures) { counts.updated++; if (lifecycle === 'active') counts.active_reposts++; else counts.inactive_clones++; - if (domainSource === 'enterprise-default') counts.enterprise_defaults++; if (document.stix.revoked === true) counts.revoked++; if (document.stix.x_mitre_deprecated === true) counts.deprecated++; auditItems.push(changedAuditItem(entry)); @@ -507,6 +604,11 @@ async function countRemainingDomainlessTargets(db) { return (await latestDomainlessTargetDocuments(db)).length; } +async function countRemainingIncorrectTargets(db) { + const domainsByRevision = await buildCanonicalTocDomainIndex(db); + return (await latestIncorrectTargetDocuments(db, domainsByRevision)).length; +} + async function countStaleDomainBypasses(db) { return db.collection('validationbypassrules').countDocuments({ fieldPath: ['x_mitre_domains'], @@ -523,12 +625,33 @@ async function removeStaleDomainBypasses(db) { }); } -async function run(db, client) { - const domainlessDocuments = await latestDomainlessTargetDocuments(db); +async function run(db, client, options = {}) { + const migrationName = options.migrationName || MIGRATION_NAME; + const correctIncorrect = options.correctIncorrect === true; + const domainsByRevision = await buildCanonicalTocDomainIndex(db); + const domainlessDocuments = correctIncorrect ? [] : await latestDomainlessTargetDocuments(db); + const incorrectCandidates = await latestIncorrectTargetDocuments(db, domainsByRevision); + const incorrectIds = new Set(incorrectCandidates.map((candidate) => candidate.document.stix.id)); + const unresolvedDomainless = correctIncorrect + ? [] + : domainlessDocuments.filter( + (document) => + !incorrectIds.has(document.stix.id) && + domainsFromCanonicalToc(document, domainsByRevision).length === 0, + ); + const candidates = [ + ...incorrectCandidates, + ...(correctIncorrect + ? [] + : resolveCandidates( + domainlessDocuments.filter((document) => !incorrectIds.has(document.stix.id)), + domainsByRevision, + )), + ]; const recorder = await createAutomationRunRecorder(db, { automationType: 'migration', - name: MIGRATION_NAME, + name: migrationName, trigger: { source: 'startup', runner: 'migrate-mongo' }, scope: { collections: ['attackObjects', 'validationbypassrules'], @@ -536,25 +659,26 @@ async function run(db, client) { target_types: TARGET_TYPES, }, metadata: { - domain_source: 'persisted-canonical-collection-provenance', + domain_source: 'exact-canonical-collection-toc-membership', canonical_collection_domains: Object.fromEntries(CANONICAL_COLLECTION_DOMAINS), - unmapped_default_domains: DEFAULT_DOMAINS, + unmapped_policy: 'leave-unchanged-and-retain-validation-bypasses', + correct_incorrect_successors: correctIncorrect, active_method: 'service-create', inactive_method: 'immutable-direct-clone', batch_size: BATCH_SIZE, active_concurrency: ACTIVE_CONCURRENCY, serialized_active_types: [...SERIAL_ACTIVE_TYPES], - latest_domainless_objects_discovered: domainlessDocuments.length, + candidates_discovered: candidates.length, }, }); const counts = { - scanned_candidates: domainlessDocuments.length, + scanned_candidates: candidates.length, active_reposts: 0, inactive_clones: 0, active_batches: 0, inactive_batches: 0, - enterprise_defaults: 0, + unmapped_skipped: unresolvedDomainless.length, revoked: 0, deprecated: 0, bypasses_removed: 0, @@ -566,9 +690,8 @@ async function run(db, client) { try { // Resolve the complete plan before deleting bypasses or creating object - // revisions. Persisted canonical collection provenance is authoritative - // when available; custom/unmapped content defaults to Enterprise. - const candidates = resolveCandidates(domainlessDocuments); + // revisions. Exact canonical collection TOC membership is authoritative; + // broad legacy collection-appearance backrefs are intentionally ignored. if (candidates.some((candidate) => candidate.lifecycle === 'active')) { ensureMongooseUsesClient(client); await assertOrganizationIdentityConfigured(); @@ -591,7 +714,12 @@ async function run(db, client) { size: batch.length, concurrency: ACTIVE_CONCURRENCY, }); - const processed = await processActiveBatch(batch, recorder, ACTIVE_CONCURRENCY); + const processed = await processActiveBatch( + batch, + recorder, + ACTIVE_CONCURRENCY, + migrationName, + ); await finalizeBatch(db, processed, recorder, counts, failures); } @@ -606,7 +734,7 @@ async function run(db, client) { concurrency: 1, stix_types: [...new Set(batch.map((candidate) => candidate.document.stix.type))], }); - const processed = await processActiveBatch(batch, recorder, 1); + const processed = await processActiveBatch(batch, recorder, 1, migrationName); await finalizeBatch(db, processed, recorder, counts, failures); } @@ -617,19 +745,21 @@ async function run(db, client) { size: batch.length, concurrency: ACTIVE_CONCURRENCY, }); - const processed = await processInactiveBatch(db, batch, recorder); + const processed = await processInactiveBatch(db, batch, recorder, migrationName); await finalizeBatch(db, processed, recorder, counts, failures); } const remainingDomainless = await countRemainingDomainlessTargets(db); - if (failures.length > 0 || remainingDomainless > 0) { + const remainingIncorrect = await countRemainingIncorrectTargets(db); + const remainingCandidates = remainingIncorrect; + if (failures.length > 0 || remainingCandidates > 0) { const failureSample = failures .slice(0, 5) .map((failure) => `${failure.stix_id}: ${failure.error}`) .join('; '); throw new Error( `Canonical-domain object repair is incomplete: ${failures.length} failed item(s), ` + - `${remainingDomainless} latest domainless target object(s). Validation bypasses ` + + `${remainingCandidates} remaining target object(s). Validation bypasses ` + `were retained.${failureSample ? ` Failures: ${failureSample}` : ''}`, ); } @@ -637,25 +767,40 @@ async function run(db, client) { // Enforcement is the final step. Leaving persisted bypasses in place until // every object is repaired prevents a partial run from activating a // stricter contract against data the same migration has not yet fixed. - const bypassResult = await removeStaleDomainBypasses(db); - counts.bypasses_removed = bypassResult.deletedCount; + if (remainingDomainless === 0) { + const bypassResult = await removeStaleDomainBypasses(db); + counts.bypasses_removed = bypassResult.deletedCount; + } verification = { remaining_latest_domainless_target_objects: remainingDomainless, + remaining_latest_incorrect_domain_objects: remainingIncorrect, remaining_domain_validation_bypasses: await countStaleDomainBypasses(db), }; - if (verification.remaining_domain_validation_bypasses > 0) { + if (remainingDomainless === 0 && verification.remaining_domain_validation_bypasses > 0) { throw new Error( `Canonical-domain enforcement is incomplete: ` + `${verification.remaining_domain_validation_bypasses} stale bypass(es).`, ); } + const warnings = + unresolvedDomainless.length === 0 + ? {} + : { + unmapped_domainless_objects: { + count: unresolvedDomainless.length, + sample: unresolvedDomainless.slice(0, 20).map((document) => document.stix.id), + message: + 'No exact canonical collection TOC membership was found; objects were left unchanged and domain validation bypasses were retained.', + }, + }; + await recorder.finish({ status: 'completed', counts, - warnings: {}, + warnings, verification, summary: { message: @@ -665,13 +810,16 @@ async function run(db, client) { errorSummary: null, }); - return { counts, verification }; + return { counts, warnings, verification }; } catch (error) { verification = { ...verification, remaining_latest_domainless_target_objects: verification.remaining_latest_domainless_target_objects ?? (await countRemainingDomainlessTargets(db).catch(() => null)), + remaining_latest_incorrect_domain_objects: + verification.remaining_latest_incorrect_domain_objects ?? + (await countRemainingIncorrectTargets(db).catch(() => null)), remaining_domain_validation_bypasses: verification.remaining_domain_validation_bypasses ?? (await countStaleDomainBypasses(db).catch(() => null)), @@ -709,11 +857,14 @@ module.exports = { TARGET_TYPES, chunkItems, countRemainingDomainlessTargets, + countRemainingIncorrectTargets, countStaleDomainBypasses, - domainsFromCollectionProvenance, + buildCanonicalTocDomainIndex, + domainsFromCanonicalToc, hasCanonicalDomains, isInactive, latestDomainlessTargetDocuments, + latestIncorrectTargetDocuments, mapWithConcurrency, nextModifiedTimestamp, prepareInactiveClone, diff --git a/migrations/20260803190000-correct-canonical-x-mitre-domains.js b/migrations/20260803190000-correct-canonical-x-mitre-domains.js new file mode 100644 index 00000000..d94822d7 --- /dev/null +++ b/migrations/20260803190000-correct-canonical-x-mitre-domains.js @@ -0,0 +1,34 @@ +'use strict'; + +/** + * Correct canonical-domain successor revisions created from legacy collection + * appearance metadata. + * + * The original backfill now uses exact canonical collection TOC membership. + * Deployments that already ran its earlier form may contain domain-only + * successor revisions with domains inherited from secondary bundle + * appearances. This forward migration recognizes only semantic domain-only + * successors whose historical predecessor has an exact canonical TOC pin and + * creates another immutable revision with that authoritative domain union. + */ + +const logger = require('../app/lib/logger'); +const canonicalDomainMigration = require('./20260730230000-backfill-canonical-x-mitre-domains'); + +const MIGRATION_NAME = '20260803190000-correct-canonical-x-mitre-domains'; + +module.exports = { + async up(db, client) { + const report = await canonicalDomainMigration._private.run(db, client, { + migrationName: MIGRATION_NAME, + correctIncorrect: true, + }); + logger.info(`[${MIGRATION_NAME}] ${JSON.stringify(report)}`); + }, + + async down() { + logger.info( + `[${MIGRATION_NAME}] down migration is a no-op: immutable correction revisions are retained`, + ); + }, +}; From 9e69977046864de531d8e7f7a86c512dc80b24ba Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:40:20 -0400 Subject: [PATCH 49/55] test(release-tracks): serialize snapshot timestamps in graph URLs Convert Mongoose Date values to ISO timestamps before URL encoding so the virtual graph regression addresses the intended conflict path. --- app/tests/api/release-tracks/virtual-graph-integrity.spec.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/tests/api/release-tracks/virtual-graph-integrity.spec.js b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js index 7c1e1c26..55f5be91 100644 --- a/app/tests/api/release-tracks/virtual-graph-integrity.spec.js +++ b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js @@ -177,7 +177,9 @@ describe('Virtual release-track graph integrity', function () { const draft = await dynamicRepo.getLatestSnapshot(virtual.id); expect(draft.graph_manifest_id).toBeUndefined(); await post( - `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(draft.modified)}/graph`, + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent( + new Date(draft.modified).toISOString(), + )}/graph`, {}, 409, ); From 756a0c235020ddfe39b605d978155ce586354cf6 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:29:19 -0400 Subject: [PATCH 50/55] feat(release-tracks): add editable snapshot notes Persist bounded snapshot-local descriptions across creation, materialization, and release workflows. Add an editor-authorized update endpoint with OpenAPI documentation and regression coverage. --- .../definitions/components/release-tracks.yml | 11 + app/api/definitions/openapi.yml | 3 + .../paths/release-tracks-paths.yml | 78 +++++- app/controllers/release-tracks-controller.js | 29 +++ .../release-tracks/release-track-schemas.js | 24 +- .../release-track-snapshot-schema.js | 4 + .../release-track-dynamic.repository.js | 1 + app/routes/release-tracks-routes.js | 8 + .../release-tracks/release-tracks-service.js | 8 + .../release-tracks/snapshot-service.js | 52 +++- .../release-tracks/versioning-service.js | 16 +- .../release-tracks/virtual-track-service.js | 5 +- .../snapshot-descriptions.spec.js | 222 ++++++++++++++++++ docs/developer/TODO.md | 10 + docs/developer/release-tracks/entities.md | 7 + .../release-tracks/implementation-notes.md | 4 + docs/user/release-tracks/api-reference.md | 41 +++- docs/user/release-tracks/virtual-tracks.md | 5 + 18 files changed, 514 insertions(+), 14 deletions(-) create mode 100644 app/tests/api/release-tracks/snapshot-descriptions.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index cbacc16d..4a652c52 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -30,6 +30,13 @@ components: Server-controlled identifier for an opt-in deterministic member graph on a tagged snapshot. Absent on drafts and graphless tagged snapshots. Clients should treat this value as opaque. + snapshot_description: + type: string + maxLength: 4000 + description: | + User-authored, snapshot-local notes. Editors may change this + workspace annotation without changing the snapshot identity, + release tag, contents, or deterministic graph. name: type: string pattern: '^[a-zA-Z0-9 &]+$' @@ -179,6 +186,10 @@ components: description: | High-level statistics for the materialized graph. Omitted when the snapshot does not reference a graph manifest. + snapshot_description: + type: string + maxLength: 4000 + description: 'User-authored notes attached only to this snapshot' name: type: string description: diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 502346c8..736d5c85 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -409,6 +409,9 @@ paths: /api/release-tracks/{id}/snapshots/{modified}/clone: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1clone' + /api/release-tracks/{id}/snapshots/{modified}/description: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1description' + /api/release-tracks/{id}/snapshots/{modified}/graph: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1graph' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 5d311459..eade09eb 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -236,6 +236,9 @@ paths: Virtual tracks may also accept a strict scheduled_materialization object containing schedule_mode and scheduled_for; it is persisted on the initial snapshot and returned by snapshot and track-list GETs. + `description` is long-lived track metadata. The optional + `snapshot_description` is a user-authored annotation on the initial + draft snapshot and is limited to 4000 characters. tags: - 'Release Tracks' # Request body validation moved to Zod in controller @@ -387,7 +390,8 @@ paths: `stix.modified` timestamp during release planning; tagged members always contain exact revision timestamps. Supply either `increment` (`major` or `minor`) or an explicit `version` in `MAJOR.MINOR` form, - but never both. Omitting both defaults to a minor increment. + but never both. Omitting both defaults to a minor increment. An + optional `description` is stored as snapshot-local release notes. tags: - 'Release Tracks' parameters: @@ -401,7 +405,8 @@ paths: description: | Version selection. `increment` and `version` are mutually exclusive; supplying both returns 400. An empty object defaults to a minor - increment. + increment. `description` optionally sets snapshot-local notes and is + limited to 4000 characters. content: application/json: schema: @@ -874,7 +879,8 @@ paths: does not re-resolve composition. Request body validated via Zod in controller. Clients may attach an optional strict scheduled_materialization object to the resulting - virtual draft. + virtual draft. `description` becomes the new snapshot's local notes; + it does not replace the release track description. tags: - 'Release Tracks' parameters: @@ -883,6 +889,19 @@ paths: required: true schema: type: string + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + description: + type: string + maxLength: 4000 + description: 'Optional notes for this snapshot' + scheduled_materialization: + $ref: '../components/release-tracks.yml#/components/schemas/scheduled-materialization' responses: '201': description: 'Virtual snapshot created successfully' @@ -1259,6 +1278,55 @@ paths: schema: $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' + /api/release-tracks/{id}/snapshots/{modified}/description: + put: + summary: 'Set or clear a snapshot description' + operationId: 'release-tracks-snapshot-description-update' + description: | + Replace the user-authored notes on one draft or tagged snapshot. + Whitespace is trimmed; an empty string clears the notes. This mutable + workspace annotation does not change the snapshot modified timestamp, + semantic version, tier contents, graph manifest, or release-track + metadata. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: modified + in: path + required: true + schema: + type: string + format: date-time + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - description + additionalProperties: false + properties: + description: + type: string + maxLength: 4000 + responses: + '200': + description: 'Snapshot description updated successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' + '400': + description: 'Invalid description payload' + '404': + description: 'Snapshot not found' + /api/release-tracks/{id}/snapshots/{modified}/graph: post: summary: 'Make a tagged snapshot member graph deterministic' @@ -1379,6 +1447,7 @@ paths: successful materialization. For standard tracks, dynamic staged references are resolved to exact object revisions when this release request is handled, including when the selected snapshot is historical. + An optional `description` is stored as snapshot-local release notes. tags: - 'Release Tracks' parameters: @@ -1397,7 +1466,8 @@ paths: description: | Version selection. `increment` and `version` are mutually exclusive; supplying both returns 400. An empty object defaults to a minor - increment. + increment. `description` optionally sets snapshot-local notes and is + limited to 4000 characters. content: application/json: schema: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index fbf51faf..3f501e60 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -38,6 +38,7 @@ const { createTrackBodySchema, createFromBundleBodySchema, updateMetadataBodySchema, + updateSnapshotDescriptionBodySchema, releaseBodySchema, releaseVersionSelectionSchema, cloneBodySchema, @@ -461,6 +462,34 @@ exports.updateMetadataByLatest = async function updateMetadataByLatest(req, res, } }; +/** PUT /api/release-tracks/:id/snapshots/:modified/description */ +exports.updateSnapshotDescription = async function updateSnapshotDescription(req, res, next) { + try { + const bodyResult = updateSnapshotDescriptionBodySchema.safeParse(req.body); + if (!bodyResult.success) { + return next( + new BadRequestError({ + message: 'Invalid snapshot description update', + details: bodyResult.error.errors, + }), + ); + } + + const result = await releaseTracksService.updateSnapshotDescription( + req.params.id, + req.params.modified, + bodyResult.data.description, + ); + logger.debug( + `Success: Updated description for snapshot ${req.params.modified} in track ${req.params.id}`, + ); + return res.status(200).send(result); + } catch (err) { + logger.error('Failed to update snapshot description: ' + err); + return next(err); + } +}; + /** POST /api/release-tracks/:id/snapshots/latest/release */ exports.releaseLatest = async function releaseLatest(req, res, next) { try { diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index ee114513..12d95569 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -73,6 +73,8 @@ const trackNameSchema = z message: 'Release track name may only contain alphanumeric characters, spaces, and ampersands', }); +const snapshotDescriptionSchema = z.string().trim().max(4000); + // ----------------------------------------------------------------------------- // Cron expression // See: https://github.com/colinhacks/zod/issues/4239#issuecomment-3161393771 @@ -392,6 +394,7 @@ const createTrackBodySchema = z .object({ name: trackNameSchema, description: z.string().optional(), + snapshot_description: snapshotDescriptionSchema.optional(), type: trackTypeQuerySchema.default('standard'), object_marking_refs: z.array(stixIdentifierSchema).optional(), composition: compositionSchema.optional(), @@ -431,6 +434,13 @@ const updateMetadataBodySchema = z.object({ object_marking_refs: z.array(stixIdentifierSchema).optional(), }); +/** PUT /release-tracks/:id/snapshots/:modified/description */ +const updateSnapshotDescriptionBodySchema = z + .object({ + description: snapshotDescriptionSchema, + }) + .strict(); + const releaseVersionSelectionSchema = z .object({ increment: releaseIncrementSchema.optional(), @@ -442,7 +452,16 @@ const releaseVersionSelectionSchema = z }); /** POST /release-tracks/:id/snapshots/{target}/release */ -const releaseBodySchema = releaseVersionSelectionSchema; +const releaseBodySchema = z + .object({ + increment: releaseIncrementSchema.optional(), + version: xMitreVersionSchema.optional(), + description: snapshotDescriptionSchema.optional(), + }) + .strict() + .refine((value) => !(value.increment && value.version), { + message: 'increment and version are mutually exclusive', + }); /** POST /release-tracks/:id/clone */ const cloneBodySchema = z @@ -516,7 +535,7 @@ const updateCompositionBodySchema = z /** POST /release-tracks/:id/virtual/snapshots/create */ const createVirtualSnapshotBodySchema = z .object({ - description: z.string().optional(), + description: snapshotDescriptionSchema.optional(), scheduled_materialization: scheduledMaterializationSchema.optional(), }) .strict() @@ -620,6 +639,7 @@ module.exports = { createTrackBodySchema, createFromBundleBodySchema, updateMetadataBodySchema, + updateSnapshotDescriptionBodySchema, releaseBodySchema, cloneBodySchema, addCandidatesBodySchema, diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index e6d6abf2..ab8545c3 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -365,6 +365,10 @@ const releaseTrackSnapshotDefinition = { validate: validateVersion, }, graph_manifest_id: { type: String }, + snapshot_description: { + type: String, + maxlength: [4000, 'Snapshot description cannot exceed 4000 characters'], + }, // Release track metadata name: { diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 45d5f59e..31ae0c22 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -289,6 +289,7 @@ class ReleaseTrackDynamicRepository { modified: 1, version: 1, graph_manifest_id: 1, + snapshot_description: 1, name: 1, description: 1, scheduled_materialization: 1, diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 51515dd0..16dc7ca9 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -241,6 +241,14 @@ router // Snapshot-specific read, release, clone, and deletion operations // ============================================================================= +router + .route('/release-tracks/:id/snapshots/:modified/description') + .put( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.updateSnapshotDescription, + ); + router .route('/release-tracks/:id/snapshots/:modified/release/preview') .get( diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 103819b5..e98a5078 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -328,6 +328,14 @@ exports.updateMetadata = function updateMetadata(trackId, updates, userId) { return snapshotService.updateMetadata(trackId, updates, userId); }; +exports.updateSnapshotDescription = function updateSnapshotDescription( + trackId, + modified, + description, +) { + return snapshotService.updateSnapshotDescription(trackId, modified, description); +}; + exports.cloneTrack = function cloneTrack(trackId, options) { return snapshotService.cloneTrack(trackId, options); }; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 9faf4c44..0dd2f698 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -137,7 +137,7 @@ exports.listTracks = async function listTracks(options) { /** * Create a new release track with an initial empty draft snapshot. * - * @param {Object} data - { name, description?, type, userAccountId?, object_marking_refs?, composition?, snapshot_schedule?, scheduled_materialization?, config? } + * @param {Object} data - { name, description?, snapshot_description?, type, userAccountId?, object_marking_refs?, composition?, snapshot_schedule?, scheduled_materialization?, config? } * @returns {Promise} The initial snapshot document */ exports.createTrack = async function createTrack(data) { @@ -152,6 +152,7 @@ exports.createTrack = async function createTrack(data) { version: null, name: data.name, description: data.description || '', + snapshot_description: data.snapshot_description || undefined, created: now, created_by_ref: data.userAccountId || undefined, object_marking_refs: data.object_marking_refs, @@ -222,6 +223,7 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { modified: snapshot.modified, version: snapshot.version, graph_manifest_id: snapshot.graph_manifest_id, + snapshot_description: snapshot.snapshot_description, graph_statistics: snapshot.graph_manifest_id ? graphStatisticsByManifestId.get(snapshot.graph_manifest_id) : undefined, @@ -301,14 +303,29 @@ exports.getSnapshotByModified = async function getSnapshotByModified(trackId, mo */ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, overrides) { const clone = deepClone(sourceSnapshot); + const hasSnapshotDescriptionOverride = Object.prototype.hasOwnProperty.call( + overrides || {}, + 'snapshot_description', + ); delete clone.graph_manifest_id; clone.modified = new Date(); clone.version = null; // clones are always drafts delete clone.scheduled_materialization; + // A rolling draft keeps its note as content changes replace that draft. A + // new release cycle cloned from a tagged snapshot starts without the prior + // release's note unless the caller explicitly supplies one. + if (sourceSnapshot.version != null && !hasSnapshotDescriptionOverride) { + delete clone.snapshot_description; + } + // Apply overrides if (overrides) { for (const [key, value] of Object.entries(overrides)) { + if (key === 'snapshot_description' && (value === undefined || value === '')) { + delete clone.snapshot_description; + continue; + } if (value !== undefined) { clone[key] = value; } @@ -386,6 +403,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { clone.created_by_ref = options.userAccountId || sourceSnapshot.created_by_ref; clone.version_history = []; delete clone.scheduled_materialization; + delete clone.snapshot_description; const normalized = tierRevisionInvariant.normalizeSnapshot(clone); await primaryRevisionService.assertStoredEntries( @@ -453,6 +471,38 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId return exports.cloneSnapshot(trackId, source, overrides); }; +/** + * Set or clear a snapshot-local description without changing its identity, + * release tag, members, or release-track registry metadata. + * + * Snapshot descriptions are editable workspace annotations rather than + * versioned publication content, so tagged and draft snapshots are both valid + * targets. + * + * @param {string} trackId + * @param {string|Date} modified + * @param {string} description + * @returns {Promise} + */ +exports.updateSnapshotDescription = async function updateSnapshotDescription( + trackId, + modified, + description, +) { + await exports.getSnapshotByModified(trackId, modified); + const update = description + ? { $set: { snapshot_description: description } } + : { $unset: { snapshot_description: '' } }; + const updated = await dynamicRepo.updateSnapshot(trackId, modified, update); + + if (!updated) { + throw new NotFoundError({ + details: `Snapshot with modified '${modified}' not found for track '${trackId}'`, + }); + } + return updated.toObject ? updated.toObject() : updated; +}; + // ============================================================================= // Configuration // ============================================================================= diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index f4c2870a..cef48e50 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -181,13 +181,23 @@ function planRelease( additionalOps.members = mergedMembers; additionalOps.staged = []; } + const updatesSnapshotDescription = options.description !== undefined; + if (updatesSnapshotDescription && options.description) { + additionalOps.snapshot_description = options.description; + } const afterSnapshot = { ...snapshot, version, members: mergedMembers, + ...(updatesSnapshotDescription && options.description + ? { snapshot_description: options.description } + : {}), ...(snapshot.type === 'standard' ? { staged: [] } : {}), }; + if (updatesSnapshotDescription && !options.description) { + delete afterSnapshot.snapshot_description; + } const after = tierCounts(afterSnapshot); const changes = isVirtual ? virtualReleaseChanges(previousTaggedSnapshot, afterSnapshot) @@ -219,6 +229,7 @@ function planRelease( version, versionHistoryEntry, additionalOps, + clearSnapshotDescription: updatesSnapshotDescription && !options.description, normalizedRemoved: normalized.removed, blockingError, summary: { @@ -282,13 +293,16 @@ async function commitPlan(plan) { if (plan.blockingError) throw plan.blockingError; const obsoleteManifestId = plan.sourceSnapshot.graph_manifest_id; + const unsetOps = {}; + if (obsoleteManifestId) unsetOps.graph_manifest_id = ''; + if (plan.clearSnapshotDescription) unsetOps.snapshot_description = ''; const tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { version: plan.version, versionHistoryEntry: plan.versionHistoryEntry, additionalOps: plan.additionalOps, // Older deployments attached graphs to drafts. Releasing changes the // member set, so that legacy draft graph cannot describe the release. - unsetOps: obsoleteManifestId ? { graph_manifest_id: '' } : undefined, + unsetOps: Object.keys(unsetOps).length ? unsetOps : undefined, }); if (!tagged) { diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 05500382..d7487215 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -504,12 +504,9 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op quarantine: quarantined, composition_resolution: compositionResolution, scheduled_materialization: options.scheduledMaterialization, + snapshot_description: options.description, }; - if (options.description !== undefined) { - overrides.description = options.description; - } - let snapshot; try { snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides); diff --git a/app/tests/api/release-tracks/snapshot-descriptions.spec.js b/app/tests/api/release-tracks/snapshot-descriptions.spec.js new file mode 100644 index 00000000..a91803fc --- /dev/null +++ b/app/tests/api/release-tracks/snapshot-descriptions.spec.js @@ -0,0 +1,222 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); + +describe('Release-track snapshot descriptions', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 200) { + return (await api('post', path, body, status)).body; + } + + async function put(path, body, status = 200) { + return (await api('put', path, body, status)).body; + } + + async function get(path, status = 200) { + return (await api('get', path, undefined, status)).body; + } + + async function createTrack(name, extra = {}) { + return post('/api/release-tracks/new', { name, type: 'standard', ...extra }, 201); + } + + function descriptionPath(track) { + return `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(track.modified)}/description`; + } + + it('sets, trims, lists, and clears a draft snapshot description without changing track metadata', async function () { + const track = await createTrack('Snapshot Description Draft', { + description: 'Long-lived track purpose', + snapshot_description: ' Initial analyst context. ', + }); + + expect(track.snapshot_description).toBe('Initial analyst context.'); + + const updated = await put(descriptionPath(track), { + description: ' Analyst context for this draft. ', + }); + + expect(updated).toMatchObject({ + id: track.id, + modified: track.modified, + version: null, + description: 'Long-lived track purpose', + snapshot_description: 'Analyst context for this draft.', + }); + + const history = await get(`/api/release-tracks/${track.id}/snapshots`); + expect(history.data[0]).toMatchObject({ + modified: track.modified, + description: 'Long-lived track purpose', + snapshot_description: 'Analyst context for this draft.', + }); + + const cleared = await put(descriptionPath(track), { description: ' ' }); + expect(cleared).not.toHaveProperty('snapshot_description'); + const unchanged = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(unchanged.description).toBe('Long-lived track purpose'); + expect(unchanged.modified).toBe(track.modified); + }); + + it('sets release notes while tagging and permits later annotation edits in place', async function () { + const track = await createTrack('Snapshot Description Release', { + description: 'Stable track description', + }); + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + version: '1.0', + description: 'What changed in the first publication.', + }); + + expect(released).toMatchObject({ + modified: track.modified, + version: '1.0', + description: 'Stable track description', + snapshot_description: 'What changed in the first publication.', + }); + + const edited = await put(descriptionPath(released), { + description: 'Corrected internal release context.', + }); + expect(edited).toMatchObject({ + modified: track.modified, + version: '1.0', + snapshot_description: 'Corrected internal release context.', + }); + + const registry = await get('/api/release-tracks'); + const registryTrack = registry.data.find((entry) => entry.track_id === track.id); + expect(registryTrack.description).toBe('Stable track description'); + }); + + it('clears existing draft notes when release explicitly supplies an empty description', async function () { + const track = await createTrack('Snapshot Description Release Clear', { + snapshot_description: 'Temporary draft context', + }); + + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + version: '1.0', + description: ' ', + }); + + expect(released).not.toHaveProperty('snapshot_description'); + }); + + it('preserves notes within a rolling draft and clears them for the next release cycle', async function () { + const initial = await createTrack('Snapshot Description Lifecycle'); + await put(descriptionPath(initial), { description: 'Notes for release 1.0' }); + + const rollingDraft = await post(`/api/release-tracks/${initial.id}/meta`, { + name: 'Snapshot Description Lifecycle Updated', + }); + expect(rollingDraft.snapshot_description).toBe('Notes for release 1.0'); + + const released = await post(`/api/release-tracks/${initial.id}/snapshots/latest/release`, { + version: '1.0', + }); + expect(released.snapshot_description).toBe('Notes for release 1.0'); + + const nextDraft = await post(`/api/release-tracks/${initial.id}/meta`, { + name: 'Snapshot Description Lifecycle Next', + }); + expect(nextDraft.version).toBeNull(); + expect(nextDraft).not.toHaveProperty('snapshot_description'); + }); + + it('stores virtual materialization descriptions as snapshot notes rather than track descriptions', async function () { + const component = await createTrack('Snapshot Description Component'); + await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { + version: '1.0', + }); + const virtual = await post( + '/api/release-tracks/new', + { + name: 'Snapshot Description Virtual', + description: 'Stable virtual track purpose', + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: component.id, + resolution_strategy: 'latest_tagged', + priority: 1, + }, + ], + deduplication: { strategy: 'prioritize_latest_object' }, + }, + }, + 201, + ); + + const materialized = await post( + `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, + { description: 'Q1 composition review and conflict decisions.' }, + 201, + ); + expect(materialized.description).toBe('Stable virtual track purpose'); + expect(materialized.snapshot_description).toBe('Q1 composition review and conflict decisions.'); + }); + + it('rejects malformed and oversized snapshot descriptions', async function () { + const track = await createTrack('Snapshot Description Validation'); + await api( + 'post', + '/api/release-tracks/new', + { + name: 'Snapshot Description Creation Validation', + type: 'standard', + snapshot_description: 'x'.repeat(4001), + }, + 400, + ); + await api('put', descriptionPath(track), { description: 'x'.repeat(4001) }, 400); + await api('put', descriptionPath(track), { description: 'valid', extra: true }, 400); + await api( + 'post', + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.0', description: 'x'.repeat(4001) }, + 400, + ); + }); + + it('returns not found when the selected snapshot does not exist', async function () { + const track = await createTrack('Snapshot Description Missing'); + await api( + 'put', + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + '2000-01-01T00:00:00.000Z', + )}/description`, + { description: 'Missing' }, + 404, + ); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 6db226ed..5ae1bafe 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -2203,3 +2203,13 @@ Verification (2026-07-30): - The 312 raw payload differences consist only of the expected canonical-domain repairs and domain-array ordering. After those agreed normalizations, zero payloads differ. + +## Snapshot descriptions + +- [x] Persist a bounded, snapshot-local description separately from release-track metadata. +- [x] Allow editors to set the description while materializing or tagging a snapshot and edit it later without changing snapshot identity or contents. +- [x] Return descriptions in snapshot history and Workbench snapshot responses. +- [x] Document the API and update the Bruno collection. +- [x] Add frontend creation, display, edit, clear, and feedback flows. +- [x] Add backend and frontend regression coverage. +- [x] Run focused tests and the complete backend and frontend verification suites. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index e352cf25..d64e7c2a 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -104,6 +104,7 @@ Each release track snapshot will be tracked as an individual MongoDB Document in // Snapshot metadata modified: "2024-01-15T16:20:00.000Z", // when the snapshot was created version: "18.0", // null if draft release + snapshot_description: "Why this snapshot matters to our team", // Release track metadata name: "ATT&CK Enterprise", @@ -204,6 +205,12 @@ Each release track snapshot will be tracked as an individual MongoDB Document in } ``` +`snapshot_description` is mutable workspace metadata stored directly on the +snapshot document. It is deliberately separate from the release track's +long-lived `description`. Editing it does not change `modified`, `version`, +tier contents, or an attached graph manifest. Rolling edits to the same draft +preserve its description; the first draft of a new release cycle starts blank. + ### Version History The `version_history` array tracks all tagged releases in reverse chronological order (newest first): diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 678bea53..226d8291 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -50,6 +50,10 @@ supported deployments. `version`, never both. Controller validation returns 400 at the HTTP boundary, and `version-utils.calculateNextVersion` repeats the invariant so internal release-planning callers cannot silently choose one selector. +- Snapshot descriptions are bounded to 4000 characters and are the narrow + mutable-metadata exception to snapshot content immutability. They are stored + as `snapshot_description` on the selected document and never update the + registry or the track-level `description`. ### ATT&CK canonical-domain migration diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index b5c70612..7eb31c99 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -60,6 +60,7 @@ GET /api/release-tracks/:id/snapshots/latest GET /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/release POST /api/release-tracks/:id/snapshots/:modified/clone +PUT /api/release-tracks/:id/snapshots/:modified/description DELETE /api/release-tracks/:id/snapshots/:modified ``` @@ -236,6 +237,7 @@ POST /api/release-tracks/new { "name": "Release Track Name", "description": "Description", + "snapshot_description": "Context for the initial draft", "type": "standard", "object_marking_refs": [], "config": { @@ -262,6 +264,10 @@ rules as [Update Configuration](#update-configuration), and the validated values are persisted on the initial draft snapshot. Omitted config fields use their model defaults. +`description` is long-lived track metadata. `snapshot_description` is an +optional, snapshot-local annotation for the initial draft and is limited to +4000 characters. + ### Bootstrap Release Track From Bundle Creates a new release track initialized with objects from a STIX bundle. This is useful for importing existing collections or bootstrapping from published ATT&CK releases. @@ -412,8 +418,9 @@ GET /api/release-tracks/:id/snapshots Filtering occurs before pagination, so `pagination.total` is the total number of snapshots matching `tagged`, not the total number in the track. -Every summary contains `id`, `type`, `modified`, `version`, `name`, -`description` (when set), and `members_count`. A tagged snapshot whose +Every summary contains `id`, `type`, `modified`, `version`, `name`, the +track-level `description` (when set), `snapshot_description` (when the snapshot +has user-authored notes), and `members_count`. A tagged snapshot whose deterministic member graph has been materialized also contains the opaque `graph_manifest_id` and `graph_statistics`; graphless snapshots omit both. Graph statistics describe the cached graph at a glance: @@ -444,6 +451,7 @@ Inapplicable count keys are omitted rather than returned as zero. "graph_manifest_id": "release-track-graph-manifest--01234567-89ab-4cde-8f01-23456789abcd", "name": "Enterprise ATT&CK", "description": "Enterprise domain release track", + "snapshot_description": "Reviewed publication for the Q1 threat model.", "members_count": 3247, "graph_statistics": { "primary_count": 3247, @@ -478,6 +486,25 @@ GET /api/release-tracks/:id/snapshots?tagged=true GET /api/release-tracks/:id/snapshots?tagged=false&limit=25&offset=25 ``` +### Update Snapshot Description + +Editors can attach or replace notes on any draft or tagged snapshot: + +``` +PUT /api/release-tracks/:id/snapshots/:modified/description +``` + +```json +{ + "description": "Reviewed publication for the Q1 threat model." +} +``` + +The value is trimmed and limited to 4000 characters. Send an empty string to +clear it. The API returns the updated snapshot as `snapshot_description` and +does not change the snapshot's `modified` timestamp, semantic version, tier +contents, graph cache, or the release track's long-lived description. + ### Update Metadata A user or team may wish to: @@ -519,6 +546,16 @@ set. Converts the latest draft snapshot to a tagged release. Tags the snapshot in-place (does not create new snapshot). Dynamically sets `x_mitre_version` based on the request body options. +The request may also include an optional `description` (up to 4000 characters) +to set the tagged snapshot's notes in the same operation: + +```json +{ + "increment": "minor", + "description": "Initial production release for the Q1 threat model." +} +``` + - If `version` is provided, uses that exact version (must be `X.Y` format) - If `increment` is provided, calculates the next `major` or `minor` version - `increment` and `version` are mutually exclusive; supplying both returns diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 34f121c3..60e193ce 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -481,6 +481,7 @@ POST /api/release-tracks/:id/virtual/snapshots/create "version": null, "name": "Enterprise ATT&CK", "description": "Virtual aggregation of Enterprise content", + "snapshot_description": "Q1 2024 Enterprise snapshot", "composition_resolution": { "resolved_at": "2024-03-01T10:00:00Z", @@ -522,6 +523,10 @@ POST /api/release-tracks/:id/virtual/snapshots/create ``` **Business Logic:** +The request `description` is stored as the snapshot-local +`snapshot_description`; it never replaces the virtual track's long-lived +description. + 1. For each component track in `composition.component_tracks`: - Resolve snapshot based on `resolution_strategy` - **Validate that resolved snapshot is tagged** (version !== null) From 85a011a750b792d0f3f9c3c61aae221ab9ce063d Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:58:49 -0400 Subject: [PATCH 51/55] fix(release-tracks): close deterministic graphs over members Select relationships through exact revision-pinned endpoints and require both endpoints to be snapshot members. Carry forward source-attested predecessor relationships while preventing secondary revision leakage and duplicate STIX objects. --- .../definitions/components/release-tracks.yml | 2 +- app/repository/relationships-repository.js | 42 +++ .../release-tracks/graph-manifest-service.js | 298 +++++++++++++++++- .../release-tracks/snapshot-service.js | 7 +- .../api/release-tracks/opt-in-graphs.spec.js | 200 +++++++++++- .../release-tracks-bundle.spec.js | 9 +- .../developer/release-tracks/bundle-export.md | 86 +++-- docs/developer/release-tracks/entities.md | 17 +- .../release-tracks/implementation-notes.md | 10 +- docs/user/release-tracks/api-reference.md | 30 +- docs/user/release-tracks/object-backrefs.md | 20 +- docs/user/release-tracks/output-formats.md | 32 +- docs/user/release-tracks/summary.md | 34 +- docs/user/release-tracks/virtual-tracks.md | 86 +++-- 14 files changed, 745 insertions(+), 128 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 4a652c52..e3e58778 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -131,7 +131,7 @@ components: secondary_count: type: integer minimum: 0 - description: 'Related objects reached while resolving the bounded member graph' + description: 'Source-attested historical non-member objects; zero for ordinary closed-member graphs' relationship_count: type: integer minimum: 0 diff --git a/app/repository/relationships-repository.js b/app/repository/relationships-repository.js index 1c5dc760..c601d736 100644 --- a/app/repository/relationships-repository.js +++ b/app/repository/relationships-repository.js @@ -155,6 +155,48 @@ class RelationshipsRepository extends BaseRepository { } } + /** + * Retrieve every relationship revision whose stored source or target pin + * exactly matches one of the supplied object revisions. + * + * The caller deliberately receives inactive and superseded relationship + * revisions. Deterministic graph capture must choose the newest revision + * for an exact endpoint pair before applying active/deprecated filters, or + * an older active revision could be resurrected. + */ + async retrieveRevisionsTouchingExactEndpoints(endpointRevisions, options = {}) { + if (!Array.isArray(endpointRevisions) || endpointRevisions.length === 0) return []; + + const batchSize = options.batchSize || 250; + const revisionsByKey = new Map(); + try { + for (let offset = 0; offset < endpointRevisions.length; offset += batchSize) { + const batch = endpointRevisions.slice(offset, offset + batchSize); + const exactEndpointQueries = batch.flatMap((entry) => { + const objectModified = new Date(entry.object_modified); + return [ + { + 'workspace.relationship_endpoints.source.object_ref': entry.object_ref, + 'workspace.relationship_endpoints.source.object_modified': objectModified, + }, + { + 'workspace.relationship_endpoints.target.object_ref': entry.object_ref, + 'workspace.relationship_endpoints.target.object_modified': objectModified, + }, + ]; + }); + const relationships = await this.model.find({ $or: exactEndpointQueries }).lean().exec(); + for (const relationship of relationships) { + const key = `${relationship.stix.id}::${new Date(relationship.stix.modified).getTime()}`; + revisionsByKey.set(key, relationship); + } + } + return [...revisionsByKey.values()]; + } catch (err) { + throw new DatabaseError(err); + } + } + async retrieveAllWithAttackURLInDescription() { const aggregation = [ { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js index 9720cc46..86db7537 100644 --- a/app/services/release-tracks/graph-manifest-service.js +++ b/app/services/release-tracks/graph-manifest-service.js @@ -16,7 +16,7 @@ const { ReleaseContentIntegrityError } = require('../../exceptions'); const primaryRevisionService = require('./primary-revision-service'); const MANIFEST_SCHEMA_VERSION = 2; -const RESOLVER_VERSION = 'bounded-member-graph-v2'; +const RESOLVER_VERSION = 'closed-member-graph-v3'; const SOURCE_BUNDLE_RESOLVER_VERSION = 'source-bundle-pointer-v2'; const TIERS = ['members', 'staged', 'candidates', 'quarantine']; const STATISTIC_FIELDS_BY_KIND = { @@ -178,10 +178,295 @@ async function resolveBoundedGraph(hydratedRoots, allowedDomains, missing) { } } +function endpointIsSelected(endpoint, membersByObjectRef) { + const member = endpoint && membersByObjectRef.get(endpoint.object_ref); + return ( + member && + revisionKey(member.object_ref, member.object_modified) === + revisionKey(endpoint.object_ref, endpoint.object_modified) + ); +} + +function exactMemberMap(entries) { + const membersByObjectRef = new Map(); + for (const entry of entries) { + const existing = membersByObjectRef.get(entry.object_ref); + if ( + existing && + revisionKey(existing.object_ref, existing.object_modified) !== + revisionKey(entry.object_ref, entry.object_modified) + ) { + throw new ReleaseContentIntegrityError( + [ + { + object_ref: entry.object_ref, + object_modified: new Date(entry.object_modified).toISOString(), + dependency: 'unique_member_revision', + }, + ], + { details: 'A deterministic snapshot cannot select two revisions of one STIX object.' }, + ); + } + membersByObjectRef.set(entry.object_ref, entry); + } + return membersByObjectRef; +} + +async function loadPredecessorRelationshipCandidates( + snapshot, + predecessorManifestId, + membersByObjectRef, +) { + if (!predecessorManifestId) return []; + + const predecessorManifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: predecessorManifestId, + track_id: snapshot.id, + snapshot_modified: { $lt: snapshot.modified }, + state: { $in: ['pending', 'active'] }, + }) + .lean() + .exec(); + if (!predecessorManifest) { + throw new ReleaseContentIntegrityError( + [{ manifest_id: predecessorManifestId, dependency: 'predecessor_graph_manifest' }], + { details: 'The preceding tagged snapshot references a missing graph manifest.' }, + ); + } + + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: predecessorManifestId, + kind: 'relationship', + }) + .lean() + .exec(); + const selectedEntries = entries.filter( + (entry) => + endpointIsSelected(entry.source, membersByObjectRef) && + endpointIsSelected(entry.target, membersByObjectRef), + ); + if (selectedEntries.length === 0) return []; + + const hydrated = await primaryRevisionService.assertStoredEntries(selectedEntries); + const documentsByRevision = new Map( + hydrated.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + const candidates = []; + for (const entry of selectedEntries) { + const relationship = documentsByRevision.get(entry.revision_key); + if ( + relationship?.stix.type !== 'relationship' || + relationship.stix.source_ref !== entry.source.object_ref || + relationship.stix.target_ref !== entry.target.object_ref + ) { + throw new ReleaseContentIntegrityError( + [{ object_ref: entry.object_ref, dependency: 'predecessor_relationship_pointer' }], + { details: 'A predecessor graph relationship no longer matches its stored endpoints.' }, + ); + } + candidates.push({ relationship, source: entry.source, target: entry.target }); + } + return candidates; +} + +async function resolveClosedMemberRelationships(snapshot, hydratedRoots, predecessorManifestId) { + const membersByObjectRef = exactMemberMap(hydratedRoots.entries); + const storedRelationships = await relationshipsRepository.retrieveRevisionsTouchingExactEndpoints( + hydratedRoots.entries, + ); + const candidatesByRevision = new Map(); + + for (const relationship of storedRelationships) { + const source = endpointFor(relationship, 'source'); + const target = endpointFor(relationship, 'target'); + if ( + !endpointIsSelected(source, membersByObjectRef) || + !endpointIsSelected(target, membersByObjectRef) + ) { + continue; + } + candidatesByRevision.set(revisionKey(relationship.stix.id, relationship.stix.modified), { + relationship, + source, + target, + }); + } + + const predecessorCandidates = await loadPredecessorRelationshipCandidates( + snapshot, + predecessorManifestId, + membersByObjectRef, + ); + for (const candidate of predecessorCandidates) { + const key = revisionKey(candidate.relationship.stix.id, candidate.relationship.stix.modified); + if (!candidatesByRevision.has(key)) candidatesByRevision.set(key, candidate); + } + + const candidatesByRelationship = new Map(); + for (const candidate of candidatesByRevision.values()) { + const entries = candidatesByRelationship.get(candidate.relationship.stix.id) || []; + entries.push(candidate); + candidatesByRelationship.set(candidate.relationship.stix.id, entries); + } + + const selected = []; + for (const [relationshipId, candidates] of candidatesByRelationship) { + const endpointPairs = new Set( + candidates.map( + ({ source, target }) => + `${revisionKey(source.object_ref, source.object_modified)}->${revisionKey( + target.object_ref, + target.object_modified, + )}`, + ), + ); + if (endpointPairs.size > 1) { + throw new ReleaseContentIntegrityError( + [{ object_ref: relationshipId, dependency: 'relationship_lineage_endpoints' }], + { + details: + 'One relationship lineage resolves to multiple endpoint pairs in the same member graph.', + }, + ); + } + + candidates.sort( + (left, right) => + new Date(right.relationship.stix.modified).getTime() - + new Date(left.relationship.stix.modified).getTime(), + ); + const newest = candidates[0]; + if ( + bundleRelationships.relationshipIsActive(newest.relationship) && + !bundleRelationships.isDeprecatedPattern(newest.relationship.stix) + ) { + selected.push(newest); + } + } + return selected; +} + +async function buildClosedMemberManifestEntries(snapshot, options) { + const rootRequests = (snapshot.members || []).map((entry) => ({ ...entry, tier: 'members' })); + const hydratedRoots = await primaryRevisionService.assertStoredEntries(rootRequests); + exactMemberMap(hydratedRoots.entries); + + const selectedRelationships = await resolveClosedMemberRelationships( + snapshot, + hydratedRoots, + options.predecessorManifestId, + ); + const relationshipDocuments = selectedRelationships.map((candidate) => candidate.relationship); + const graphResolver = new BundleGraphResolver({ + attackObjectsRepository, + detectionStrategiesRepository, + repositoryMap: primaryRevisionService.getRepositoryMap(), + policy: { + isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, + relationshipIsActive: bundleRelationships.relationshipIsActive, + secondaryObjectIsValid: () => false, + }, + options: { + inferDomains: false, + includeRevoked: true, + includeDeprecated: true, + includeMissingAttackId: true, + }, + relationships: relationshipDocuments, + prefetchedDocuments: hydratedRoots.documents, + }); + const supportingDocuments = await graphResolver.loadSupportingDocuments([ + ...hydratedRoots.documents.map((document) => document.stix), + ...relationshipDocuments.map((document) => document.stix), + ]); + + const selectedObjectRefs = new Set(hydratedRoots.entries.map((entry) => entry.object_ref)); + const rootMetadata = new Map( + hydratedRoots.entries.map((entry) => [ + revisionKey(entry.object_ref, entry.object_modified), + entry, + ]), + ); + const supportingByObjectRef = new Map(); + for (const document of supportingDocuments) { + if (!selectedObjectRefs.has(document.stix.id)) { + supportingByObjectRef.set(document.stix.id, document); + } + } + + const selectedByAttackId = new Map(); + for (const document of hydratedRoots.documents) { + const attackId = linkById.getAttackId(document.stix); + if (attackId) selectedByAttackId.set(attackId, document); + } + const linkTargets = new Map(); + for (const document of [...hydratedRoots.documents, ...relationshipDocuments]) { + for (const attackId of linkById.extractLinkByIds(document.stix)) { + if (selectedByAttackId.has(attackId) || linkTargets.has(attackId)) continue; + const target = await linkById.getAttackObjectFromDatabase(attackId); + if (target) linkTargets.set(attackId, target); + } + } + + const entries = hydratedRoots.documents.map((document) => { + const key = revisionKey(document.stix.id, document.stix.modified); + const root = rootMetadata.get(key); + return { + revision_key: key, + kind: 'root', + tier: 'members', + object_status: root?.object_status, + object_ref: document.stix.id, + object_modified: document.stix.modified, + }; + }); + for (const candidate of selectedRelationships) { + entries.push({ + revision_key: revisionKey( + candidate.relationship.stix.id, + candidate.relationship.stix.modified, + ), + kind: 'relationship', + object_ref: candidate.relationship.stix.id, + object_modified: candidate.relationship.stix.modified, + source: candidate.source, + target: candidate.target, + }); + } + for (const document of supportingByObjectRef.values()) { + const isVersioned = Boolean(document.stix.modified); + entries.push({ + revision_key: isVersioned + ? revisionKey(document.stix.id, document.stix.modified) + : `${document.stix.id}::unversioned`, + kind: 'supporting', + object_ref: document.stix.id, + object_modified: document.stix.modified, + frozen_stix: isVersioned ? undefined : document.stix, + }); + } + for (const document of linkTargets.values()) { + entries.push({ + revision_key: revisionKey(document.stix.id, document.stix.modified), + kind: 'link_target', + object_ref: document.stix.id, + object_modified: document.stix.modified, + }); + } + return entries; +} + async function buildManifestEntries(snapshot, options = {}) { + if (options.memberOnly) { + return buildClosedMemberManifestEntries(snapshot, options); + } + const allowedDomains = virtualSnapshotDomains(snapshot); const rootRequests = []; - const rootTiers = options.memberOnly ? ['members'] : TIERS; + const rootTiers = TIERS; for (const tier of rootTiers) { for (const entry of snapshot[tier] || []) { rootRequests.push({ ...entry, tier }); @@ -239,7 +524,7 @@ async function buildManifestEntries(snapshot, options = {}) { object_status: root?.object_status, object_ref: document.stix.id, object_modified: document.stix.modified, - discovered_from: options.memberOnly ? undefined : discoverySources.get(key) || [], + discovered_from: discoverySources.get(key) || [], }); } for (const candidate of selectedRelationships) { @@ -256,7 +541,7 @@ async function buildManifestEntries(snapshot, options = {}) { // Live previews reuse the legacy replay selector, which carries the // request-local relationship payload without persisting it. Persisted // schema-v2 member manifests deliberately omit this field. - frozen_stix: options.memberOnly ? undefined : candidate.relationship.stix, + frozen_stix: candidate.relationship.stix, }); } for (const document of supportingDocuments) { @@ -288,7 +573,10 @@ async function prepare(snapshot, options = {}) { const schemaVersion = options.schemaVersion ?? MANIFEST_SCHEMA_VERSION; const memberOnly = schemaVersion >= MANIFEST_SCHEMA_VERSION; const resolverVersion = memberOnly ? RESOLVER_VERSION : 'bounded-attack-graph-v1'; - const entries = await buildManifestEntries(snapshot, { memberOnly }); + const entries = await buildManifestEntries(snapshot, { + memberOnly, + predecessorManifestId: options.predecessorManifestId, + }); const common = { manifest_id: manifestId, track_id: snapshot.id, diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 0dd2f698..4a730f76 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -608,7 +608,12 @@ async function createGraph(trackId, modified, prepareManifest, validateExisting) } exports.createGraph = function createLiveGraph(trackId, modified) { - return createGraph(trackId, modified, (snapshot) => graphManifestService.prepare(snapshot)); + return createGraph(trackId, modified, async (snapshot) => { + const predecessor = await dynamicRepo.getLatestTaggedSnapshotBefore(trackId, snapshot.modified); + return graphManifestService.prepare(snapshot, { + predecessorManifestId: predecessor?.graph_manifest_id, + }); + }); }; exports.reconstructGraph = function reconstructGraph(trackId, modified, plan) { diff --git a/app/tests/api/release-tracks/opt-in-graphs.spec.js b/app/tests/api/release-tracks/opt-in-graphs.spec.js index 01258437..14a90f07 100644 --- a/app/tests/api/release-tracks/opt-in-graphs.spec.js +++ b/app/tests/api/release-tracks/opt-in-graphs.spec.js @@ -14,6 +14,7 @@ const { } = require('../../../models/release-tracks/release-track-graph-manifest-model'); const relationshipsRepository = require('../../../repository/relationships-repository'); const AttackObject = require('../../../models/attack-object-model'); +const Relationship = require('../../../models/relationship-model'); const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -87,7 +88,7 @@ describe('Opt-in deterministic release-track graphs', function () { return post('/api/release-tracks/new', { name, type: 'standard' }); } - async function sourcePlan(primary, secondary, relationshipRevision) { + async function sourcePlan(primary, secondary, relationshipRevision, secondaryKind = 'secondary') { const supporting = await AttackObject.find({ 'stix.id': { $in: [primary.stix.created_by_ref, markingDefinitionId], @@ -111,7 +112,7 @@ describe('Opt-in deterministic release-track graphs', function () { omitted_optional_defaults: ['revoked'], }, { - kind: 'secondary', + kind: secondaryKind, object_ref: secondary.stix.id, object_modified: secondary.stix.modified, }, @@ -145,7 +146,7 @@ describe('Opt-in deterministic release-track graphs', function () { const secondary = await post('/api/techniques', technique('Opt-in Graph Secondary')); const originalRelationship = await post('/api/relationships', relationship(primary, secondary)); const track = await createTrack('Opt in Graph Track'); - const released = await releaseExactMembers(app, passportCookie, track.id, [primary]); + const released = await releaseExactMembers(app, passportCookie, track.id, [primary, secondary]); expect(released).not.toHaveProperty('graph_manifest_id'); expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: track.id })).toBe(0); @@ -185,7 +186,7 @@ describe('Opt-in deterministic release-track graphs', function () { object_modified: expect.any(Date), }), expect.objectContaining({ - kind: 'secondary', + kind: 'root', object_ref: secondary.stix.id, object_modified: expect.any(Date), }), @@ -198,7 +199,8 @@ describe('Opt-in deterministic release-track graphs', function () { ); const relationshipEntry = entries.find((entry) => entry.kind === 'relationship'); expect(relationshipEntry).not.toHaveProperty('frozen_stix'); - for (const entry of entries.filter((item) => ['root', 'secondary'].includes(item.kind))) { + expect(entries.filter((entry) => entry.kind === 'secondary')).toHaveLength(0); + for (const entry of entries.filter((item) => item.kind === 'root')) { expect(entry.discovered_from).toBeUndefined(); } const markingEntry = entries.find((entry) => entry.object_ref === markingDefinitionId); @@ -259,6 +261,194 @@ describe('Opt-in deterministic release-track graphs', function () { expect(liveRelationship.description).toBe('New relationship revision'); }); + it('closes deterministic graphs over exact members without pulling secondary revisions', async function () { + const member = await post('/api/techniques', technique('Closed Graph Member')); + const outside = await post('/api/techniques', technique('Closed Graph Outside Object')); + const excludedRelationship = await post('/api/relationships', relationship(member, outside)); + const track = await createTrack('Closed Member Graph Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [member]); + + const graphSnapshot = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + {}, + ); + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: graphSnapshot.graph_manifest_id, + }) + .lean() + .exec(); + + expect(entries.filter((entry) => entry.kind === 'root')).toHaveLength(1); + expect(entries.some((entry) => entry.object_ref === outside.stix.id)).toBe(false); + expect(entries.some((entry) => entry.object_ref === excludedRelationship.stix.id)).toBe(false); + expect(entries.some((entry) => entry.kind === 'secondary')).toBe(false); + }); + + it('does not leak a newer endpoint revision or its remapped relationship', async function () { + const original = await post('/api/techniques', technique('Revision-pinned Graph Member')); + const peer = await post('/api/techniques', technique('Revision-pinned Graph Peer')); + const originalRelationship = await post('/api/relationships', relationship(original, peer)); + const track = await createTrack('Pinned Member Graph'); + const released = await releaseExactMembers(app, passportCookie, track.id, [original, peer]); + + const revisedPayload = structuredClone(original); + revisedPayload.stix.modified = new Date( + new Date(original.stix.modified).getTime() + 1000, + ).toISOString(); + revisedPayload.stix.description = 'A later revision that is not a snapshot member'; + const revised = await post('/api/techniques', revisedPayload); + + const advancedRelationship = await Relationship.findOne({ + 'stix.id': originalRelationship.stix.id, + 'workspace.relationship_endpoints.source.object_modified': revised.stix.modified, + }) + .sort({ 'stix.modified': -1 }) + .lean() + .exec(); + expect(advancedRelationship).toBeTruthy(); + + const graphSnapshot = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + {}, + ); + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: graphSnapshot.graph_manifest_id, + }) + .lean() + .exec(); + const objectEntries = entries.filter((entry) => + [original.stix.id, peer.stix.id].includes(entry.object_ref), + ); + const relationshipEntries = entries.filter( + (entry) => entry.object_ref === originalRelationship.stix.id, + ); + + expect(objectEntries).toHaveLength(2); + expect(objectEntries.every((entry) => entry.kind === 'root')).toBe(true); + expect( + objectEntries.find((entry) => entry.object_ref === original.stix.id).object_modified, + ).toEqual(new Date(original.stix.modified)); + expect(entries.some((entry) => entry.kind === 'secondary')).toBe(false); + expect(relationshipEntries).toHaveLength(1); + expect(relationshipEntries[0].object_modified).toEqual( + new Date(originalRelationship.stix.modified), + ); + + const bundle = ( + await authenticated( + request(app).get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle`, + ), + ).expect(200) + ).body; + expect(bundle.objects.filter((object) => object.id === original.stix.id)).toEqual([ + expect.objectContaining({ modified: original.stix.modified }), + ]); + expect( + bundle.objects.some( + (object) => + object.id === originalRelationship.stix.id && + object.modified === new Date(advancedRelationship.stix.modified).toISOString(), + ), + ).toBe(false); + }); + + it('does not resurrect an older active relationship when the newest exact revision is inactive', async function () { + const source = await post('/api/techniques', technique('Inactive Relationship Source')); + const target = await post('/api/techniques', technique('Inactive Relationship Target')); + const active = await post('/api/relationships', relationship(source, target)); + const inactivePayload = relationship(source, target, active); + inactivePayload.stix.x_mitre_deprecated = true; + const inactive = await post('/api/relationships', inactivePayload); + const track = await createTrack('Inactive Relationship Graph Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [source, target]); + + const graphSnapshot = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + {}, + ); + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: graphSnapshot.graph_manifest_id, + object_ref: active.stix.id, + }) + .lean() + .exec(); + + expect(inactive.stix.id).toBe(active.stix.id); + expect(entries).toHaveLength(0); + }); + + it('carries source-attested v19.1 relationship pins into the next member graph', async function () { + const source = await post('/api/techniques', technique('Predecessor Graph Source')); + const target = await post('/api/techniques', technique('Predecessor Graph Target')); + const relationshipRevision = await post('/api/relationships', relationship(source, target)); + const track = await createTrack('Predecessor Manifest Graph Track'); + const baseline = await releaseExactMembers(app, passportCookie, track.id, [source, target], { + version: '1.0', + }); + const plan = await sourcePlan(source, target, relationshipRevision, 'root'); + await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + baseline.modified, + )}/graph/reconstruct`, + plan, + ); + + const storedRelationship = await Relationship.findOne({ + 'stix.id': relationshipRevision.stix.id, + 'stix.modified': relationshipRevision.stix.modified, + }) + .lean() + .exec(); + await Relationship.collection.updateOne( + { _id: storedRelationship._id }, + { $unset: { 'workspace.relationship_endpoints': '' } }, + ); + + try { + await post(`/api/release-tracks/${track.id}/meta`, { description: 'v1.1 draft' }, 200); + const next = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.1' }, + 200, + ); + const graphSnapshot = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(next.modified)}/graph`, + {}, + ); + const carried = await ReleaseTrackGraphManifestEntry.findOne({ + manifest_id: graphSnapshot.graph_manifest_id, + object_ref: relationshipRevision.stix.id, + }) + .lean() + .exec(); + + expect(carried).toMatchObject({ + kind: 'relationship', + source: { + object_ref: source.stix.id, + object_modified: new Date(source.stix.modified), + }, + target: { + object_ref: target.stix.id, + object_modified: new Date(target.stix.modified), + }, + }); + expect(carried.object_modified).toEqual(new Date(relationshipRevision.stix.modified)); + } finally { + await Relationship.collection.updateOne( + { _id: storedRelationship._id }, + { + $set: { + 'workspace.relationship_endpoints': storedRelationship.workspace.relationship_endpoints, + }, + }, + ); + } + }); + it('rejects graph creation for an untagged snapshot', async function () { const track = await createTrack('Draft Graph Rejection'); await authenticated( diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index 9fbf65ca..f4dfdea2 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -11,8 +11,8 @@ * Covered behavior: * - Default bundle contains members only, plus referenced identities and * marking definitions (self-contained bundle) - * - Active relationships and their bounded secondary objects are frozen in - * a snapshot graph manifest + * - A deterministic snapshot graph contains active relationships only when + * both exact endpoint revisions are members * - `include` adds staged and/or candidate tiers (comma-separated or * repeated, singular or plural tier names) * - `state` narrows the included staged/candidate entries by workflow @@ -205,7 +205,7 @@ describe('Release Tracks Bundle Export API', function () { created: new Date().toISOString(), modified: new Date().toISOString(), name: 'Bundle Secondary Group', - description: 'A relationship-discovered secondary object.', + description: 'A member endpoint for relationship graph tests.', spec_version: '2.1', type: 'intrusion-set', object_marking_refs: [staticMarkingDefinitionId], @@ -252,6 +252,7 @@ describe('Release Tracks Bundle Export API', function () { memberObject, linkedMemberObject, relationshipSource, + secondaryGroup, ]); taggedModified = tagged.modified; await postAction( @@ -438,7 +439,7 @@ describe('Release Tracks Bundle Export API', function () { created: timestamp, modified: timestamp, name: 'Graph protection cascade fixture', - description: 'Attempts to cascade-delete a protected secondary object.', + description: 'Attempts to cascade-delete a protected graph member.', x_mitre_version: '1.0', x_mitre_contents: [ { diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index c13efb49..b99027d2 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -106,10 +106,11 @@ STIX version serialization. The pipeline: the tagged snapshot has explicitly opted in. Graphless snapshots resolve a live bounded graph. Any request that includes `staged` or `candidates` is also live; determinism is promised for `members` only. -3. **Bounded secondary selection** — graph resolution starts from the selected - roots and emits a relationship only when both exact endpoint revisions are - selected. Persisted schema-v2 manifests store exact-revision pointers, not - cloned STIX payloads. +3. **Closed member graph** — persisted deterministic graphs emit only exact + `members` revisions as graph objects. A relationship is selected only when + both of its stored exact endpoint revisions are members; relationships do + not pull additional SDOs into the graph. Persisted schema-v2 manifests store + exact-revision pointers, not cloned STIX payloads. 4. **Supporting objects** — referenced identities and marking definitions are appended. Versioned supporting objects use pointers; unversioned marking definitions retain a frozen payload in persisted graphs. @@ -154,32 +155,35 @@ persist canonical domains so virtual composition, snapshot export, and ephemeral export observe the same membership. Because snapshot contents are explicitly curated, primary entries do **not** -receive the legacy attack-id / deprecated / revoked filters. Secondary graph -resolution retains the established bounded ATT&CK expansion rules. It is -frozen only when a tagged snapshot opts into a graph. +receive the legacy attack-id / deprecated / revoked filters. Graphless and +candidate/staged exports retain the established live bounded ATT&CK expansion +rules. A persisted deterministic member graph instead closes over `members` +and never discovers additional SDO revisions through relationships. -#### Relationship and secondary-object consistency boundary +#### Closed-member relationship consistency boundary -Release-track snapshots distinguish **primary** and **secondary** content: +Release-track exports distinguish persisted deterministic content from live +compatibility expansion: - Primary objects are explicit snapshot tier entries. Members and quarantine record exact `(object_ref, object_modified)` revisions. Standard candidates and staged entries may instead store `"latest"` and are resolved just in time when a draft export includes those tiers. -- Secondary objects are not snapshot members. They are discovered when the - graph is resolved because an exact-pinned SRO connects them to a primary, - the bounded ATT&CK rules identify a detection strategy, or the bundle needs - a supporting identity, marking definition, or LinkById render target. +- A persisted deterministic graph contains only `members` as graph objects. + Relationships, supporting identities/marking definitions, and non-emitted + LinkById targets are dependencies, not implicit membership. A relationship + endpoint outside `members` causes that relationship to be omitted. +- Graphless and candidate/staged exports remain live and may use the legacy + secondary-object expansion rules. They carry no determinism guarantee. Tagged standard membership is deterministic because release planning resolves staged selectors before promoting them to members. Virtual materialization likewise copies exact member revisions from tagged component snapshots and never follows a component's later `track_latest` candidate movement. -When a virtual component declares `filters.domains`, the same allowed-domain -set bounds relationship-discovered secondary objects during graph capture. -An explicitly domain-bearing secondary object from another domain is not -included merely because it has a relationship to an included primary root. -Domainless supporting metadata remains eligible. +When a virtual component declares `filters.domains`, virtual materialization +uses those filters to choose exact primary members. Deterministic graph capture +does not perform a second domain-inference pass: the materialized member set is +the complete SDO boundary. Domainless supporting metadata remains eligible. Every relationship revision stores server-controlled exact source and target pins under `workspace.relationship_endpoints`. These fields identify the @@ -190,11 +194,13 @@ updated pins rather than rewriting the older SRO. Snapshots are graphless by default. After tagging, an editor may call `POST /api/release-tracks/:id/snapshots/:modified/graph`. The service builds a -schema-v2 member graph, writes a pending manifest and decoupled entry rows, -rehydrates every pointer while those pending rows already protect deletion, -then atomically attaches the manifest ID to the still-tagged snapshot. Replay -can self-activate a complete linked pending manifest after an interrupted -activation. `DELETE` on the same graph resource detaches and removes it. +schema-v2 closed-member graph. It rejects duplicate member revisions for one +STIX ID, selects relationship revisions only when both exact endpoint pins are +members, writes a pending manifest and decoupled entry rows, rehydrates every +pointer while those pending rows already protect deletion, then atomically +attaches the manifest ID to the still-tagged snapshot. Replay can self-activate +a complete linked pending manifest after an interrupted activation. `DELETE` +on the same graph resource detaches and removes it. Historical baselines whose relationships predate endpoint-pin capture require a different, admin-only path: @@ -230,14 +236,32 @@ those false values and retain them. True values and every other payload difference remain significant. Ordinary release-track exports retain their existing serialization. -Graph creation uses an indexed relationship frontier rather than scanning all -relationships. It starts with member IDs, queries only current relationship -lineages touching the frontier, batch-hydrates exact endpoints by STIX type, -and repeats only when bounded resolution discovers another relevant object -ID. This retains secondary-to-secondary edges without rebuilding unrelated -database state. Incremental reuse from a previous snapshot is deliberately -deferred: an unchanged member set does not prove an unchanged graph because a -new relationship can connect to an old member. +Ordinary graph creation uses the compound indexes on +`workspace.relationship_endpoints.{source,target}` rather than scanning all +relationships. Exact member revisions are queried in bounded batches. A +candidate survives only when both exact endpoint pairs occur in `members`. +Candidates are then grouped by relationship lineage and exact endpoint pair; +the newest revision wins before revoked, deprecated, and obsolete-pattern +filters run, so an older active revision cannot be resurrected by a newer +inactive revision. + +The immediately preceding tagged graph also seeds relationship candidates +whose exact endpoints remain members. This creates a provenance chain from a +source-attested v19.1 baseline, including legacy relationships whose current +`workspace.relationship_endpoints` metadata cannot be reconstructed +truthfully. The indexed database query is still performed on every graph so a +new relationship connecting unchanged members is discovered. Current exact +relationship revisions override carried history; removed or revised member +endpoints naturally drop predecessor edges. + +Ordinary manifests created by this algorithm use resolver version +`closed-member-graph-v3`. Existing `bounded-member-graph-v2` manifests are not +rewritten in place. To repair an affected post-v19.1 graph, preserve the +source-attested v1.0 baseline, DELETE only the affected later snapshot's graph, +then POST that graph again. If the tagged snapshot's member pins are already +correct, deleting the snapshot itself is unnecessary; the recreated graph uses +v1.0 (or the immediately preceding tagged graph) as its predecessor. Published +artifacts produced from the removed graph must be regenerated. Active and pending manifests protect every exact versioned dependency from hard deletion. Persisted STIX content is globally immutable through PUT, diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index d64e7c2a..a41044b9 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -453,14 +453,15 @@ snapshots, and later component activity cannot change the persisted virtual snapshot. A tagged snapshot may optionally reference an internal schema-v2 member graph -manifest. `POST /api/release-tracks/:id/snapshots/:modified/graph` resolves the -bounded graph from `members` and stores exact-revision pointers for primary, -relationship, secondary, versioned supporting, and LinkById objects. Only -unversioned supporting objects such as marking definitions retain a frozen -payload. Drafts are always graphless. A tagged snapshot without a manifest is -exportable, but graph relationships and secondary objects are resolved live. -Exports that include `candidates` or `staged` are also live even when the -tagged snapshot has a member manifest. +manifest. `POST /api/release-tracks/:id/snapshots/:modified/graph` closes the +graph over exact `members` and stores exact-revision pointers for those roots, +relationships whose two endpoint revisions are members, versioned supporting +objects, and LinkById targets. Ordinary graphs contain no relationship-added +secondary SDOs. Only unversioned supporting objects such as marking definitions +retain a frozen payload. Drafts are always graphless. A tagged snapshot without +a manifest is exportable, but graph relationships and secondary objects are +resolved live. Exports that include `candidates` or `staged` are also live even +when the tagged snapshot has a member manifest. The three valid `snapshot_schedule` shapes are: diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 226d8291..884298ac 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -237,9 +237,13 @@ Snapshot retrieval never re-runs composition, so there is no `resolve` query parameter or `resolved_content` response wrapper. Workbench retrieval returns the persisted primary membership. Bundle export replays a graph only after a tagged snapshot explicitly opts in; otherwise it resolves the current bounded -graph. Relationship revisions carry server-controlled exact endpoint pins in -`workspace.relationship_endpoints`, and schema-v2 manifests reference those -exact revisions without emitting the internal fields in STIX output. +graph. Persisted graphs close over exact `members`: relationship revisions +carry server-controlled exact endpoint pins in +`workspace.relationship_endpoints` and are included only when both pinned +revisions are members. Schema-v2 manifests reference those exact revisions +without emitting the internal fields in STIX output. The preceding tagged +graph seeds still-valid relationship pointers so source-attested legacy +provenance can continue into later releases. Snapshot schedules use the same strict, mode-discriminated Zod schema at the controller and service boundaries. `manual` has no selector field, `cron` diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 7eb31c99..7b99500e 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -426,7 +426,9 @@ deterministic member graph has been materialized also contains the opaque Graph statistics describe the cached graph at a glance: - `primary_count`: member objects deliberately selected for the snapshot. -- `secondary_count`: related objects reached by graph resolution. +- `secondary_count`: source-attested historical non-member objects. Ordinary + deterministic member graphs report zero because relationships do not expand + SDO membership. - `relationship_count`: relationships connecting cached graph objects. - `supporting_count`: supporting identities and marking definitions. - `link_target_count`: objects pinned for deterministic LinkById expansion. @@ -455,11 +457,11 @@ Inapplicable count keys are omitted rather than returned as zero. "members_count": 3247, "graph_statistics": { "primary_count": 3247, - "secondary_count": 812, + "secondary_count": 0, "relationship_count": 6841, "supporting_count": 5, "link_target_count": 17, - "total_count": 10922 + "total_count": 10110 }, "staged_count": 18, "candidates_count": 5 @@ -699,10 +701,19 @@ DELETE /api/release-tracks/:id/snapshots/:modified/graph Only tagged snapshots may have graphs. POST resolves the snapshot's `members` into a pointer-only exact-revision manifest and returns `201`; repeating it is idempotent and returns `200`. DELETE removes the manifest and returns `204` -even when no graph exists. Graphless bundles resolve relationships and +even when no graph exists. Ordinary graph creation emits only member SDO +revisions and relationships whose two exact stored endpoint revisions are both +members. It never follows a relationship to add a secondary SDO or a newer +revision of an existing member. Graphless bundles resolve relationships and secondary objects live. Requests that include candidates or staged objects remain live even if the tagged snapshot has a graph. +When the immediately preceding tagged snapshot has a graph, its still-valid +relationship pointers seed the new graph. Current exact relationship revisions +are selected through indexed endpoint lookups and take precedence. This lets a +source-attested historical baseline anchor later releases without preventing +new relationships between unchanged members from being discovered. + User interfaces may present this operation as **caching the bundle**: a cached indicator means member-only bundle exports reuse the exact object and relationship revisions selected when the cache was created. This is not a @@ -1445,11 +1456,12 @@ retrieval never recomputes virtual composition. As long as the track does not acquire a newer snapshot, `/snapshots/latest` selects the same primary revision set, and `/snapshots/:modified` addresses that set explicitly. -Virtual snapshot persistence freezes primary membership, not the bounded -bundle graph. A tagged snapshot may opt into the graph separately through the -graph endpoint above. Until then, relationships and secondary objects resolve -live. Hard deletes of graph-pinned revisions return `409 Conflict`; every -STIX-changing PUT returns `409` regardless of graph state. +Virtual snapshot persistence freezes primary membership, not the bundle graph. +A tagged snapshot may opt into the graph separately through the graph endpoint +above. The deterministic graph is closed over those exact members; until it is +created, relationships and secondary objects resolve live. Hard deletes of +graph-pinned revisions return `409 Conflict`; every STIX-changing PUT returns +`409` regardless of graph state. Candidate and staged exports are intentionally live, including exact-selector entries, because determinism is guaranteed only for `members`. A `"latest"` diff --git a/docs/user/release-tracks/object-backrefs.md b/docs/user/release-tracks/object-backrefs.md index 83213514..6b8b6c23 100644 --- a/docs/user/release-tracks/object-backrefs.md +++ b/docs/user/release-tracks/object-backrefs.md @@ -24,11 +24,11 @@ scanning tracks. } ``` -| Field | Values | Meaning | -|-------|--------|---------| -| `id` | `release-track--` | The referencing release track | -| `type` | `standard`, `virtual` | The type of the referencing release track | -| `tier` | `members`, `staged`, `candidates`, `quarantine` | Which tier of the track references this revision; values match the snapshot tier array names | +| Field | Values | Meaning | +| -------- | ---------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | `release-track--` | The referencing release track | +| `type` | `standard`, `virtual` | The type of the referencing release track | +| `tier` | `members`, `staged`, `candidates`, `quarantine` | Which tier of the track references this revision; values match the snapshot tier array names | | `status` | `modified-in-place`, `work-in-progress`, `awaiting-review`, `reviewed` | Track-scoped workflow status (`modified-in-place` is retained for legacy data but is no longer produced because STIX revisions are immutable) | An object referenced by multiple tracks carries one entry per track. @@ -54,7 +54,7 @@ An object referenced by multiple tracks carries one entry per track. is, while an explicitly chosen `"latest"` selector still follows the newest revision because that behavior is inherent in the selector; use `?versions=all` to see membership across revisions. -- **Reflects the latest snapshot.** Backrefs mirror the track's *current* +- **Reflects the latest snapshot.** Backrefs mirror the track's _current_ (most recent) snapshot. Deleting the latest snapshot reverts backrefs to the previous snapshot's membership; deleting a track removes all of its entries. Entries written before the `type` field existed are backfilled @@ -91,9 +91,11 @@ Release tracks are never blind to changes in the objects they pin: (`revoked: true`); revision sync enrolls it as a candidate in tracks where the object is a member and moves candidate/staged pins to it. The revoking object and the `revoked-by` relationship are not direct track members. - Snapshot creation captures them as bounded secondary graph dependencies - when applicable; later member-only bundle export replays its exact revision - pointers. Unversioned marking definitions are the frozen-payload exception. + Live graphless exports may discover them through compatibility expansion. + An opt-in deterministic graph includes the relationship only when both + exact endpoint revisions are direct members; it never promotes an endpoint + to secondary membership. Unversioned marking definitions are the + frozen-payload exception. ## Lifecycle example diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index 463f93a3..4a8f43f9 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -40,6 +40,7 @@ shape for snapshot retrieval endpoints and is intended for the Workbench fronten ``` **Characteristics:** + - Preserves the release-track snapshot structure - Includes `members`, `staged`, `candidates`, and `quarantine` tier arrays when present - Member and quarantine `object_modified` values are exact timestamps. @@ -83,7 +84,7 @@ Standard STIX bundle format: { "type": "attack-pattern", "id": "attack-pattern--aaa", - "name": "Technique A", + "name": "Technique A" // ... STIX properties only, no workflow info } ] @@ -91,6 +92,7 @@ Standard STIX bundle format: ``` **Characteristics:** + - STIX compliant (2.1 by default; 2.0 via `stixVersion=2.0`). Per the STIX specifications, the bundle object carries `spec_version` only for STIX 2.0; STIX 2.1 bundles omit it and each object declares its own `spec_version`. @@ -101,7 +103,9 @@ Standard STIX bundle format: - `LinkById` tags in descriptions are converted to markdown citations - Drafts, graphless tagged snapshots, and every export that includes candidate or staged tiers resolve the bounded graph live. A tagged member-only export - is deterministic only after its snapshot opts into a graph manifest. + is deterministic only after its snapshot opts into a graph manifest. That + manifest is closed over exact members: relationships are included only when + both exact endpoint revisions are members, and do not add secondary SDOs. - Frontends may describe manifest creation as **caching the bundle**. The cache pins the exact member graph for repeatable export; it is not a general performance cache, and candidate or staged additions remain live. @@ -115,12 +119,12 @@ Standard STIX bundle format: **Bundle query parameters** (apply only when `format=bundle`): -| Parameter | Values | Default | Description | -|-----------|--------|---------|-------------| -| `include` | `staged`, `candidates` (comma-separated or repeated) | _(members only)_ | Additional tiers to include in the bundle alongside members | -| `state` | `work-in-progress`, `awaiting-review` (comma-separated or repeated) | _(no filter)_ | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included, irrespective of this parameter. Members are unaffected. | -| `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to | -| `includeToc` | `true`, `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) as the first object in the bundle | +| Parameter | Values | Default | Description | +| ------------- | ------------------------------------------------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `include` | `staged`, `candidates` (comma-separated or repeated) | _(members only)_ | Additional tiers to include in the bundle alongside members | +| `state` | `work-in-progress`, `awaiting-review` (comma-separated or repeated) | _(no filter)_ | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included, irrespective of this parameter. Members are unaffected. | +| `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to | +| `includeToc` | `true`, `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) as the first object in the bundle | Examples: @@ -170,6 +174,7 @@ collection-123/ ``` **Example Response:** + ```json { "format": "filesystemstore", @@ -177,21 +182,24 @@ collection-123/ "x-mitre-collection": [ { "filename": "x-mitre-collection--123.json", - "content": { /* STIX object */ } + "content": { + /* STIX object */ + } } ], "attack-pattern": [ { "filename": "attack-pattern--aaa.json", - "content": { /* STIX object */ } + "content": { + /* STIX object */ + } } ] } } ``` -> **NOTE**: The `filesystemstore` is still a *concept* that will need additional refinement before it can be implemented. We will need to figure out an optimal way to return JSON files to the user. Optionally, we can attempt to generate an archive and serialize it over the wire, though this may be slow and error prone. Additionally, we can allow users to specify an output path via S3, FTP, etc. - +> **NOTE**: The `filesystemstore` is still a _concept_ that will need additional refinement before it can be implemented. We will need to figure out an optimal way to return JSON files to the user. Optionally, we can attempt to generate an archive and serialize it over the wire, though this may be slow and error prone. Additionally, we can allow users to specify an output path via S3, FTP, etc. ### Format Usage diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index 8547fe1a..b78499d6 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -23,12 +23,14 @@ The existing Collections API has five major issues: The Release Tracks API supports two types of release tracks: **Standard Release Tracks** - Direct object lifecycle management (the traditional model) + - Manage objects through the candidate → staged → released workflow - Source of truth for specific object types or content domains - Create snapshots when objects are added/removed or configuration changes - Examples: "GroupsMonthly", "TechniquesQuarterly", "SoftwareBiannual" **Virtual Release Tracks** - Computed aggregations of other release tracks (NEW) + - Compose content from multiple standard tracks; virtual-track nesting is not supported - No duplicate object tracking - objects managed in source tracks only @@ -46,6 +48,7 @@ See [virtual-tracks.md](./virtual-tracks.md) for complete virtual track document ### 1. Unified API Structure **Old API:** + ``` GET /api/stix-bundles (ephemeral bundles) GET /api/collection-bundles (export) @@ -58,6 +61,7 @@ GET /api/collections/:id (retrieve) **New API V2 (partial preview):** The new API is still a work in progress. The source of truth is located in [api-reference.md](./api-reference.md). The following is a preview. If there are any discrepencies between what is shown here and what is shown in [api-reference.md](./api-reference.md), defer to the latter. + ``` # Ephemeral bundles (stateless) GET /api/release-tracks/ephemeral/:domain @@ -92,6 +96,7 @@ DELETE /api/release-tracks/:id/snapshots/:modified/graph We borrow heavily concepts from git. Snapshots are sort of like commits and tagged releases are like git tags. A release track contains snapshots: delta permutations that can be linearly tracked to deduce how the release track has evolved over time. A snapshot is generated every time a supported draft operation changes state, such as adding or promoting candidates, updating release-track configuration, or renaming the release track. **Snapshots** (like Git commits) + - Every supported modification creates a replacement draft snapshot - Identified by `stix.modified` timestamp - Immutable once created @@ -99,6 +104,7 @@ We borrow heavily concepts from git. Snapshots are sort of like commits and tagg - May be a **draft release** (untagged) or **tagged release** (has version number) **Tagged Releases** (like Git tags) + - Snapshots are tagged with `version`, which when exported/retrieved as a STIX bundle, will be expressed as `x_mitre_version`. Draft snapshots are denoted by the fact that their `version` key is set to `null`. - Uses MAJOR.MINOR versioning (not MAJOR.MINOR.PATCH), as specified by the [`x_mitre_version` ADM schema](https://github.com/mitre-attack/attack-data-model/blob/f249442b3588de9cca84b819d480306b106d2c1f/src/schemas/common/property-schemas/attack-versioning.ts#L21:L26) - Snapshots are tagged in-place (no duplicate data) @@ -110,11 +116,12 @@ We borrow heavily concepts from git. Snapshots are sort of like commits and tagg We use the preexisting object workflow statuses, `work-in-progress`, `awaiting-review`, and `reviewed`, to control each object's "standing" in a release track. There are three types of membership "standings": - 1. **Candidate**: When an object is first added to a release track, is it considered a candidate. It does not have full membership yet; if the snapshot were to be tagged and released right now, candidates would not be included. - 2. **Staged**: Once a candidate's workflow status meets the release track's ["candidacy threshold"](./release-workflow.md#candidacy-threshold-configuration) criteria, it will automatically become staged. Once the snapshot is tagged/released, staged objects will be included in the resultant bundle's `x_mitre_contents`. - 3. **Member**: Objects are considered "members" if they are "cooked" into the `x_mitre_contents` array of the current snapshot. These are considered already released. -This presents a tenable solution to the classic "STIX freeze" dilemma wherein editors cannot begin working on the next-*next* (e.g., v20) release until all objects in the next (e.g., v19) release have been released. Staged objects are locked in for the imminent release, but editors are free to continue iterating on future object changes and can queue them up as candidates without affecting the permutation that has already been staged for the imminent release. +1. **Candidate**: When an object is first added to a release track, is it considered a candidate. It does not have full membership yet; if the snapshot were to be tagged and released right now, candidates would not be included. +2. **Staged**: Once a candidate's workflow status meets the release track's ["candidacy threshold"](./release-workflow.md#candidacy-threshold-configuration) criteria, it will automatically become staged. Once the snapshot is tagged/released, staged objects will be included in the resultant bundle's `x_mitre_contents`. +3. **Member**: Objects are considered "members" if they are "cooked" into the `x_mitre_contents` array of the current snapshot. These are considered already released. + +This presents a tenable solution to the classic "STIX freeze" dilemma wherein editors cannot begin working on the next-_next_ (e.g., v20) release until all objects in the next (e.g., v19) release have been released. Staged objects are locked in for the imminent release, but editors are free to continue iterating on future object changes and can queue them up as candidates without affecting the permutation that has already been staged for the imminent release. Candidate requests may use `modified: "latest"` (or omit it) to create a dynamic workflow reference. That selector remains `"latest"` while the entry @@ -134,9 +141,13 @@ records the revision resolved by the commit itself. Snapshots are graphless by default. After tagging, callers may opt into a deterministic member graph with `POST .../snapshots/:modified/graph`. The graph -stores exact-revision pointers for relationships, secondary objects, -versioned supporting objects, and LinkById targets; unversioned marking -definitions are frozen by value. `DELETE` on the graph resource returns the +stores the exact `members` revisions plus pointer-only relationships whose two +exact endpoint revisions are both members. Relationships never pull secondary +SDOs or newer revisions into a deterministic graph. Versioned supporting +objects and LinkById targets are also pointers; unversioned marking definitions +are frozen by value. A new graph carries still-valid relationship pointers +from the preceding tagged graph, allowing a source-attested historical +baseline to anchor later releases. `DELETE` on the graph resource returns the snapshot to live graph resolution. Candidate/staged bundle additions are always live. The generated bundle-envelope ID itself is not stable. @@ -178,9 +189,9 @@ Snapshot tagged Each release track can set its own candidacy threshold: ```javascript -workspace.config.candidacy_threshold = "reviewed" // Default -workspace.config.candidacy_threshold = "awaiting-review" // Permissive -workspace.config.candidacy_threshold = "work-in-progress" // Very permissive +workspace.config.candidacy_threshold = 'reviewed'; // Default +workspace.config.candidacy_threshold = 'awaiting-review'; // Permissive +workspace.config.candidacy_threshold = 'work-in-progress'; // Very permissive ``` ### Multiple Output Formats @@ -192,17 +203,20 @@ workspace.config.candidacy_threshold = "work-in-progress" // Very permissive ### Release previews The default format provides a before/after summary: + ``` GET /api/release-tracks/:id/snapshots/latest/release/preview ?format=summary &increment=minor ``` + `format=filesystemstore` is reserved for future FileSystemStore export support and currently returns HTTP 501. Use `format=workbench` for the literal would-be snapshot or `format=bundle` for its publication representation. Previewing never persists. Commit whichever snapshot is latest when the release request is handled: + ``` POST /api/release-tracks/:id/snapshots/latest/release { diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 60e193ce..b57dd78f 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -5,6 +5,7 @@ Virtual release tracks are computed aggregations of standard release tracks. They provide a way to compose releases from multiple source tracks without duplicating object tracking, reducing mental overhead and storage requirements. **Key Characteristics:** + - Virtual tracks **compute** their contents from component standard tracks - Only reference **tagged snapshots** from standard tracks (never drafts) - Maintain their own **independent snapshot history and versioning** @@ -26,6 +27,7 @@ Virtual Track (aggregation): ``` **Workflow:** + 1. Each standard track releases independently on its own schedule 2. Enterprise virtual track snapshots twice yearly (Jan 1, July 1) 3. Each snapshot captures the **latest tagged release** from each component track @@ -197,6 +199,7 @@ not silently discarded. Virtual tracks **only sync from component tracks' `members` tier** (`x_mitre_contents`). This ensures that virtual tracks only aggregate objects that have been officially released in their source tracks. **Important:** + - Virtual tracks reference **tagged snapshots only** (never drafts) - Virtual tracks pull objects from **`members` tier only** (never staged or candidates) - This guarantees that virtual track releases are composed of stable, released content @@ -228,11 +231,12 @@ filter and a Mobile filter, while `["mobile-attack"]` is excluded by an Enterprise filter. Objects without `x_mitre_domains` are excluded when a domain filter is set. -The domain constraint also bounds the snapshot's publication graph. A -relationship cannot pull a secondary object with an explicit, nonmatching -`x_mitre_domains` value into the virtual bundle. Domainless identities, -marking definitions, and other supporting metadata may still be included -when referenced by an included object. +The domain constraint determines the virtual snapshot's exact member set. An +opt-in deterministic graph is closed over that set, so no relationship can +pull any secondary SDO into the virtual bundle. Graphless live exports retain +the compatibility domain check for relationship-discovered secondaries. +Domainless identities, marking definitions, and other supporting metadata may +still be included when referenced by an included object. `x_mitre_domains` is canonical object data. A cross-domain object has one revision containing the complete domain union; Workbench does not create or @@ -275,11 +279,12 @@ Keep the version with the newest `modified` timestamp, regardless of which compo ```javascript deduplication: { - strategy: "prioritize_latest_object" + strategy: 'prioritize_latest_object'; } ``` **Example:** + ```javascript // GroupsMonthly v5.2 has: // intrusion-set--APT1, modified: 2024-02-01T10:00:00Z @@ -300,11 +305,12 @@ Keep the version from the component track whose resolved snapshot has the newest ```javascript deduplication: { - strategy: "prioritize_latest_snapshot" + strategy: 'prioritize_latest_snapshot'; } ``` **Example:** + ```javascript // GroupsMonthly v5.2 // - Snapshot created: 2024-02-15T10:00:00Z @@ -349,6 +355,7 @@ composition: { ``` **Example:** + ```javascript // Authoritative track (priority: 1) has: // intrusion-set--APT1, modified: 2024-01-01T10:00:00Z @@ -377,11 +384,12 @@ entry for each distinct revision rather than one entry per component. ```javascript deduplication: { - strategy: "quarantine" + strategy: 'quarantine'; } ``` **Example:** + ```javascript // GroupsMonthly has: intrusion-set--APT1, modified: 2024-02-01 // MobileGroups has: intrusion-set--APT1, modified: 2024-01-15 @@ -440,13 +448,13 @@ Unlike standard release tracks (which use a three-tier system: candidates → st **Comparison to Standard Tracks:** -| Feature | Standard Track | Virtual Track | -|---------|---------------|---------------| -| Tiers | candidates, staged, members | quarantine, members | -| Object management | Direct (add/remove objects) | Indirect (synced from components) | -| Workflow states | work-in-progress, awaiting-review, reviewed | N/A | -| Auto-promotion | Based on candidacy threshold | N/A | -| Manual promotion | candidates → staged → members | quarantine → members | +| Feature | Standard Track | Virtual Track | +| ----------------- | ------------------------------------------- | --------------------------------- | +| Tiers | candidates, staged, members | quarantine, members | +| Object management | Direct (add/remove objects) | Indirect (synced from components) | +| Workflow states | work-in-progress, awaiting-review, reviewed | N/A | +| Auto-promotion | Based on candidacy threshold | N/A | +| Manual promotion | candidates → staged → members | quarantine → members | **Why only two tiers?** @@ -465,6 +473,7 @@ POST /api/release-tracks/:id/virtual/snapshots/create ``` **Request:** + ```json { "description": "Q1 2024 Enterprise snapshot" @@ -472,6 +481,7 @@ POST /api/release-tracks/:id/virtual/snapshots/create ``` **Response:** + ```json { "id": "release-track--uuid-virtual", @@ -609,6 +619,7 @@ GET /api/release-tracks/:id/snapshots/:modified?format=workbench&include=all ``` **Response includes:** + - All objects that will be in the release - Composition resolution details (which component versions were used) - The exact persisted members and quarantine tiers @@ -636,13 +647,15 @@ POST /api/release-tracks/:id/snapshots/:modified/release ``` **Request:** + ```json { - "increment": "major", // or "minor", or explicit "version": "14.0" + "increment": "major" // or "minor", or explicit "version": "14.0" } ``` **Response:** + ```json { "id": "release-track--uuid-virtual", @@ -680,6 +693,7 @@ produced its frozen contents. Standard release history entries omit this virtual-only property. **Business Logic:** + 1. Validate snapshot exists and is a draft (version === null) 2. Calculate/validate version number 3. Set version on snapshot (in-place update) @@ -700,6 +714,7 @@ GET /api/release-tracks/:id/snapshots/:modified?format=bundle&stixVersion=2.0 ``` **Response:** + ```json { "type": "bundle", @@ -717,7 +732,7 @@ GET /api/release-tracks/:id/snapshots/:modified?format=bundle&stixVersion=2.0 { "object_ref": "attack-pattern--T1234", "object_modified": "2024-01-10T10:00:00Z" } // ... all 870 objects ] - }, + } // ... all 870 actual STIX objects ] } @@ -811,13 +826,14 @@ for (const component of composition.component_tracks) { if (snapshot.version === null) { throw new ValidationError( `Component track ${component.track_id} resolved to draft snapshot. ` + - `Virtual tracks can only reference tagged snapshots.` + `Virtual tracks can only reference tagged snapshots.`, ); } } ``` **User experience:** + ```bash POST /api/release-tracks/release-track--uuid-virtual/virtual/snapshots/create @@ -840,10 +856,10 @@ async function validateComponentsAreStandard(virtualTrack) { for (const component of virtualTrack.composition.component_tracks) { const track = await getReleaseTrack(component.track_id); - if (track.type === "virtual") { + if (track.type === 'virtual') { throw new ValidationError( `Virtual tracks can only compose from standard tracks. ` + - `Component track ${component.track_id} is a virtual track.` + `Component track ${component.track_id} is a virtual track.`, ); } } @@ -859,6 +875,7 @@ POST /api/release-tracks/new ``` **Request:** + ```json { "type": "virtual", @@ -895,6 +912,7 @@ PUT /api/release-tracks/:id/virtual/composition ``` **Request:** + ```json { "component_tracks": [ @@ -935,6 +953,7 @@ POST /api/release-tracks/:id/virtual/snapshots/create ``` **Request:** + ```json { "description": "Q1 2024 snapshot" @@ -948,6 +967,7 @@ POST /api/release-tracks/:id/snapshots/:modified/release ``` **Request:** + ```json { "increment": "major" @@ -978,6 +998,7 @@ GET /api/release-tracks/:id/snapshots/latest?format=workbench&include=all ``` **Query params:** + - `format`: `bundle` | `workbench` | `filesystemstore` (`filesystemstore` is not yet implemented and returns HTTP 501) - `include`: `members` | `quarantine` | `all` @@ -1003,29 +1024,33 @@ Consequently, while the track does not acquire a newer snapshot, `latest` path segment selects the most recent snapshot; it is not a dynamic object-revision selector. -This guarantee also covers the bounded `format=bundle` object graph. -Relationship endpoint revisions, secondary objects, supporting objects, and -LinkById render targets are frozen only after a tagged snapshot opts into a -graph manifest; graphless snapshots resolve them live. +This guarantee also covers `format=bundle` after the tagged snapshot opts into +a graph manifest. The manifest emits only exact members plus relationships +whose two exact endpoint revisions are members; supporting objects and LinkById +render targets are pinned as dependencies. Graphless snapshots resolve the +legacy bounded graph live. Repeated exports may use a different bundle-envelope UUID, but replay the same snapshot object graph. See -[Bundle Export](../../developer/release-tracks/bundle-export.md#relationship-and-secondary-object-consistency-boundary). +[Bundle Export](../../developer/release-tracks/bundle-export.md#closed-member-relationship-consistency-boundary). ## Quarantine Management When using the `quarantine` deduplication strategy, conflicting objects are stored in the virtual track's `quarantine` tier. Users must manually resolve these conflicts: **View quarantined objects:** + ```bash GET /api/release-tracks/:id/snapshots/latest?include=quarantine ``` **Manually promote a quarantined object to members:** + ```bash POST /api/release-tracks/:id/virtual/quarantine/promote ``` **Request:** + ```json { "object_ref": "intrusion-set--11111111-1111-4111-8111-111111111111", @@ -1034,6 +1059,7 @@ POST /api/release-tracks/:id/virtual/quarantine/promote ``` **Effect:** + - Requires the exact `(object_ref, object_modified)` pair to be quarantined - Creates a new draft with the selected revision in `members` - Replaces any prior member revision with the same `object_ref` @@ -1156,7 +1182,7 @@ Resolve component tracks in parallel: const resolutions = await Promise.all( composition.component_tracks.map(async (component) => { return await resolveComponentSnapshot(component); - }) + }), ); ``` @@ -1226,10 +1252,10 @@ Add metadata to virtual track for documentation: ```javascript { - description: "Enterprise ATT&CK v14.0 includes:\n" + - "- Groups Monthly v1.3 (47 Groups)\n" + - "- Techniques Quarterly v2.1 (823 Techniques)\n" + - "- Software Biannual v1.0 (450 Software)" + description: 'Enterprise ATT&CK v14.0 includes:\n' + + '- Groups Monthly v1.3 (47 Groups)\n' + + '- Techniques Quarterly v2.1 (823 Techniques)\n' + + '- Software Biannual v1.0 (450 Software)'; } ``` From b000e0181bcdb270d49c49f8cea513958b326f8b Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:02:41 -0400 Subject: [PATCH 52/55] feat(release-tracks): bound snapshot publication versions Map snapshot descriptions onto exported collection TOCs. Enforce chronological release bounds and serialize release commits. --- .../definitions/components/release-tracks.yml | 8 +- .../paths/release-tracks-paths.yml | 15 +- app/lib/release-tracks/export-schemas.js | 3 +- app/lib/release-tracks/version-utils.js | 95 ++++++++++--- .../release-track-registry-model.js | 8 ++ .../release-track-registry.repository.js | 35 +++++ .../release-tracks/release-history-service.js | 5 +- .../release-tracks/versioning-service.js | 58 +++++++- .../release-tracks-bundle.spec.js | 17 +++ .../release-tracks-release.spec.js | 128 ++++++++++++++---- docs/developer/TODO.md | 35 +++++ .../developer/release-tracks/bundle-export.md | 5 +- .../release-tracks/implementation-notes.md | 9 +- docs/user/release-tracks/api-reference.md | 11 ++ docs/user/release-tracks/output-formats.md | 9 +- docs/user/release-tracks/versioning.md | 31 ++++- 16 files changed, 408 insertions(+), 64 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index e3e58778..ba356ae0 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -36,7 +36,9 @@ components: description: | User-authored, snapshot-local notes. Editors may change this workspace annotation without changing the snapshot identity, - release tag, contents, or deterministic graph. + release tag, contents, or deterministic graph. Bundle exports map + it to x-mitre-collection.description, falling back to the track + description when it is absent. name: type: string pattern: '^[a-zA-Z0-9 &]+$' @@ -189,7 +191,9 @@ components: snapshot_description: type: string maxLength: 4000 - description: 'User-authored notes attached only to this snapshot' + description: | + User-authored notes attached only to this snapshot and mapped to + x-mitre-collection.description during bundle export. name: type: string description: diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index eade09eb..bf9f5e29 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -392,6 +392,10 @@ paths: (`major` or `minor`) or an explicit `version` in `MAJOR.MINOR` form, but never both. Omitting both defaults to a minor increment. An optional `description` is stored as snapshot-local release notes. + Relative increments use the nearest earlier tagged snapshot. The + selected version must be strictly between the nearest earlier and + later tagged snapshots; the later bound is relevant to retroactive + releases. tags: - 'Release Tracks' parameters: @@ -418,7 +422,7 @@ paths: '400': description: 'Invalid release request' '409': - description: 'Already released, conflicting snapshot, or missing persisted primary revisions' + description: 'Already released, another release is in progress, conflicting snapshot, or missing persisted primary revisions' '500': description: 'The release may be tagged, but durable membership-protection reconciliation failed' content: @@ -439,6 +443,8 @@ paths: compare the persisted draft with its chronologically preceding tagged release; composition is never recomputed. A virtual draft whose composition has not been materialized returns 409. + Summary responses include the exclusive lower and upper tagged + snapshot version_bounds used by release planning. tags: - 'Release Tracks' parameters: @@ -1448,6 +1454,9 @@ paths: references are resolved to exact object revisions when this release request is handled, including when the selected snapshot is historical. An optional `description` is stored as snapshot-local release notes. + Relative increments use the nearest earlier tagged snapshot, and the + selected version must be strictly below the nearest later tagged + snapshot when one exists. tags: - 'Release Tracks' parameters: @@ -1479,7 +1488,7 @@ paths: '400': description: 'Invalid release request' '409': - description: 'Already released, conflicting snapshot, or missing persisted primary revisions' + description: 'Already released, another release is in progress, conflicting snapshot, or missing persisted primary revisions' '500': description: 'The release may be tagged, but durable membership-protection reconciliation failed' content: @@ -1498,6 +1507,8 @@ paths: timestamp precedes this selected snapshot; never recompute composition. An unmaterialized virtual draft returns 409. Standard previews resolve dynamic staged references exactly as the corresponding release would. + Summary responses include the exclusive lower and upper tagged + snapshot version_bounds used by release planning. tags: - 'Release Tracks' parameters: diff --git a/app/lib/release-tracks/export-schemas.js b/app/lib/release-tracks/export-schemas.js index 52f8ec58..34d3a488 100644 --- a/app/lib/release-tracks/export-schemas.js +++ b/app/lib/release-tracks/export-schemas.js @@ -35,6 +35,7 @@ const snapshotSchema = z.looseObject({ version: z.string().nullable().optional(), name: z.string(), description: z.string().optional(), + snapshot_description: z.string().optional(), created: z.date().or(z.string()).optional(), created_by_ref: z.string().optional(), object_marking_refs: z.array(z.string()).optional(), @@ -110,7 +111,7 @@ function buildTocObject(snapshot, bundleObjects, options) { x_mitre_attack_spec_version: options.attackSpecVersion, name: snapshot.name, x_mitre_version: snapshot.version || '0.1', - description: snapshot.description, + description: snapshot.snapshot_description ?? snapshot.description, created_by_ref: snapshot.created_by_ref || '', created: snapshot.created || snapshot.modified, modified: snapshot.modified, diff --git a/app/lib/release-tracks/version-utils.js b/app/lib/release-tracks/version-utils.js index f58fa57f..7e326e4b 100644 --- a/app/lib/release-tracks/version-utils.js +++ b/app/lib/release-tracks/version-utils.js @@ -46,7 +46,49 @@ exports.compareVersions = function compareVersions(a, b) { }; /** - * Calculate the next version based on version history and release increment. + * Find the exclusive semantic-version bounds around a snapshot timestamp. + * Tagged snapshots without modified metadata are treated as legacy lower + * bounds so internal callers using the older history shape remain safe. + * + * @param {Array<{ version: string, modified?: string|Date }>} versionHistory + * @param {string|Date} [sourceModified] + * @returns {{ lower: Object|null, upper: Object|null }} + */ +exports.findVersionBounds = function findVersionBounds(versionHistory, sourceModified) { + const history = versionHistory || []; + const sourceTime = sourceModified == null ? NaN : new Date(sourceModified).getTime(); + const timestamped = history.filter( + (entry) => entry.modified != null && !Number.isNaN(new Date(entry.modified).getTime()), + ); + + if (Number.isNaN(sourceTime) || timestamped.length !== history.length) { + let highest = null; + for (const entry of history) { + if (!highest || exports.compareVersions(entry.version, highest.version) > 0) { + highest = entry; + } + } + return { lower: highest, upper: null }; + } + + let lower = null; + let upper = null; + for (const entry of timestamped) { + const entryTime = new Date(entry.modified).getTime(); + if (entryTime < sourceTime && (!lower || entryTime > new Date(lower.modified).getTime())) { + lower = entry; + } + if (entryTime > sourceTime && (!upper || entryTime < new Date(upper.modified).getTime())) { + upper = entry; + } + } + + return { lower, upper }; +}; + +/** + * Calculate the next version based on the nearest chronologically preceding + * tagged snapshot and release increment. * * If an explicit version is provided, it is returned as-is (validation * is handled separately by validateVersionProgression). @@ -55,9 +97,10 @@ exports.compareVersions = function compareVersions(a, b) { * * If the version history is empty, the first version defaults to "1.0". * - * @param {Array<{ version: string }>} versionHistory - Existing version history entries + * @param {Array<{ version: string, modified?: string|Date }>} versionHistory - Existing tagged snapshots * @param {string} [increment='minor'] - 'major' or 'minor' * @param {string} [explicitVersion] - Explicit version override + * @param {string|Date} [sourceModified] - Snapshot being tagged * @returns {string} The calculated version string * @throws {InvalidVersionError} If both selectors are supplied or the explicit * version is invalid @@ -66,6 +109,7 @@ exports.calculateNextVersion = function calculateNextVersion( versionHistory, increment, explicitVersion, + sourceModified, ) { if (increment && explicitVersion) { throw new InvalidVersionError('increment and version are mutually exclusive'); @@ -77,44 +121,59 @@ exports.calculateNextVersion = function calculateNextVersion( return explicitVersion; } - if (!versionHistory || versionHistory.length === 0) { + const { lower } = exports.findVersionBounds(versionHistory, sourceModified); + if (!lower) { return '1.0'; } - // Find the highest existing version (history may not be sorted) - let highest = null; - for (const entry of versionHistory) { - if (!highest || exports.compareVersions(entry.version, highest) > 0) { - highest = entry.version; - } - } - - const { major, minor } = exports.parseVersion(highest); + const { major, minor } = exports.parseVersion(lower.version); const type = increment || 'minor'; return type === 'major' ? `${major + 1}.0` : `${major}.${minor + 1}`; }; /** - * Validate that a new version is strictly greater than all existing versions. + * Validate that a version is unique and lies strictly between the nearest + * tagged snapshots before and after the snapshot being released. * * @param {string} newVersion - The version to validate - * @param {Array<{ version: string }>} versionHistory - Existing version history entries - * @throws {InvalidVersionError} If the version is not greater than all existing versions + * @param {Array<{ version: string, modified?: string|Date }>} versionHistory - Existing tagged snapshots + * @param {string|Date} [sourceModified] - Snapshot being tagged + * @throws {InvalidVersionError} If the version is duplicated or outside its bounds */ exports.validateVersionProgression = function validateVersionProgression( newVersion, versionHistory, + sourceModified, ) { + exports.parseVersion(newVersion); if (!versionHistory || versionHistory.length === 0) { - return; // No history — any valid version is acceptable + return; } for (const entry of versionHistory) { - if (exports.compareVersions(newVersion, entry.version) <= 0) { + if (exports.compareVersions(newVersion, entry.version) === 0) { throw new InvalidVersionError( - `Version "${newVersion}" must be greater than existing version "${entry.version}"`, + `Version "${newVersion}" is already assigned to another snapshot in this release track`, ); } } + + const { lower, upper } = exports.findVersionBounds(versionHistory, sourceModified); + if (lower && upper && exports.compareVersions(lower.version, upper.version) >= 0) { + throw new InvalidVersionError( + `Cannot tag this snapshot because surrounding versions "${lower.version}" and ` + + `"${upper.version}" are not chronologically increasing`, + ); + } + if (lower && exports.compareVersions(newVersion, lower.version) <= 0) { + throw new InvalidVersionError( + `Version "${newVersion}" must be greater than preceding version "${lower.version}"`, + ); + } + if (upper && exports.compareVersions(newVersion, upper.version) >= 0) { + throw new InvalidVersionError( + `Version "${newVersion}" must be less than following version "${upper.version}"`, + ); + } }; diff --git a/app/models/release-tracks/release-track-registry-model.js b/app/models/release-tracks/release-track-registry-model.js index 833ad6ce..8fade81d 100644 --- a/app/models/release-tracks/release-track-registry-model.js +++ b/app/models/release-tracks/release-track-registry-model.js @@ -36,6 +36,13 @@ const taggedReleaseDefinition = { tagged_by: { type: String, required: true }, }; const taggedReleaseSchema = new mongoose.Schema(taggedReleaseDefinition, { _id: false }); +const releaseLockSchema = new mongoose.Schema( + { + token: { type: String, required: true }, + acquired_at: { type: Date, required: true }, + }, + { _id: false }, +); // --- Registry document definition --- @@ -68,6 +75,7 @@ const releaseTrackRegistryDefinition = { snapshot_count: { type: Number, default: 0 }, tagged_release_count: { type: Number, default: 0 }, tagged_releases: { type: [taggedReleaseSchema], default: [] }, + release_lock: { type: releaseLockSchema, default: undefined }, // Virtual tracks only snapshot_schedule: { diff --git a/app/repository/release-tracks/release-track-registry.repository.js b/app/repository/release-tracks/release-track-registry.repository.js index e7ef8c02..1c2d65a3 100644 --- a/app/repository/release-tracks/release-track-registry.repository.js +++ b/app/repository/release-tracks/release-track-registry.repository.js @@ -61,6 +61,8 @@ class ReleaseTrackRegistryRepository { }); } + aggregation.push({ $project: { release_lock: 0 } }); + // Total count before pagination const totalCountResult = await this.model.aggregate(aggregation).count('totalCount').exec(); const totalCount = totalCountResult[0]?.totalCount || 0; @@ -141,6 +143,39 @@ class ReleaseTrackRegistryRepository { } } + async acquireReleaseLock(trackId, token, acquiredAt, staleBefore) { + try { + return await this.model + .findOneAndUpdate( + { + track_id: trackId, + $or: [ + { release_lock: { $exists: false } }, + { 'release_lock.acquired_at': { $lt: staleBefore } }, + ], + }, + { $set: { release_lock: { token, acquired_at: acquiredAt } } }, + { new: true, runValidators: true, lean: true }, + ) + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async releaseReleaseLock(trackId, token) { + try { + return await this.model + .updateOne( + { track_id: trackId, 'release_lock.token': token }, + { $unset: { release_lock: '' } }, + ) + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async updateByTrackId(trackId, updates) { try { const result = await this.model diff --git a/app/services/release-tracks/release-history-service.js b/app/services/release-tracks/release-history-service.js index f920ba7b..6738c026 100644 --- a/app/services/release-tracks/release-history-service.js +++ b/app/services/release-tracks/release-history-service.js @@ -60,7 +60,10 @@ async function mapWithConcurrency(items, concurrency, mapper) { exports.getTrackWideVersionHistory = async function getTrackWideVersionHistory(trackId) { const snapshots = await dynamicRepo.getTaggedSnapshotMetadata(trackId); - return snapshots.map((snapshot) => ({ version: snapshot.version })); + return snapshots.map((snapshot) => ({ + version: snapshot.version, + modified: snapshot.modified, + })); }; exports.reconcileTaggedReleases = async function reconcileTaggedReleases(trackId) { diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index cef48e50..923eb855 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -13,13 +13,18 @@ const revisionReference = require('../../lib/release-tracks/revision-reference') const releaseHistoryService = require('./release-history-service'); const primaryRevisionService = require('./primary-revision-service'); const graphManifestService = require('./graph-manifest-service'); +const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); +const uuid = require('uuid'); const logger = require('../../lib/logger'); const { AlreadyReleasedError, ReleaseConflictError, + TrackNotFoundError, VirtualSnapshotNotMaterializedError, } = require('../../exceptions'); +const RELEASE_LOCK_TIMEOUT_MS = 15 * 60 * 1000; + function iso(value) { return new Date(value).toISOString(); } @@ -140,8 +145,10 @@ function planRelease( versionHistory, options.increment, options.version, + sourceSnapshot.modified, ); - versionUtils.validateVersionProgression(version, versionHistory); + versionUtils.validateVersionProgression(version, versionHistory, sourceSnapshot.modified); + const versionBounds = versionUtils.findVersionBounds(versionHistory, sourceSnapshot.modified); const isVirtual = snapshot.type === 'virtual'; const before = isVirtual @@ -237,6 +244,20 @@ function planRelease( type: snapshot.type, source_snapshot_modified: iso(sourceSnapshot.modified), version, + version_bounds: { + lower: versionBounds.lower + ? { + version: versionBounds.lower.version, + modified: iso(versionBounds.lower.modified), + } + : null, + upper: versionBounds.upper + ? { + version: versionBounds.upper.version, + modified: iso(versionBounds.upper.modified), + } + : null, + }, releasable: !blockingError, ...(isVirtual ? { @@ -339,6 +360,33 @@ async function commitPlan(plan) { return tagged; } +async function withReleaseLock(trackId, operation) { + const token = uuid.v4(); + const acquiredAt = new Date(); + const staleBefore = new Date(acquiredAt.getTime() - RELEASE_LOCK_TIMEOUT_MS); + const lock = await registryRepo.acquireReleaseLock(trackId, token, acquiredAt, staleBefore); + if (!lock) { + if (!(await registryRepo.findByTrackId(trackId))) { + throw new TrackNotFoundError(trackId); + } + throw new ReleaseConflictError('Another release operation is already in progress', { + track_id: trackId, + }); + } + + try { + return await operation(); + } finally { + try { + await registryRepo.releaseReleaseLock(trackId, token); + } catch (err) { + logger.error( + `VersioningService: Failed to release version lock for "${trackId}": ${err.message}`, + ); + } + } +} + exports.planRelease = planRelease; exports._private = { memberRevisions, @@ -362,9 +410,13 @@ exports.planReleaseByModified = async function planReleaseByModified( }; exports.releaseLatest = async function releaseLatest(trackId, options = {}) { - return commitPlan(await exports.planLatestRelease(trackId, options)); + return withReleaseLock(trackId, async () => + commitPlan(await exports.planLatestRelease(trackId, options)), + ); }; exports.releaseByModified = async function releaseByModified(trackId, modified, options = {}) { - return commitPlan(await exports.planReleaseByModified(trackId, modified, options)); + return withReleaseLock(trackId, async () => + commitPlan(await exports.planReleaseByModified(trackId, modified, options)), + ); }; diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index f4dfdea2..f095cbc0 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -231,6 +231,7 @@ describe('Release Tracks Bundle Export API', function () { { name: 'Bundle Test Track', description: 'Release track bundle export test', + snapshot_description: 'Virtual snapshot', type: 'standard', }, 201, @@ -329,6 +330,9 @@ describe('Release Tracks Bundle Export API', function () { expect(toc.type).toBe('x-mitre-collection'); expect(toc.id).toBe(`x-mitre-collection--${trackUuid}`); expect(toc.name).toBe('Bundle Test Track'); + // This rolling draft belongs to the next release cycle, so it has no + // snapshot-local description and falls back to the track description. + expect(toc.description).toBe('Release track bundle export test'); // Draft snapshots (version: null) fall back to '0.1' expect(toc.x_mitre_version).toBe('0.1'); expect(toc.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion); @@ -424,6 +428,19 @@ describe('Release Tracks Bundle Export API', function () { .expect(409); }); + it('maps a graph-backed snapshot description onto the collection TOC', async function () { + const bundle = await getBundle( + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( + taggedModified, + )}?format=bundle`, + ); + + expect(bundle.objects[0]).toMatchObject({ + type: 'x-mitre-collection', + description: 'Virtual snapshot', + }); + }); + it('protects graph dependencies from collection cascade deletion', async function () { const timestamp = new Date().toISOString(); const collection = await postObject('/api/collections', { diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 01139942..7a492aae 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -2,13 +2,11 @@ const request = require('supertest'); const { expect } = require('expect'); -const sinon = require('sinon'); const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); -const releaseHistoryService = require('../../../services/release-tracks/release-history-service'); const versioningService = require('../../../services/release-tracks/versioning-service'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const { releaseExactMembers } = require('./release-track-test-helpers'); @@ -174,27 +172,19 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version_history.at(-1)).not.toHaveProperty('component_versions'); }); - it('allows only one concurrent release to claim a version', async function () { + it('preserves not-found semantics when acquiring a release lock', async function () { + await post( + '/api/release-tracks/release-track--00000000-0000-4000-8000-000000000099/snapshots/latest/release', + {}, + 404, + ); + }); + + it('serializes concurrent release operations for one track', async function () { const track = await createTrack('Concurrent Release Version'); const newerDraft = await post(`/api/release-tracks/${track.id}/meta`, { description: 'A distinct draft racing for the same release version', }); - const originalHistoryLookup = releaseHistoryService.getTrackWideVersionHistory; - let waiting = 0; - let releaseBarrier; - const bothPlanned = new Promise((resolve) => { - releaseBarrier = resolve; - }); - const historyStub = sinon - .stub(releaseHistoryService, 'getTrackWideVersionHistory') - .callsFake(async (...args) => { - const history = await originalHistoryLookup(...args); - waiting += 1; - if (waiting === 2) releaseBarrier(); - await bothPlanned; - return history; - }); - const release = (modified) => request(app) .post(`/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(modified)}/release`) @@ -202,15 +192,10 @@ describe('Release-track release planning and commit API', function () { .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); - let responses; - try { - responses = await Promise.all([ - release(newerDraft.body.modified), - release(newerDraft.body.modified), - ]); - } finally { - historyStub.restore(); - } + const responses = await Promise.all([ + release(newerDraft.body.modified), + release(newerDraft.body.modified), + ]); expect(responses.map((response) => response.status).sort()).toEqual([200, 409]); const tagged = await dynamicRepo.getAllSnapshots(track.id, { taggedOnly: true }); @@ -527,6 +512,87 @@ describe('Release-track release planning and commit API', function () { expect(released.body.version).toBe('3.0'); }); + it('bases relative bumps on an explicitly tagged preceding release', async function () { + const track = await createTrack('Mixed Explicit Relative Versions'); + await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + version: '19.1', + }); + const draft = await post(`/api/release-tracks/${track.id}/meta`, { + description: 'Draft after the explicit v19.1 release', + }); + + const minor = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?increment=minor`, + ); + const major = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?increment=major`, + ); + + expect(minor.body).toMatchObject({ + source_snapshot_modified: draft.body.modified, + version: '19.2', + version_bounds: { + lower: { version: '19.1' }, + upper: null, + }, + }); + expect(major.body.version).toBe('20.0'); + + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + increment: 'minor', + }); + expect(released.body.version).toBe('19.2'); + }); + + it('bounds a retroactive release between its adjacent tagged snapshots', async function () { + const track = await createTrack('Chronological Version Bounds', 'virtual'); + const created = new Date(track.modified); + const firstTaggedModified = new Date(created.getTime() + 1000); + const historicalDraftModified = new Date(created.getTime() + 3000); + const laterTaggedModified = new Date(created.getTime() + 5000); + + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: firstTaggedModified, + version: '1.0', + }); + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: historicalDraftModified, + version: null, + composition_resolution: compositionResolution(historicalDraftModified), + }); + await dynamicRepo.saveSnapshot(track.id, { + ...snapshotBase(track), + modified: laterTaggedModified, + version: '3.0', + }); + + const releasePath = + `/api/release-tracks/${track.id}/snapshots/` + + `${encodeURIComponent(historicalDraftModified.toISOString())}/release`; + const minor = await get(`${releasePath}/preview?increment=minor`); + const major = await get(`${releasePath}/preview?increment=major`); + const explicit = await get(`${releasePath}/preview?version=2.7`); + + expect(minor.body).toMatchObject({ + version: '1.1', + version_bounds: { + lower: { version: '1.0', modified: firstTaggedModified.toISOString() }, + upper: { version: '3.0', modified: laterTaggedModified.toISOString() }, + }, + }); + expect(major.body.version).toBe('2.0'); + expect(explicit.body.version).toBe('2.7'); + + await get(`${releasePath}/preview?version=1.0`, 400); + await get(`${releasePath}/preview?version=3.0`, 400); + await get(`${releasePath}/preview?version=3.1`, 400); + + const released = await post(releasePath, { increment: 'minor' }); + expect(released.body.version).toBe('1.1'); + }); + it('compares the latest virtual draft with its preceding tagged release', async function () { const updatedOld = ( await post('/api/techniques', buildTechnique('Virtual Preview Updated Old'), 201) @@ -636,8 +702,12 @@ describe('Release-track release planning and commit API', function () { }); const preview = await get( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(historicalDraftModified.toISOString())}/release/preview?version=3.0`, + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(historicalDraftModified.toISOString())}/release/preview?version=1.5`, ); + expect(preview.body.version_bounds).toEqual({ + lower: { version: '1.0', modified: firstTaggedModified.toISOString() }, + upper: { version: '2.0', modified: laterTaggedModified.toISOString() }, + }); expect(preview.body.previous_release).toEqual({ version: '1.0', modified: firstTaggedModified.toISOString(), diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 5ae1bafe..e0a7bba2 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,40 @@ # Release Track TODOs +## Snapshot collection descriptions and bounded release versions + +- [x] Map each snapshot's user-authored description onto emitted + `x-mitre-collection.description` while preserving the track description + as the fallback for snapshots without notes. +- [x] Add backend regressions for bundle and graph-backed bundle exports using + snapshot descriptions. +- [x] Calculate relative and explicit release versions between the nearest + earlier and later tagged snapshots, with exclusive chronological bounds. +- [x] Add regression coverage for mixed explicit/relative tags, retroactive + releases, invalid boundary values, and exact-version uniqueness. +- [x] Wire exact `MAJOR.MINOR` release selection into the Angular release + preview dialog and connector flow with component/page tests. +- [x] Update OpenAPI, user/developer docs, and Bruno release requests. +- [x] Run focused backend and frontend tests, then the complete backend + `npm test` suite and the relevant frontend verification commands. +- [x] Propose conventional commit messages for both repositories. + +Verification (2026-08-04): + +- Focused backend release and bundle specs pass (23 and 18 cases), including + mixed explicit/relative tags, retroactive bounds, concurrent release locking, + and snapshot-description export. +- Backend lint, OpenAPI/config validation, middleware (29 cases), scheduler + (10 cases), and every isolated full-suite failure pass. Four complete + `npm test` attempts reached 1007-1008 passing API cases before the documented + shared-server flake roamed to a different unrelated spec on each run; the + isolated targets pass under both Node 22 and Node 24. +- The complete frontend suite passes (163 files, 376 tests), targeted ESLint and + Prettier checks pass, and the production build succeeds with existing budget + warnings. +- Proposed backend commit: `feat(release-tracks): bound snapshot publication + versions`. Proposed frontend commit: `feat(release-tracks): tag snapshots + with exact versions`. + ## Frontend graph cache lifecycle controls - [x] Replace the static cache-materialization hourglass with the existing diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index b99027d2..b1adf496 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -128,8 +128,9 @@ STIX version serialization. The pipeline: TOC is derived from the release track itself: - `id`: `x-mitre-collection--` — stable across exports of the same track - - `name`/`description`/`created_by_ref`/`object_marking_refs`: from the - snapshot metadata + - `name`/`created_by_ref`/`object_marking_refs`: from the snapshot metadata + - `description`: from `snapshot_description` when present, otherwise the + snapshot's long-lived track `description` - `x_mitre_version`: the snapshot's tagged version, or `0.1` for drafts - `modified`: the snapshot's `modified` timestamp - `x_mitre_contents`: every bundle object except marking definitions diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 884298ac..ca89c0f1 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -50,10 +50,17 @@ supported deployments. `version`, never both. Controller validation returns 400 at the HTTP boundary, and `version-utils.calculateNextVersion` repeats the invariant so internal release-planning callers cannot silently choose one selector. +- Release versions are ordered by snapshot time. Relative increments use the + nearest earlier tagged snapshot; explicit and calculated values must be + greater than that lower bound and less than the nearest later tag. Version + uniqueness remains track-wide. Commits acquire a per-track registry lock so + separate API processes cannot validate and write incompatible tags from the + same stale bounds; abandoned locks become reclaimable after 15 minutes. - Snapshot descriptions are bounded to 4000 characters and are the narrow mutable-metadata exception to snapshot content immutability. They are stored as `snapshot_description` on the selected document and never update the - registry or the track-level `description`. + registry or the track-level `description`. Bundle exports map the local value + to `x-mitre-collection.description`, falling back to the track description. ### ATT&CK canonical-domain migration diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 7b99500e..f7a3de56 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -564,6 +564,9 @@ to set the tagged snapshot's notes in the same operation: `400 Bad Request` rather than choosing one - If both are omitted, defaults to a minor release - If this is the first release, the version will be `1.0` +- Relative increments use the nearest chronologically earlier tagged snapshot. + The result, or an explicit version, must also be lower than the nearest later + tagged snapshot when retroactively releasing a historical draft. ``` POST /api/release-tracks/:id/snapshots/latest/release @@ -1026,6 +1029,10 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview "type": "standard", "source_snapshot_modified": "2024-01-15T16:20:00.000Z", "version": "1.2", + "version_bounds": { + "lower": { "version": "1.1", "modified": "2024-01-01T12:00:00.000Z" }, + "upper": null + }, "releasable": true, "before": { "members_count": 10, "staged_count": 2, "candidates_count": 1 }, "after": { "members_count": 12, "staged_count": 0, "candidates_count": 1 }, @@ -1034,6 +1041,10 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview } ``` +`version_bounds` reports the exclusive adjacent tagged releases used by both +relative and explicit selection. A historical draft can have both a `lower` +and an `upper` bound. + `format=workbench` returns the complete would-be persisted snapshot. `format=bundle` returns its publication-ready STIX bundle. Thus “dry run” is not a separate command: it is a release preview with the desired format. diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index 4a8f43f9..fd2bf12b 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -75,6 +75,7 @@ Standard STIX bundle format: "type": "x-mitre-collection", "id": "x-mitre-collection--123", "name": "ATT&CK Enterprise", + "description": "Q1 publication snapshot", "x_mitre_version": "1.1", "x_mitre_contents": [ { "object_ref": "attack-pattern--aaa", "object_modified": "2024-01-10T10:00:00.000Z" } @@ -114,7 +115,9 @@ Standard STIX bundle format: `(object_ref, object_modified)` pair in `missing_references`; it never emits a partial bundle. A repository/database failure is returned as a server error rather than being mistaken for missing content. -- Notes are never included (notes are Workbench-native objects, not STIX objects) +- Workbench note objects are never included. The snapshot's own + `snapshot_description` is publication metadata and becomes the TOC + `description`. - Suitable for external publication **Bundle query parameters** (apply only when `format=bundle`): @@ -148,7 +151,9 @@ By default, bundles begin with an `x-mitre-collection` object that acts as a table of contents. It is derived from the release-track metadata: - `id` — stable per track (reuses the track UUID) -- `name` / `description` — from the release track +- `name` — from the release track snapshot +- `description` — from the snapshot's `snapshot_description`; falls back to + the long-lived track `description` when no snapshot-local value is set - `x_mitre_version` — the snapshot's tagged version, or `0.1` for draft snapshots - `modified` — the snapshot's modified timestamp - `x_mitre_attack_spec_version` — the deployment's default ATT&CK spec version diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index ca25180c..06d4df9c 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -16,6 +16,7 @@ This approach allows continuous development while providing stable, versioned re ### Snapshots A **snapshot** is an immutable state of a release track at a specific point in time, identified by: + - `id` - The release track's STIX identifier (constant across all snapshots) - `modified` - ISO 8601 timestamp when the snapshot was created (unique per snapshot) @@ -30,6 +31,7 @@ A snapshot may be either a **draft release** (untagged) or a **tagged release** A **draft release** is a snapshot without a version number (`version === null`). It represents work-in-progress. A **tagged release** is a snapshot that has been marked as production-ready for publication, identified by: + - `version` - Version string in MAJOR.MINOR format (e.g., "1.0") **Note:** ATT&CK release tracks use a two-part versioning scheme (MAJOR.MINOR), not the three-part semver format (MAJOR.MINOR.PATCH). The patch component is not tracked in `version`. @@ -37,6 +39,7 @@ A **tagged release** is a snapshot that has been marked as production-ready for Not all snapshots are tagged releases. Only snapshots explicitly tagged via the **release** operation become tagged releases. **Example Timeline with Tagged Releases (standard track):** + ``` id: "release-track--123", modified: "2024-01-01T10:00:00.000Z" version: null ← FIRST ROLLING DRAFT @@ -77,6 +80,7 @@ a new snapshot. `release` is the command; `tagged` describes the resulting snapshot state. This is analogous to Git's tagging system: + - Git commits = release track snapshots (identified by `modified` key) - Git tags = tagged releases (identified by `version` key) @@ -109,6 +113,7 @@ When you release a snapshot: 4. The `modified` timestamp **does not change** **Why in-place?** + - Avoids duplicate data (no need to copy the entire release track) - Clear semantics: tagging is metadata, not a content change - Snapshots remain immutable except for the version tag @@ -117,6 +122,7 @@ When you release a snapshot: ### Tagging Endpoints #### Release Latest Snapshot + ``` POST /api/release-tracks/:id/snapshots/latest/release ``` @@ -124,6 +130,7 @@ POST /api/release-tracks/:id/snapshots/latest/release Releases the most recent snapshot (highest `modified`) as a tagged release. **Request Body:** + ```json { "increment": "major" @@ -141,6 +148,7 @@ Callers that need to pin the operation to one snapshot should use the **Examples:** 1. **Automatic version calculation:** + ```bash # Current latest tagged release: 1.2 # Tag as: 1.3 (minor increment) @@ -151,6 +159,7 @@ POST /api/release-tracks/release--123/snapshots/latest/release ``` 1. **Major version increment:** + ```bash # Current latest tagged release: 1.2 # Tag as: 2.0 (major increment) @@ -161,8 +170,9 @@ POST /api/release-tracks/release--123/snapshots/latest/release ``` 1. **Explicit version:** + ```bash -# Set specific version (must be greater than previous) +# Set a specific version within the selected snapshot's chronological bounds POST /api/release-tracks/release--123/snapshots/latest/release { "version": "2.0" @@ -170,6 +180,7 @@ POST /api/release-tracks/release--123/snapshots/latest/release ``` 1. **Default version selection:** + ```bash # Defaults to minor increment POST /api/release-tracks/release--123/snapshots/latest/release @@ -177,6 +188,7 @@ POST /api/release-tracks/release--123/snapshots/latest/release ``` #### Release Specific Snapshot + ``` POST /api/release-tracks/:id/snapshots/:modified/release ``` @@ -184,11 +196,15 @@ POST /api/release-tracks/:id/snapshots/:modified/release Tags a specific snapshot as a tagged release. Can tag retroactively, (i.e., a non-latest snapshot can be tagged), granted no [versioning rules](#versioning-rules) are violated. **Use Cases:** + - You want to tag snapshot 3, then later also tag snapshot 5 - You forgot to tag a snapshot and want to mark it retroactively - You want to create multiple tagged releases from different development branches -**Constraint:** The version must be greater than any previously tagged version (no semver regression). +**Constraint:** The version must be greater than the nearest earlier tagged +snapshot and less than the nearest later tagged snapshot. Both bounds are +exclusive. This allows a forgotten historical draft to be tagged without +breaking the version order of the timeline. ## Versioning Rules @@ -203,7 +219,9 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema ### Version Constraints -1. **Monotonically increasing** - New versions must always be greater than previous versions +1. **Chronologically increasing** - Tagged versions increase with snapshot + `modified` time. A retroactive tag is exclusively lower- and upper-bounded + by its adjacent tagged snapshots. 2. **Immutable once set** - Once a snapshot has `version` assigned, it cannot be changed 3. **Cannot re-tag** - A snapshot can only be tagged once (throws `AlreadyReleasedError` if attempted) 4. **Valid version format** - Must match `/^\d+\.\d+$/` (MAJOR.MINOR only, no patch component) @@ -212,8 +230,15 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema succeeds and the other receives `409 Conflict` with the conflicting `track_id` and `version`. +Relative `minor` and `major` increments are calculated from the nearest +earlier tagged snapshot, not from the numerically highest tag elsewhere in the +track. For example, a draft after explicit v19.1 previews as v19.2 for `minor` +and v20.0 for `major`. A historical draft between v1.0 and v3.0 previews as +v1.1 or v2.0 and may use any explicit version strictly inside that interval. + ### First Tagged Release For release tracks with no prior tagged releases: + - The first tag sets `version: "1.0"` (regardless of increment type) - Or you can specify an explicit version like `"0.1"` From 665876e6b968b2a55b3cb1978a6bab382053eabb Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Tue, 4 Aug 2026 20:47:45 -0400 Subject: [PATCH 53/55] feat(release-tracks): persist snapshot bundle hashes Freeze the collection object in deterministic graph manifests and store SHA-256 values for exact STIX 2.0 and 2.1 downloads. Reject snapshot-note edits while a bundle cache exists so the reported hashes remain valid. --- .../definitions/components/release-tracks.yml | 38 ++++++-- .../paths/release-tracks-paths.yml | 7 +- app/lib/release-tracks/export-schemas.js | 16 +++- .../release-track-graph-manifest-model.js | 2 +- .../release-track-snapshot-schema.js | 10 ++ .../release-track-dynamic.repository.js | 21 ++++- app/services/release-tracks/export-service.js | 7 ++ .../release-tracks/graph-manifest-service.js | 63 ++++++++++--- .../release-tracks/snapshot-service.js | 61 +++++++++++- .../release-tracks/versioning-service.js | 5 +- .../api/release-tracks/opt-in-graphs.spec.js | 73 +++++++++++++++ .../snapshot-descriptions.spec.js | 92 +++++++++++++++++++ .../release-tracks/snapshot-history.spec.js | 10 ++ .../developer/release-tracks/bundle-export.md | 23 ++++- docs/user/release-tracks/api-reference.md | 20 +++- 15 files changed, 406 insertions(+), 42 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index ba356ae0..e412dccb 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -30,15 +30,17 @@ components: Server-controlled identifier for an opt-in deterministic member graph on a tagged snapshot. Absent on drafts and graphless tagged snapshots. Clients should treat this value as opaque. + bundle_hashes: + $ref: '#/components/schemas/bundle-hashes' snapshot_description: type: string maxLength: 4000 description: | User-authored, snapshot-local notes. Editors may change this - workspace annotation without changing the snapshot identity, - release tag, contents, or deterministic graph. Bundle exports map - it to x-mitre-collection.description, falling back to the track - description when it is absent. + workspace annotation without changing the snapshot identity or + release tag while no pinned member graph exists. Cached snapshots + reject note edits until their graph is deleted and regenerated; + graphless exports fall back to the track description when absent. name: type: string pattern: '^[a-zA-Z0-9 &]+$' @@ -149,7 +151,28 @@ components: total_count: type: integer minimum: 0 - description: 'Total entries across all graph manifest roles' + description: 'Total emitted graph dependencies, excluding the collection metadata entry' + + bundle-hashes: + type: object + readOnly: true + description: | + SHA-256 digests of the exact UTF-8, four-space-indented JSON files + downloaded for a deterministic snapshot. The manifest ID binds the + digests to the cached graph that produced them. + required: + - manifest_id + - stix_2_0 + - stix_2_1 + properties: + manifest_id: + type: string + stix_2_0: + type: string + pattern: '^[a-f0-9]{64}$' + stix_2_1: + type: string + pattern: '^[a-f0-9]{64}$' snapshot-summary: type: object @@ -183,6 +206,8 @@ components: description: | Opaque identifier for the tagged snapshot's deterministic member graph. Omitted when the snapshot has not been materialized. + bundle_hashes: + $ref: '#/components/schemas/bundle-hashes' graph_statistics: $ref: '#/components/schemas/graph-statistics' description: | @@ -193,7 +218,8 @@ components: maxLength: 4000 description: | User-authored notes attached only to this snapshot and mapped to - x-mitre-collection.description during bundle export. + x-mitre-collection.description during bundle export. Notes cannot + be changed while the snapshot has a deterministic graph cache. name: type: string description: diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index bf9f5e29..accb0c6d 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -1292,8 +1292,9 @@ paths: Replace the user-authored notes on one draft or tagged snapshot. Whitespace is trimmed; an empty string clears the notes. This mutable workspace annotation does not change the snapshot modified timestamp, - semantic version, tier contents, graph manifest, or release-track - metadata. + semantic version, tier contents, or release-track metadata. A snapshot + with a graph manifest is immutable and returns 409 until its bundle + cache is deleted. tags: - 'Release Tracks' parameters: @@ -1332,6 +1333,8 @@ paths: description: 'Invalid description payload' '404': description: 'Snapshot not found' + '409': + description: 'Delete the snapshot bundle cache before editing its notes' /api/release-tracks/{id}/snapshots/{modified}/graph: post: diff --git a/app/lib/release-tracks/export-schemas.js b/app/lib/release-tracks/export-schemas.js index 34d3a488..a733d04d 100644 --- a/app/lib/release-tracks/export-schemas.js +++ b/app/lib/release-tracks/export-schemas.js @@ -57,6 +57,8 @@ const exportOptionsSchema = z stixVersion: z.enum(['2.0', '2.1']).default('2.1'), includeToc: z.boolean().default(true), attackSpecVersion: z.string().optional(), + collectionObject: z.looseObject({}).optional(), + bundleId: z.string().optional(), }) .optional() .default({}); @@ -113,8 +115,8 @@ function buildTocObject(snapshot, bundleObjects, options) { x_mitre_version: snapshot.version || '0.1', description: snapshot.snapshot_description ?? snapshot.description, created_by_ref: snapshot.created_by_ref || '', - created: snapshot.created || snapshot.modified, - modified: snapshot.modified, + created: options.created || snapshot.created || snapshot.modified, + modified: options.modified || snapshot.modified, x_mitre_contents: [], object_marking_refs: [], }; @@ -160,7 +162,7 @@ function buildTocObject(snapshot, bundleObjects, options) { // ----------------------------------------------------------------------------- const bundleTransformSchema = exportInputSchema.transform((input) => { - const { stixVersion, includeToc, attackSpecVersion } = input.options; + const { stixVersion, includeToc, attackSpecVersion, collectionObject, bundleId } = input.options; const objects = input.hydratedObjects .map((doc) => doc.stix) @@ -171,12 +173,16 @@ const bundleTransformSchema = exportInputSchema.transform((input) => { } if (includeToc) { - objects.unshift(buildTocObject(input.snapshot, objects, { stixVersion, attackSpecVersion })); + const tocObject = collectionObject + ? structuredClone(collectionObject) + : buildTocObject(input.snapshot, objects, { stixVersion, attackSpecVersion }); + conformToStixVersion(tocObject, stixVersion); + objects.unshift(tocObject); } return { type: 'bundle', - id: `bundle--${uuid.v4()}`, + id: bundleId || `bundle--${uuid.v4()}`, // STIX 2.0 bundles must declare spec_version; STIX 2.1 bundles must not ...(stixVersion === '2.0' ? { spec_version: '2.0' } : {}), objects, diff --git a/app/models/release-tracks/release-track-graph-manifest-model.js b/app/models/release-tracks/release-track-graph-manifest-model.js index 26b2fdec..1356ee58 100644 --- a/app/models/release-tracks/release-track-graph-manifest-model.js +++ b/app/models/release-tracks/release-track-graph-manifest-model.js @@ -43,7 +43,7 @@ const entrySchema = new mongoose.Schema( revision_key: { type: String, required: true }, kind: { type: String, - enum: ['root', 'relationship', 'secondary', 'supporting', 'link_target'], + enum: ['root', 'relationship', 'secondary', 'supporting', 'link_target', 'collection'], required: true, }, tier: { diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index ab8545c3..1f94acd2 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -340,6 +340,15 @@ const versionHistoryEntrySchema = new mongoose.Schema(versionHistoryEntryDefinit _id: false, }); +const bundleHashesSchema = new mongoose.Schema( + { + manifest_id: { type: String, required: true }, + stix_2_0: { type: String, required: true, match: /^[a-f0-9]{64}$/ }, + stix_2_1: { type: String, required: true, match: /^[a-f0-9]{64}$/ }, + }, + { _id: false }, +); + // ============================================================================= // Main snapshot schema // ============================================================================= @@ -365,6 +374,7 @@ const releaseTrackSnapshotDefinition = { validate: validateVersion, }, graph_manifest_id: { type: String }, + bundle_hashes: { type: bundleHashesSchema }, snapshot_description: { type: String, maxlength: [4000, 'Snapshot description cannot exceed 4000 characters'], diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 31ae0c22..4b4ad57a 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -289,6 +289,7 @@ class ReleaseTrackDynamicRepository { modified: 1, version: 1, graph_manifest_id: 1, + bundle_hashes: 1, snapshot_description: 1, name: 1, description: 1, @@ -424,7 +425,25 @@ class ReleaseTrackDynamicRepository { version: { $type: 'string' }, graph_manifest_id: manifestId, }, - { $unset: { graph_manifest_id: '' } }, + { $unset: { graph_manifest_id: '', bundle_hashes: '' } }, + { new: true, runValidators: true, lean: true }, + ).exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + async attachBundleHashes(trackId, modified, manifestId, bundleHashes) { + try { + const Model = this._getModel(trackId); + return await Model.findOneAndUpdate( + { + id: trackId, + modified, + version: { $type: 'string' }, + graph_manifest_id: manifestId, + }, + { $set: { bundle_hashes: bundleHashes } }, { new: true, runValidators: true, lean: true }, ).exec(); } catch (err) { diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index bec58eb5..c1e39b2e 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -93,6 +93,11 @@ function normalizeSourceBundleDefaults(documents, graph) { }); } +function bundleIdForManifest(manifest) { + const uuid = manifest?.manifest_id?.split('--')[1]; + return uuid ? `bundle--${uuid}` : undefined; +} + // ============================================================================= // Format helpers (delegating to Zod transform schemas) // ============================================================================= @@ -178,6 +183,8 @@ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options stixVersion: options.stixVersion, includeToc: options.includeToc, attackSpecVersion: config.app.attackSpecVersion, + collectionObject: graph.collectionObject, + bundleId: bundleIdForManifest(graph.manifest), }); } diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js index 86db7537..beee695b 100644 --- a/app/services/release-tracks/graph-manifest-service.js +++ b/app/services/release-tracks/graph-manifest-service.js @@ -2,6 +2,7 @@ const { isDeepStrictEqual } = require('node:util'); const { v4: uuidv4 } = require('uuid'); +const config = require('../../config/config'); const linkById = require('../../lib/linkById'); const bundleRelationships = require('../../lib/stix-bundle-relationships'); const attackObjectsRepository = require('../../repository/attack-objects-repository'); @@ -13,6 +14,7 @@ const { ReleaseTrackGraphManifestEntry, } = require('../../models/release-tracks/release-track-graph-manifest-model'); const { ReleaseContentIntegrityError } = require('../../exceptions'); +const { buildTocObject } = require('../../lib/release-tracks/export-schemas'); const primaryRevisionService = require('./primary-revision-service'); const MANIFEST_SCHEMA_VERSION = 2; @@ -74,6 +76,44 @@ function revisionKey(objectRef, objectModified) { return `${objectRef}::${new Date(objectModified).getTime()}`; } +async function getFirstCollectionCreated(trackId, fallback) { + const firstCollection = await ReleaseTrackGraphManifestEntry.findOne({ + track_id: trackId, + kind: 'collection', + }) + .sort({ _id: 1 }) + .select('frozen_stix.created') + .lean() + .exec(); + return firstCollection?.frozen_stix?.created || fallback; +} + +async function appendCollectionEntry(snapshot, entries, manifest) { + const graph = await replayEntries(entries, manifest, {}); + const created = await getFirstCollectionCreated(snapshot.id, manifest.created_at); + const collectionObject = buildTocObject( + snapshot, + graph.documents.map((document) => document.stix), + { + stixVersion: '2.1', + attackSpecVersion: config.app.attackSpecVersion, + created, + modified: manifest.created_at, + }, + ); + const entry = { + manifest_id: manifest.manifest_id, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + revision_key: `${collectionObject.id}::collection`, + kind: 'collection', + object_ref: collectionObject.id, + frozen_stix: collectionObject, + }; + await ReleaseTrackGraphManifestEntry.create(entry); + entries.push(entry); +} + function endpointFor(relationship, side) { const endpoint = relationship.workspace?.relationship_endpoints?.[side]; const objectRef = relationship.stix[`${side}_ref`]; @@ -583,13 +623,15 @@ async function prepare(snapshot, options = {}) { snapshot_modified: snapshot.modified, }; - await ReleaseTrackGraphManifest.create({ + const manifest = { ...common, state: 'pending', schema_version: schemaVersion, resolver_version: resolverVersion, baseline_reconstruction: options.baselineReconstruction === true, - }); + created_at: new Date(), + }; + await ReleaseTrackGraphManifest.create(manifest); try { if (entries.length > 0) { await ReleaseTrackGraphManifestEntry.insertMany( @@ -599,16 +641,7 @@ async function prepare(snapshot, options = {}) { // The pending manifest now protects every inserted pointer from deletion. // Rehydrate once inside that protection window so a revision deleted // during graph discovery cannot leave an attachable dangling manifest. - await replayEntries( - entries, - { - ...common, - state: 'pending', - schema_version: schemaVersion, - resolver_version: resolverVersion, - }, - {}, - ); + await appendCollectionEntry(snapshot, entries, manifest); } catch (err) { await discard(manifestId); throw err; @@ -782,6 +815,7 @@ async function prepareSourceReconstruction(snapshot, plan) { resolver_version: SOURCE_BUNDLE_RESOLVER_VERSION, baseline_reconstruction: true, source_attestation: plan.source_attestation, + created_at: new Date(), }; await ReleaseTrackGraphManifest.create(manifest); @@ -789,7 +823,7 @@ async function prepareSourceReconstruction(snapshot, plan) { await ReleaseTrackGraphManifestEntry.insertMany( entries.map((entry) => ({ ...common, ...entry })), ); - await replayEntries(entries, manifest, {}); + await appendCollectionEntry(snapshot, entries, manifest); } catch (err) { await discard(manifestId); throw err; @@ -1029,6 +1063,7 @@ async function replayEntries(entries, manifest, options) { .filter((entry) => entry.omitted_optional_defaults?.length) .map((entry) => [entry.object_ref, entry.omitted_optional_defaults]), ); + const collectionObject = entries.find((entry) => entry.kind === 'collection')?.frozen_stix; const emittedByRevision = new Map(); for (const document of [...selectedDocuments, ...supportingDocuments]) { @@ -1042,6 +1077,7 @@ async function replayEntries(entries, manifest, options) { documents: [...emittedByRevision.values()], linkTargetDocuments, sourceOmittedDefaults, + collectionObject, manifest, }; } @@ -1086,6 +1122,7 @@ async function replay(snapshot, options = {}) { const entries = await ReleaseTrackGraphManifestEntry.find({ manifest_id: manifest.manifest_id, }) + .sort({ _id: 1 }) .lean() .exec(); return replayEntries(entries, manifest, options); diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 4a730f76..82a81010 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -11,6 +11,7 @@ // clone or read snapshots. // ============================================================================= +const crypto = require('node:crypto'); const { v4: uuidv4 } = require('uuid'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); @@ -22,6 +23,7 @@ const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-in const primaryRevisionService = require('./primary-revision-service'); const reconciliationService = require('./reconciliation-service'); const graphManifestService = require('./graph-manifest-service'); +const exportService = require('./export-service'); const { TrackNotFoundError, NotFoundError, @@ -55,6 +57,25 @@ function normalizeTierSummary(summary) { }; } +function hashDownloadPayload(payload) { + return crypto + .createHash('sha256') + .update(JSON.stringify(payload, null, 4), 'utf8') + .digest('hex'); +} + +async function generateBundleHashes(snapshot) { + const [stix20Bundle, stix21Bundle] = await Promise.all([ + exportService.exportSnapshot(snapshot, 'bundle', { stixVersion: '2.0' }), + exportService.exportSnapshot(snapshot, 'bundle', { stixVersion: '2.1' }), + ]); + return { + manifest_id: snapshot.graph_manifest_id, + stix_2_0: hashDownloadPayload(stix20Bundle), + stix_2_1: hashDownloadPayload(stix21Bundle), + }; +} + /** * Recompute and persist denormalized registry counters from actual snapshot data. * @@ -223,6 +244,7 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { modified: snapshot.modified, version: snapshot.version, graph_manifest_id: snapshot.graph_manifest_id, + bundle_hashes: snapshot.bundle_hashes, snapshot_description: snapshot.snapshot_description, graph_statistics: snapshot.graph_manifest_id ? graphStatisticsByManifestId.get(snapshot.graph_manifest_id) @@ -308,6 +330,7 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov 'snapshot_description', ); delete clone.graph_manifest_id; + delete clone.bundle_hashes; clone.modified = new Date(); clone.version = null; // clones are always drafts delete clone.scheduled_materialization; @@ -395,6 +418,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { const clone = deepClone(sourceSnapshot); delete clone.graph_manifest_id; + delete clone.bundle_hashes; clone.id = newTrackId; clone.modified = now; clone.version = null; @@ -475,9 +499,9 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId * Set or clear a snapshot-local description without changing its identity, * release tag, members, or release-track registry metadata. * - * Snapshot descriptions are editable workspace annotations rather than - * versioned publication content, so tagged and draft snapshots are both valid - * targets. + * Snapshot descriptions are editable workspace annotations until a graph + * manifest freezes the bundle content. Cached snapshots must have their graph + * deleted before their description can change. * * @param {string} trackId * @param {string|Date} modified @@ -489,7 +513,15 @@ exports.updateSnapshotDescription = async function updateSnapshotDescription( modified, description, ) { - await exports.getSnapshotByModified(trackId, modified); + const snapshot = await exports.getSnapshotByModified(trackId, modified); + if (snapshot.graph_manifest_id) { + throw new ReleaseConflictError('Delete the bundle cache before editing snapshot notes.', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + graph_manifest_id: snapshot.graph_manifest_id, + }); + } + const update = description ? { $set: { snapshot_description: description } } : { $unset: { snapshot_description: '' } }; @@ -604,7 +636,26 @@ async function createGraph(trackId, modified, prepareManifest, validateExisting) `SnapshotService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, ); } - return { snapshot: attached, created: true }; + try { + const bundleHashes = await generateBundleHashes(attached); + const hashed = await dynamicRepo.attachBundleHashes( + trackId, + snapshot.modified, + manifestId, + bundleHashes, + ); + if (!hashed) { + throw new ReleaseConflictError('Snapshot graph changed while its hashes were generated', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + return { snapshot: hashed, created: true }; + } catch (err) { + await dynamicRepo.detachGraphManifest(trackId, snapshot.modified, manifestId); + await graphManifestService.discard(manifestId); + throw err; + } } exports.createGraph = function createLiveGraph(trackId, modified) { diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 923eb855..700d2c4c 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -315,7 +315,10 @@ async function commitPlan(plan) { const obsoleteManifestId = plan.sourceSnapshot.graph_manifest_id; const unsetOps = {}; - if (obsoleteManifestId) unsetOps.graph_manifest_id = ''; + if (obsoleteManifestId) { + unsetOps.graph_manifest_id = ''; + unsetOps.bundle_hashes = ''; + } if (plan.clearSnapshotDescription) unsetOps.snapshot_description = ''; const tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { version: plan.version, diff --git a/app/tests/api/release-tracks/opt-in-graphs.spec.js b/app/tests/api/release-tracks/opt-in-graphs.spec.js index 14a90f07..78cb868e 100644 --- a/app/tests/api/release-tracks/opt-in-graphs.spec.js +++ b/app/tests/api/release-tracks/opt-in-graphs.spec.js @@ -1,5 +1,6 @@ 'use strict'; +const crypto = require('node:crypto'); const request = require('supertest'); const { expect } = require('expect'); @@ -165,6 +166,11 @@ describe('Opt-in deterministic release-track graphs', function () { relationshipsRepository.retrieveAllForBundle = globalRelationshipScan; } expect(graphSnapshot.graph_manifest_id).toBeDefined(); + expect(graphSnapshot.bundle_hashes).toEqual({ + manifest_id: graphSnapshot.graph_manifest_id, + stix_2_0: expect.stringMatching(/^[a-f0-9]{64}$/), + stix_2_1: expect.stringMatching(/^[a-f0-9]{64}$/), + }); const manifest = await ReleaseTrackGraphManifest.findOne({ manifest_id: graphSnapshot.graph_manifest_id, @@ -205,6 +211,46 @@ describe('Opt-in deterministic release-track graphs', function () { } const markingEntry = entries.find((entry) => entry.object_ref === markingDefinitionId); expect(markingEntry.frozen_stix).toBeDefined(); + const collectionEntry = entries.find((entry) => entry.kind === 'collection'); + expect(collectionEntry).toMatchObject({ + manifest_id: graphSnapshot.graph_manifest_id, + track_id: track.id, + object_ref: `x-mitre-collection--${track.id.split('--')[1]}`, + frozen_stix: { + type: 'x-mitre-collection', + id: `x-mitre-collection--${track.id.split('--')[1]}`, + description: '', + created: manifest.created_at, + modified: manifest.created_at, + }, + }); + + for (const stixVersion of ['2.0', '2.1']) { + const bundle = ( + await authenticated( + request(app).get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle&stixVersion=${stixVersion}`, + ), + ).expect(200) + ).body; + const hash = crypto + .createHash('sha256') + .update(JSON.stringify(bundle, null, 4), 'utf8') + .digest('hex'); + expect(hash).toBe(graphSnapshot.bundle_hashes[`stix_2_${stixVersion.split('.')[1]}`]); + expect(bundle.id).toBe( + graphSnapshot.graph_manifest_id.replace('release-track-graph-manifest--', 'bundle--'), + ); + expect(bundle.objects[0]).toEqual( + expect.objectContaining({ + id: collectionEntry.frozen_stix.id, + created: collectionEntry.frozen_stix.created.toISOString(), + modified: collectionEntry.frozen_stix.modified.toISOString(), + }), + ); + } const correctedRelationship = await post( '/api/relationships', @@ -233,6 +279,33 @@ describe('Opt-in deterministic release-track graphs', function () { 200, ); expect(idempotent.graph_manifest_id).toBe(graphSnapshot.graph_manifest_id); + expect(idempotent.bundle_hashes).toEqual(graphSnapshot.bundle_hashes); + + await post(`/api/release-tracks/${track.id}/meta`, { name: 'Opt in Graph Track Next' }, 200); + const nextRelease = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '2.0' }, + 200, + ); + const nextGraph = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(nextRelease.modified)}/graph`, + {}, + ); + const nextManifest = await ReleaseTrackGraphManifest.findOne({ + manifest_id: nextGraph.graph_manifest_id, + }) + .lean() + .exec(); + const nextCollection = await ReleaseTrackGraphManifestEntry.findOne({ + manifest_id: nextGraph.graph_manifest_id, + kind: 'collection', + }) + .lean() + .exec(); + expect(nextCollection.frozen_stix.id).toBe(collectionEntry.frozen_stix.id); + expect(nextCollection.frozen_stix.created).toEqual(collectionEntry.frozen_stix.created); + expect(nextCollection.frozen_stix.modified).toEqual(nextManifest.created_at); + expect(nextCollection.frozen_stix.modified).not.toEqual(collectionEntry.frozen_stix.modified); await authenticated( request(app).delete( diff --git a/app/tests/api/release-tracks/snapshot-descriptions.spec.js b/app/tests/api/release-tracks/snapshot-descriptions.spec.js index a91803fc..3b18ee38 100644 --- a/app/tests/api/release-tracks/snapshot-descriptions.spec.js +++ b/app/tests/api/release-tracks/snapshot-descriptions.spec.js @@ -1,5 +1,6 @@ 'use strict'; +const crypto = require('node:crypto'); const request = require('supertest'); const { expect } = require('expect'); @@ -7,6 +8,9 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); +const { + ReleaseTrackGraphManifestEntry, +} = require('../../../models/release-tracks/release-track-graph-manifest-model'); describe('Release-track snapshot descriptions', function () { let app; @@ -118,6 +122,94 @@ describe('Release-track snapshot descriptions', function () { expect(registryTrack.description).toBe('Stable track description'); }); + it('rejects cached note edits until the cache is deleted and regenerated', async function () { + const track = await createTrack('Snapshot Description Cached', { + description: 'Stable fallback description', + }); + const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { + version: '1.0', + description: 'Initial cached notes.', + }); + const cached = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + {}, + 201, + ); + const originalHashes = cached.bundle_hashes; + const originalCollection = await ReleaseTrackGraphManifestEntry.findOne({ + manifest_id: cached.graph_manifest_id, + kind: 'collection', + }) + .lean() + .exec(); + + const conflict = await put( + descriptionPath(released), + { description: 'Corrected cached notes.' }, + 409, + ); + expect(conflict.message).toBe('Delete the bundle cache before editing snapshot notes.'); + + const unchangedSnapshot = await get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}`, + ); + expect(unchangedSnapshot.snapshot_description).toBe('Initial cached notes.'); + expect(unchangedSnapshot.bundle_hashes).toEqual(originalHashes); + + const unchangedCollection = await ReleaseTrackGraphManifestEntry.findOne({ + manifest_id: cached.graph_manifest_id, + kind: 'collection', + }) + .lean() + .exec(); + expect(unchangedCollection.frozen_stix).toEqual(originalCollection.frozen_stix); + + for (const stixVersion of ['2.0', '2.1']) { + const bundle = await get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle&stixVersion=${stixVersion}`, + ); + const hash = crypto + .createHash('sha256') + .update(JSON.stringify(bundle, null, 4), 'utf8') + .digest('hex'); + expect(hash).toBe(originalHashes[`stix_2_${stixVersion.split('.')[1]}`]); + expect(bundle.objects[0].description).toBe('Initial cached notes.'); + } + + await api( + 'delete', + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + undefined, + 204, + ); + const edited = await put(descriptionPath(released), { + description: 'Corrected cached notes.', + }); + expect(edited.snapshot_description).toBe('Corrected cached notes.'); + expect(edited).not.toHaveProperty('graph_manifest_id'); + expect(edited).not.toHaveProperty('bundle_hashes'); + + const recached = await post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, + {}, + 201, + ); + expect(recached.graph_manifest_id).not.toBe(cached.graph_manifest_id); + expect(recached.bundle_hashes.stix_2_0).not.toBe(originalHashes.stix_2_0); + expect(recached.bundle_hashes.stix_2_1).not.toBe(originalHashes.stix_2_1); + + const regeneratedCollection = await ReleaseTrackGraphManifestEntry.findOne({ + manifest_id: recached.graph_manifest_id, + kind: 'collection', + }) + .lean() + .exec(); + expect(regeneratedCollection.frozen_stix.description).toBe('Corrected cached notes.'); + expect(regeneratedCollection.frozen_stix.id).toBe(originalCollection.frozen_stix.id); + }); + it('clears existing draft notes when release explicitly supplies an empty description', async function () { const track = await createTrack('Snapshot Description Release Clear', { snapshot_description: 'Temporary draft context', diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js index 04f29de9..3553408b 100644 --- a/app/tests/api/release-tracks/snapshot-history.spec.js +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -78,6 +78,11 @@ describe('GET /api/release-tracks/:id/snapshots', function () { modified: standardTaggedModified, version: '1.0', graph_manifest_id: 'release-track-graph-manifest--snapshot-history', + bundle_hashes: { + manifest_id: 'release-track-graph-manifest--snapshot-history', + stix_2_0: 'a'.repeat(64), + stix_2_1: 'b'.repeat(64), + }, members: [memberEntry(0), memberEntry(1)], staged: [stagedEntry(2, standardTaggedModified)], candidates: [ @@ -213,6 +218,11 @@ describe('GET /api/release-tracks/:id/snapshots', function () { modified: standardTaggedModified.toISOString(), version: '1.0', graph_manifest_id: 'release-track-graph-manifest--snapshot-history', + bundle_hashes: { + manifest_id: 'release-track-graph-manifest--snapshot-history', + stix_2_0: 'a'.repeat(64), + stix_2_1: 'b'.repeat(64), + }, members_count: 2, staged_count: 1, candidates_count: 3, diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index b1adf496..0fc2986c 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -123,18 +123,28 @@ STIX version serialization. The pipeline: `stixVersion=2.0` — STIX 2.1 removed `spec_version` from the bundle object). 7. **TOC** — unless `includeToc=false`, an `x-mitre-collection` object is - prepended. Unlike the legacy exporter (which hardcoded per-domain - metadata) and the ephemeral endpoint (which uses ephemeral defaults), the - TOC is derived from the release track itself: + prepended. Graphless exports derive it from live snapshot metadata. Graph + creation instead freezes it as a `collection` manifest entry and every + member-only replay uses that stored value: - `id`: `x-mitre-collection--` — stable across exports of the same track - `name`/`created_by_ref`/`object_marking_refs`: from the snapshot metadata - `description`: from `snapshot_description` when present, otherwise the snapshot's long-lived track `description` - `x_mitre_version`: the snapshot's tagged version, or `0.1` for drafts - - `modified`: the snapshot's `modified` timestamp + - `created`: the first cached collection object's creation timestamp for + the release track + - `modified`: the current graph manifest's creation timestamp - `x_mitre_contents`: every bundle object except marking definitions (which are recorded in `object_marking_refs`), sorted by `object_ref` +8. **Deterministic file identity** — graph-backed member-only bundles use the + graph manifest UUID for the bundle envelope ID. After graph creation, the + server serializes each STIX version with `JSON.stringify(bundle, null, 4)`, + hashes those exact UTF-8 bytes with SHA-256, and stores both digests on the + snapshot as `bundle_hashes`. The graph, collection object, notes, and hashes + form one immutable cache boundary. Snapshot-note edits return `409 Conflict` + until the graph is deleted; callers then edit the notes and regenerate the + graph and hashes. ### Canonical domains and the legacy graph renderer @@ -201,7 +211,10 @@ members, writes a pending manifest and decoupled entry rows, rehydrates every pointer while those pending rows already protect deletion, then atomically attaches the manifest ID to the still-tagged snapshot. Replay can self-activate a complete linked pending manifest after an interrupted activation. `DELETE` -on the same graph resource detaches and removes it. +on the same graph resource detaches and removes it. Each manifest also owns one +frozen `x-mitre-collection` entry. The attached snapshot records SHA-256 values +for the exact STIX 2.0 and STIX 2.1 browser-download serialization, bound to the +same manifest ID. Historical baselines whose relationships predate endpoint-pin capture require a different, admin-only path: diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index f7a3de56..f8e8568e 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -422,7 +422,10 @@ Every summary contains `id`, `type`, `modified`, `version`, `name`, the track-level `description` (when set), `snapshot_description` (when the snapshot has user-authored notes), and `members_count`. A tagged snapshot whose deterministic member graph has been materialized also contains the opaque -`graph_manifest_id` and `graph_statistics`; graphless snapshots omit both. +`graph_manifest_id`, `graph_statistics`, and `bundle_hashes`; graphless +snapshots omit all three. `bundle_hashes` contains the manifest ID plus the +SHA-256 digests in `stix_2_0` and `stix_2_1` for the exact four-space-indented +UTF-8 JSON files downloaded by the browser. Graph statistics describe the cached graph at a glance: - `primary_count`: member objects deliberately selected for the snapshot. @@ -432,7 +435,8 @@ Graph statistics describe the cached graph at a glance: - `relationship_count`: relationships connecting cached graph objects. - `supporting_count`: supporting identities and marking definitions. - `link_target_count`: objects pinned for deterministic LinkById expansion. -- `total_count`: all entries across those manifest roles. +- `total_count`: all emitted dependency entries across those manifest roles; + the collection metadata entry is excluded. The UI groups supporting and LinkById targets together as **Dependencies**. Snapshot tier count keys continue to reflect the track type: @@ -505,7 +509,10 @@ PUT /api/release-tracks/:id/snapshots/:modified/description The value is trimmed and limited to 4000 characters. Send an empty string to clear it. The API returns the updated snapshot as `snapshot_description` and does not change the snapshot's `modified` timestamp, semantic version, tier -contents, graph cache, or the release track's long-lived description. +contents, or the release track's long-lived description. Cached snapshots are +immutable: this endpoint returns `409 Conflict` while a graph manifest exists. +Delete the bundle cache, edit the notes, and cache the bundle again to generate +a new frozen collection object and matching hashes. ### Update Metadata @@ -723,6 +730,13 @@ relationship revisions selected when the cache was created. This is not a general response cache and does not make candidate or staged exports deterministic. +Graph creation also stores one stateful `x-mitre-collection` manifest entry. +Its ID is stable for the release track, `created` comes from the track's first +cached collection object, and `modified` is the current manifest creation time. +The graph-backed bundle envelope uses the manifest UUID, so repeated STIX 2.0 +or STIX 2.1 downloads are byte-for-byte stable. The graph-creation response and +snapshot history expose SHA-256 hashes for both exact download files. + Administrators may use the separate `/graph/reconstruct` POST for a historical baseline backed by an independently verified source bundle. The request sends the bundle's SHA-256/collection/release/domain attestation plus exact graph From c2c017c146fae040caba559333b35536bfbd1189 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 5 Aug 2026 11:09:18 -0400 Subject: [PATCH 54/55] fix(release-tracks): repair deterministic bundle integrity Use a track-stable collection ID and the configured organization identity for STIX 2.1 table-of-contents objects. Omit collection objects from STIX 2.0 exports and migrate existing graph entries and bundle hashes. --- .../paths/release-tracks-paths.yml | 7 +- .../definitions/paths/stix-bundles-paths.yml | 3 +- app/lib/release-tracks/export-schemas.js | 29 +++- .../release-tracks/bundle-hash-service.js | 28 ++++ app/services/release-tracks/export-service.js | 7 + .../release-tracks/graph-manifest-service.js | 49 +++++-- .../release-tracks/snapshot-service.js | 24 +--- app/services/stix/stix-bundles-service-old.js | 2 +- app/services/stix/stix-bundles-service.js | 2 +- .../deterministic-graph-migration.spec.js | 109 ++++++++++++++ .../release-tracks/ephemeral-bundle.spec.js | 1 + .../api/release-tracks/opt-in-graphs.spec.js | 23 ++- .../release-tracks-bundle.spec.js | 2 + .../snapshot-descriptions.spec.js | 7 +- .../api/release-tracks/virtual-bundle.spec.js | 1 + docs/README.md | 1 + ...elease-track-bundle-integrity-migration.md | 27 ++++ docs/developer/TODO.md | 28 ++++ .../developer/release-tracks/bundle-export.md | 21 ++- docs/user/release-tracks/api-reference.md | 14 +- docs/user/release-tracks/output-formats.md | 13 +- ...0-repair-release-track-bundle-integrity.js | 136 ++++++++++++++++++ 22 files changed, 468 insertions(+), 66 deletions(-) create mode 100644 app/services/release-tracks/bundle-hash-service.js create mode 100644 docs/admin/release-track-bundle-integrity-migration.md create mode 100644 migrations/20260805150000-repair-release-track-bundle-integrity.js diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index accb0c6d..649338c9 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -48,7 +48,7 @@ paths: in: query description: | Whether to include a table-of-contents object (of type `x-mitre-collection`) - in the bundle (bundle format only). + in STIX 2.1 bundles (bundle format only). STIX 2.0 always omits it. schema: type: boolean default: true @@ -500,6 +500,7 @@ paths: enum: ['2.0', '2.1'] - name: includeToc in: query + description: 'Include the x-mitre-collection TOC in STIX 2.1 bundle previews; STIX 2.0 always omits it' schema: type: boolean responses: @@ -1107,7 +1108,7 @@ paths: default: '2.1' - name: includeToc in: query - description: 'Include the x-mitre-collection TOC in bundle responses' + description: 'Include the x-mitre-collection TOC in STIX 2.1 bundle responses; STIX 2.0 bundles never include it' schema: type: boolean default: true @@ -1208,6 +1209,7 @@ paths: description: | Whether to include a table-of-contents object (of type `x-mitre-collection`) derived from the release-track metadata (bundle format only). + This applies only to STIX 2.1; STIX 2.0 bundles never include it. schema: type: boolean default: true @@ -1565,6 +1567,7 @@ paths: enum: ['2.0', '2.1'] - name: includeToc in: query + description: 'Include the x-mitre-collection TOC in STIX 2.1 bundle previews; STIX 2.0 always omits it' schema: type: boolean responses: diff --git a/app/api/definitions/paths/stix-bundles-paths.yml b/app/api/definitions/paths/stix-bundles-paths.yml index e22b6e07..ff8fb1c2 100644 --- a/app/api/definitions/paths/stix-bundles-paths.yml +++ b/app/api/definitions/paths/stix-bundles-paths.yml @@ -88,7 +88,8 @@ paths: - name: includeCollectionObject in: query description: | - Whether to create an object of type `x-mitre-collection` for objects in the bundle. + Whether to create an object of type `x-mitre-collection` for + objects in a STIX 2.1 bundle. STIX 2.0 always omits it. schema: type: boolean default: false diff --git a/app/lib/release-tracks/export-schemas.js b/app/lib/release-tracks/export-schemas.js index a733d04d..2b87f89a 100644 --- a/app/lib/release-tracks/export-schemas.js +++ b/app/lib/release-tracks/export-schemas.js @@ -58,6 +58,8 @@ const exportOptionsSchema = z includeToc: z.boolean().default(true), attackSpecVersion: z.string().optional(), collectionObject: z.looseObject({}).optional(), + collectionId: z.string().optional(), + createdByRef: z.string().optional(), bundleId: z.string().optional(), }) .optional() @@ -109,12 +111,12 @@ function buildTocObject(snapshot, bundleObjects, options) { const tocObject = { type: 'x-mitre-collection', - id: `x-mitre-collection--${trackUuid}`, + id: options.collectionId || `x-mitre-collection--${trackUuid}`, x_mitre_attack_spec_version: options.attackSpecVersion, name: snapshot.name, x_mitre_version: snapshot.version || '0.1', description: snapshot.snapshot_description ?? snapshot.description, - created_by_ref: snapshot.created_by_ref || '', + created_by_ref: options.createdByRef || snapshot.created_by_ref || '', created: options.created || snapshot.created || snapshot.modified, modified: options.modified || snapshot.modified, x_mitre_contents: [], @@ -154,7 +156,7 @@ function buildTocObject(snapshot, bundleObjects, options) { // only for STIX 2.0 — the STIX 2.1 specification removed spec_version // from the bundle object (objects declare their own spec_version). // - includeToc (default true): prepend an x-mitre-collection object derived -// from the snapshot metadata +// from the snapshot metadata for STIX 2.1; STIX 2.0 always omits it // - attackSpecVersion: x_mitre_attack_spec_version for the TOC object // // Notes are Workbench-native objects, not STIX objects, so they are never @@ -162,7 +164,15 @@ function buildTocObject(snapshot, bundleObjects, options) { // ----------------------------------------------------------------------------- const bundleTransformSchema = exportInputSchema.transform((input) => { - const { stixVersion, includeToc, attackSpecVersion, collectionObject, bundleId } = input.options; + const { + stixVersion, + includeToc, + attackSpecVersion, + collectionObject, + collectionId, + createdByRef, + bundleId, + } = input.options; const objects = input.hydratedObjects .map((doc) => doc.stix) @@ -172,10 +182,17 @@ const bundleTransformSchema = exportInputSchema.transform((input) => { conformToStixVersion(stixObject, stixVersion); } - if (includeToc) { + // x-mitre-collection is a STIX 2.1 ATT&CK extension object. It must never be + // emitted in a STIX 2.0 bundle, even when includeToc retains its default. + if (includeToc && stixVersion === '2.1') { const tocObject = collectionObject ? structuredClone(collectionObject) - : buildTocObject(input.snapshot, objects, { stixVersion, attackSpecVersion }); + : buildTocObject(input.snapshot, objects, { + stixVersion, + attackSpecVersion, + collectionId, + createdByRef, + }); conformToStixVersion(tocObject, stixVersion); objects.unshift(tocObject); } diff --git a/app/services/release-tracks/bundle-hash-service.js b/app/services/release-tracks/bundle-hash-service.js new file mode 100644 index 00000000..aad2e0e7 --- /dev/null +++ b/app/services/release-tracks/bundle-hash-service.js @@ -0,0 +1,28 @@ +'use strict'; + +const crypto = require('node:crypto'); +const exportService = require('./export-service'); + +function hashDownloadPayload(payload) { + return crypto + .createHash('sha256') + .update(JSON.stringify(payload, null, 4), 'utf8') + .digest('hex'); +} + +async function generateBundleHashes(snapshot) { + const [stix20Bundle, stix21Bundle] = await Promise.all([ + exportService.exportSnapshot(snapshot, 'bundle', { stixVersion: '2.0' }), + exportService.exportSnapshot(snapshot, 'bundle', { stixVersion: '2.1' }), + ]); + return { + manifest_id: snapshot.graph_manifest_id, + stix_2_0: hashDownloadPayload(stix20Bundle), + stix_2_1: hashDownloadPayload(stix21Bundle), + }; +} + +module.exports = { + generateBundleHashes, + hashDownloadPayload, +}; diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index c1e39b2e..0695c463 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -22,6 +22,7 @@ const logger = require('../../lib/logger'); const linkById = require('../../lib/linkById'); const primaryRevisionService = require('./primary-revision-service'); const graphManifestService = require('./graph-manifest-service'); +const systemConfigurationService = require('../system/system-configuration-service'); const { bundleTransformSchema, workbenchTransformSchema, @@ -178,12 +179,18 @@ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options : await graphManifestService.replay(snapshot, options); const allObjects = normalizeSourceBundleDefaults(graph.documents, graph); await convertLinkByIdTags(allObjects, graph.linkTargetDocuments); + let createdByRef; + if (options.stixVersion !== '2.0' && options.includeToc !== false && !graph.collectionObject) { + const organizationIdentity = await systemConfigurationService.retrieveOrganizationIdentity(); + createdByRef = organizationIdentity.stix.id; + } return exports.formatAsBundle(snapshot, allObjects, { stixVersion: options.stixVersion, includeToc: options.includeToc, attackSpecVersion: config.app.attackSpecVersion, collectionObject: graph.collectionObject, + createdByRef, bundleId: bundleIdForManifest(graph.manifest), }); } diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js index beee695b..d8c2d7c3 100644 --- a/app/services/release-tracks/graph-manifest-service.js +++ b/app/services/release-tracks/graph-manifest-service.js @@ -15,6 +15,7 @@ const { } = require('../../models/release-tracks/release-track-graph-manifest-model'); const { ReleaseContentIntegrityError } = require('../../exceptions'); const { buildTocObject } = require('../../lib/release-tracks/export-schemas'); +const systemConfigurationService = require('../system/system-configuration-service'); const primaryRevisionService = require('./primary-revision-service'); const MANIFEST_SCHEMA_VERSION = 2; @@ -81,37 +82,57 @@ async function getFirstCollectionCreated(trackId, fallback) { track_id: trackId, kind: 'collection', }) - .sort({ _id: 1 }) + .sort({ 'frozen_stix.created': 1, _id: 1 }) .select('frozen_stix.created') .lean() .exec(); return firstCollection?.frozen_stix?.created || fallback; } -async function appendCollectionEntry(snapshot, entries, manifest) { +function collectionIdForTrack(trackId) { + return `x-mitre-collection--${trackId.split('--')[1]}`; +} + +async function organizationIdentityRef() { + const organizationIdentity = await systemConfigurationService.retrieveOrganizationIdentity(); + return organizationIdentity.stix.id; +} + +async function upsertCollectionEntry(snapshot, entries, manifest) { const graph = await replayEntries(entries, manifest, {}); - const created = await getFirstCollectionCreated(snapshot.id, manifest.created_at); + const created = await getFirstCollectionCreated(manifest.track_id, manifest.created_at); + const createdByRef = await organizationIdentityRef(); + const collectionId = collectionIdForTrack(manifest.track_id); const collectionObject = buildTocObject( snapshot, graph.documents.map((document) => document.stix), { stixVersion: '2.1', attackSpecVersion: config.app.attackSpecVersion, + collectionId, + createdByRef, created, modified: manifest.created_at, }, ); const entry = { manifest_id: manifest.manifest_id, - track_id: snapshot.id, + track_id: manifest.track_id, snapshot_modified: snapshot.modified, revision_key: `${collectionObject.id}::collection`, kind: 'collection', object_ref: collectionObject.id, frozen_stix: collectionObject, }; - await ReleaseTrackGraphManifestEntry.create(entry); - entries.push(entry); + const storedEntry = await ReleaseTrackGraphManifestEntry.findOneAndUpdate( + { manifest_id: manifest.manifest_id, kind: 'collection' }, + { $set: entry }, + { new: true, upsert: true, runValidators: true, lean: true }, + ).exec(); + const existingIndex = entries.findIndex((candidate) => candidate.kind === 'collection'); + if (existingIndex === -1) entries.push(storedEntry); + else entries[existingIndex] = storedEntry; + return storedEntry; } function endpointFor(relationship, side) { @@ -641,7 +662,7 @@ async function prepare(snapshot, options = {}) { // The pending manifest now protects every inserted pointer from deletion. // Rehydrate once inside that protection window so a revision deleted // during graph discovery cannot leave an attachable dangling manifest. - await appendCollectionEntry(snapshot, entries, manifest); + await upsertCollectionEntry(snapshot, entries, manifest); } catch (err) { await discard(manifestId); throw err; @@ -823,7 +844,7 @@ async function prepareSourceReconstruction(snapshot, plan) { await ReleaseTrackGraphManifestEntry.insertMany( entries.map((entry) => ({ ...common, ...entry })), ); - await appendCollectionEntry(snapshot, entries, manifest); + await upsertCollectionEntry(snapshot, entries, manifest); } catch (err) { await discard(manifestId); throw err; @@ -1144,6 +1165,16 @@ async function replayPlannedSnapshot(snapshot, options = {}) { ); } +async function refreshCollectionEntry(snapshot, manifest) { + const entries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: manifest.manifest_id, + }) + .sort({ _id: 1 }) + .lean() + .exec(); + return upsertCollectionEntry(snapshot, entries, manifest); +} + async function findPinsForRevision(objectRef, objectModified) { const entries = await ReleaseTrackGraphManifestEntry.find({ object_ref: objectRef, @@ -1219,6 +1250,8 @@ module.exports = { discardTrack, replay, replayPlannedSnapshot, + refreshCollectionEntry, + collectionIdForTrack, getStatisticsByManifestIds, findPinsForRevision, findPinsForObject, diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 82a81010..4c63ddfa 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -11,7 +11,6 @@ // clone or read snapshots. // ============================================================================= -const crypto = require('node:crypto'); const { v4: uuidv4 } = require('uuid'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); @@ -23,7 +22,7 @@ const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-in const primaryRevisionService = require('./primary-revision-service'); const reconciliationService = require('./reconciliation-service'); const graphManifestService = require('./graph-manifest-service'); -const exportService = require('./export-service'); +const bundleHashService = require('./bundle-hash-service'); const { TrackNotFoundError, NotFoundError, @@ -57,25 +56,6 @@ function normalizeTierSummary(summary) { }; } -function hashDownloadPayload(payload) { - return crypto - .createHash('sha256') - .update(JSON.stringify(payload, null, 4), 'utf8') - .digest('hex'); -} - -async function generateBundleHashes(snapshot) { - const [stix20Bundle, stix21Bundle] = await Promise.all([ - exportService.exportSnapshot(snapshot, 'bundle', { stixVersion: '2.0' }), - exportService.exportSnapshot(snapshot, 'bundle', { stixVersion: '2.1' }), - ]); - return { - manifest_id: snapshot.graph_manifest_id, - stix_2_0: hashDownloadPayload(stix20Bundle), - stix_2_1: hashDownloadPayload(stix21Bundle), - }; -} - /** * Recompute and persist denormalized registry counters from actual snapshot data. * @@ -637,7 +617,7 @@ async function createGraph(trackId, modified, prepareManifest, validateExisting) ); } try { - const bundleHashes = await generateBundleHashes(attached); + const bundleHashes = await bundleHashService.generateBundleHashes(attached); const hashed = await dynamicRepo.attachBundleHashes( trackId, snapshot.modified, diff --git a/app/services/stix/stix-bundles-service-old.js b/app/services/stix/stix-bundles-service-old.js index 9acae2b3..77c4913e 100644 --- a/app/services/stix/stix-bundles-service-old.js +++ b/app/services/stix/stix-bundles-service-old.js @@ -597,7 +597,7 @@ class StixBundlesService extends BaseService { StixBundlesService.conformToStixVersion(stixObject, options.stixVersion); } - if (options.includeCollectionObject) { + if (options.includeCollectionObject && options.stixVersion === '2.1') { StixBundlesService.addCollectionObject(bundle, options); } return bundle; diff --git a/app/services/stix/stix-bundles-service.js b/app/services/stix/stix-bundles-service.js index 199b7f60..26e3d917 100644 --- a/app/services/stix/stix-bundles-service.js +++ b/app/services/stix/stix-bundles-service.js @@ -468,7 +468,7 @@ class StixBundlesService extends BaseService { StixBundlesService.conformToStixVersion(stixObject, options.stixVersion); } - if (options.includeCollectionObject) { + if (options.includeCollectionObject && options.stixVersion === '2.1') { StixBundlesService.addCollectionObject(bundle, options); } return bundle; diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 2bd2ff5b..0df0a791 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -1,5 +1,6 @@ 'use strict'; +const crypto = require('node:crypto'); const mongoose = require('mongoose'); const request = require('supertest'); const { expect } = require('expect'); @@ -9,6 +10,7 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const migration = require('../../../../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); +const bundleIntegrityMigration = require('../../../../migrations/20260805150000-repair-release-track-bundle-integrity'); const Relationship = require('../../../models/relationship-model'); const { ReleaseTrackGraphManifest, @@ -242,6 +244,113 @@ describe('Deterministic snapshot graph migration', function () { expect(objectIds).toContain(relationship.stix.id); }); + it('repairs graph collection identities and recomputes tagged bundle hashes', async function () { + const organizationIdentity = ( + await request(app) + .get('/api/config/organization-identity') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200) + ).body; + const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId }) + .sort({ created_at: 1 }) + .lean() + .exec(); + const collectionEntries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) }, + kind: 'collection', + }) + .sort({ snapshot_modified: 1 }) + .lean() + .exec(); + + for (const [index, entry] of collectionEntries.entries()) { + const badId = `x-mitre-collection--00000000-0000-4000-8000-${String(index).padStart( + 12, + '0', + )}`; + await ReleaseTrackGraphManifestEntry.updateOne( + { _id: entry._id }, + { + $set: { + object_ref: badId, + revision_key: `${badId}::collection`, + 'frozen_stix.id': badId, + 'frozen_stix.created_by_ref': 'identity--00000000-0000-4000-8000-000000000000', + }, + }, + ).exec(); + } + await mongoose.connection.db.collection(trackId).updateMany( + { graph_manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) } }, + { + $set: { + bundle_hashes: { + manifest_id: manifests[0].manifest_id, + stix_2_0: '0'.repeat(64), + stix_2_1: '0'.repeat(64), + }, + }, + }, + ); + + const preview = await bundleIntegrityMigration._private.run(mongoose.connection.db, null, { + dryRun: true, + }); + expect(preview.collection_entries_repaired).toBe(collectionEntries.length); + expect(preview.bundle_hashes_recomputed).toBeGreaterThan(0); + + const report = await bundleIntegrityMigration._private.run(mongoose.connection.db); + expect(report.collection_entries_repaired).toBe(collectionEntries.length); + expect(report.bundle_hashes_recomputed).toBeGreaterThan(0); + + const repairedEntries = await ReleaseTrackGraphManifestEntry.find({ + manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) }, + kind: 'collection', + }) + .lean() + .exec(); + const expectedCollectionId = `x-mitre-collection--${trackId.split('--')[1]}`; + expect(new Set(repairedEntries.map((entry) => entry.frozen_stix.id))).toEqual( + new Set([expectedCollectionId]), + ); + expect( + repairedEntries.every( + (entry) => entry.frozen_stix.created_by_ref === organizationIdentity.stix.id, + ), + ).toBe(true); + + const taggedSnapshots = await mongoose.connection.db + .collection(trackId) + .find({ graph_manifest_id: { $exists: true }, version: { $type: 'string' } }) + .toArray(); + for (const snapshot of taggedSnapshots) { + for (const stixVersion of ['2.0', '2.1']) { + const bundle = ( + await request(app) + .get( + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( + snapshot.modified.toISOString(), + )}?format=bundle&stixVersion=${stixVersion}`, + ) + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200) + ).body; + if (stixVersion === '2.0') { + expect(bundle.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); + } + const hash = crypto + .createHash('sha256') + .update(JSON.stringify(bundle, null, 4), 'utf8') + .digest('hex'); + expect(hash).toBe(snapshot.bundle_hashes?.[`stix_2_${stixVersion.split('.')[1]}`]); + } + } + + const rerun = await bundleIntegrityMigration._private.run(mongoose.connection.db); + expect(rerun.collection_entries_repaired).toBe(0); + expect(rerun.bundle_hashes_recomputed).toBe(0); + }); + it('replays and activates a complete linked pending manifest after interruption', async function () { const snapshot = await mongoose.connection.db .collection(trackId) diff --git a/app/tests/api/release-tracks/ephemeral-bundle.spec.js b/app/tests/api/release-tracks/ephemeral-bundle.spec.js index 8c5cb002..215f6c73 100644 --- a/app/tests/api/release-tracks/ephemeral-bundle.spec.js +++ b/app/tests/api/release-tracks/ephemeral-bundle.spec.js @@ -342,6 +342,7 @@ describe('Ephemeral Bundle API', function () { const bundle = await getEphemeral('?stixVersion=2.0'); expect(bundle.spec_version).toBe('2.0'); + expect(bundle.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); const technique = bundle.objects.find((o) => o.id === enterpriseTechnique.stix.id); expect(technique.spec_version).toBeUndefined(); }); diff --git a/app/tests/api/release-tracks/opt-in-graphs.spec.js b/app/tests/api/release-tracks/opt-in-graphs.spec.js index 78cb868e..49e36c9a 100644 --- a/app/tests/api/release-tracks/opt-in-graphs.spec.js +++ b/app/tests/api/release-tracks/opt-in-graphs.spec.js @@ -212,6 +212,9 @@ describe('Opt-in deterministic release-track graphs', function () { const markingEntry = entries.find((entry) => entry.object_ref === markingDefinitionId); expect(markingEntry.frozen_stix).toBeDefined(); const collectionEntry = entries.find((entry) => entry.kind === 'collection'); + const organizationIdentity = ( + await authenticated(request(app).get('/api/config/organization-identity')).expect(200) + ).body; expect(collectionEntry).toMatchObject({ manifest_id: graphSnapshot.graph_manifest_id, track_id: track.id, @@ -219,6 +222,7 @@ describe('Opt-in deterministic release-track graphs', function () { frozen_stix: { type: 'x-mitre-collection', id: `x-mitre-collection--${track.id.split('--')[1]}`, + created_by_ref: organizationIdentity.stix.id, description: '', created: manifest.created_at, modified: manifest.created_at, @@ -243,13 +247,18 @@ describe('Opt-in deterministic release-track graphs', function () { expect(bundle.id).toBe( graphSnapshot.graph_manifest_id.replace('release-track-graph-manifest--', 'bundle--'), ); - expect(bundle.objects[0]).toEqual( - expect.objectContaining({ - id: collectionEntry.frozen_stix.id, - created: collectionEntry.frozen_stix.created.toISOString(), - modified: collectionEntry.frozen_stix.modified.toISOString(), - }), - ); + if (stixVersion === '2.0') { + expect(bundle.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); + } else { + expect(bundle.objects[0]).toEqual( + expect.objectContaining({ + id: collectionEntry.frozen_stix.id, + created_by_ref: organizationIdentity.stix.id, + created: collectionEntry.frozen_stix.created.toISOString(), + modified: collectionEntry.frozen_stix.modified.toISOString(), + }), + ); + } } const correctedRelationship = await post( diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index f095cbc0..a73f9547 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -337,6 +337,7 @@ describe('Release Tracks Bundle Export API', function () { expect(toc.x_mitre_version).toBe('0.1'); expect(toc.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion); expect(toc.spec_version).toBe('2.1'); + expect(toc.created_by_ref).toBe(organizationIdentityId); // Marking definitions are tracked in object_marking_refs, everything else // in x_mitre_contents @@ -581,6 +582,7 @@ describe('Release Tracks Bundle Export API', function () { ); expect(bundle.spec_version).toBe('2.0'); + expect(bundle.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); const member = bundle.objects.find((o) => o.id === memberObject.stix.id); expect(member.spec_version).toBeUndefined(); }); diff --git a/app/tests/api/release-tracks/snapshot-descriptions.spec.js b/app/tests/api/release-tracks/snapshot-descriptions.spec.js index 3b18ee38..44ff04f5 100644 --- a/app/tests/api/release-tracks/snapshot-descriptions.spec.js +++ b/app/tests/api/release-tracks/snapshot-descriptions.spec.js @@ -175,7 +175,12 @@ describe('Release-track snapshot descriptions', function () { .update(JSON.stringify(bundle, null, 4), 'utf8') .digest('hex'); expect(hash).toBe(originalHashes[`stix_2_${stixVersion.split('.')[1]}`]); - expect(bundle.objects[0].description).toBe('Initial cached notes.'); + const collection = bundle.objects.find((object) => object.type === 'x-mitre-collection'); + if (stixVersion === '2.0') { + expect(collection).toBeUndefined(); + } else { + expect(collection.description).toBe('Initial cached notes.'); + } } await api( diff --git a/app/tests/api/release-tracks/virtual-bundle.spec.js b/app/tests/api/release-tracks/virtual-bundle.spec.js index 76839c7a..9582f99a 100644 --- a/app/tests/api/release-tracks/virtual-bundle.spec.js +++ b/app/tests/api/release-tracks/virtual-bundle.spec.js @@ -125,6 +125,7 @@ describe('Virtual Release Track Bundle Export API', function () { expect(bundle.type).toBe('bundle'); expect(bundle.spec_version).toBe('2.0'); + expect(bundle.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); expect(bundle.objects.every((object) => object.spec_version === undefined)).toBe(true); const exportedMalware = bundle.objects.find((object) => object.id === malware.stix.id); diff --git a/docs/README.md b/docs/README.md index 921c41fa..e1f4d9b9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -62,6 +62,7 @@ Configuration, deployment, and identity provider setup. - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures - [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator track-deletion attempts - [Release-Track Deterministic Graph Migration](admin/release-track-graph-migration.md): Preview and operate the relationship-pin and snapshot-manifest backfill +- [Release-Track Bundle Integrity Migration](admin/release-track-bundle-integrity-migration.md): Repair frozen collection identities and deterministic bundle hashes - [ATT&CK Canonical-Domain Migration](admin/canonical-domain-migration.md): Understand the release-agnostic startup repair, inactive-revision handling, strict validation, and verification procedure ### Authentication diff --git a/docs/admin/release-track-bundle-integrity-migration.md b/docs/admin/release-track-bundle-integrity-migration.md new file mode 100644 index 00000000..dadf0fca --- /dev/null +++ b/docs/admin/release-track-bundle-integrity-migration.md @@ -0,0 +1,27 @@ +# Release-Track Bundle Integrity Migration + +The `20260805150000-repair-release-track-bundle-integrity` migration repairs +bundle metadata persisted by earlier deterministic release-track graph +implementations. + +For every active or pending graph manifest that is still linked to a snapshot, +the migration creates or refreshes its frozen `x-mitre-collection` entry. The +entry uses one ID derived from the release-track UUID across the track's full +history, and its `created_by_ref` is the STIX ID returned by the configured +organization-identity service. + +For tagged snapshots, the migration then recomputes `bundle_hashes.stix_2_0` +and `bundle_hashes.stix_2_1` from the exact four-space-indented download bytes. +STIX 2.0 serialization never includes the `x-mitre-collection` object; STIX +2.1 includes the repaired frozen object. Historical draft graphs are live +exports rather than deterministic caches, so any stale hashes on them are +removed. + +The migration runs during normal startup when +`WB_REST_DATABASE_MIGRATION_ENABLE=true`. It is rerunnable: already-correct +collection entries and hashes are retained. Orphaned manifests that are no +longer linked from their recorded snapshot are reported and skipped. + +The down migration is intentionally a no-op because restoring inconsistent +identifiers, creator references, or hashes would reintroduce invalid integrity +metadata. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index e0a7bba2..2ef45982 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,33 @@ # Release Track TODOs +## Deterministic graph collection identity repair + +- [x] Reproduce the incorrect graph collection creator, STIX 2.0 TOC + inclusion, and persisted cross-manifest collection-ID drift. +- [x] Resolve graph collection `created_by_ref` from the configured + organization identity and enforce one collection ID per release track. +- [x] Exclude `x-mitre-collection` from STIX 2.0 snapshot bundles and hashes. +- [x] Add a rerunnable forward migration that repairs existing graph + collection entries and recomputes tagged-snapshot bundle hashes. +- [x] Update release-track user, developer, and administrator documentation. +- [x] Run focused regression specs. +- [x] Complete an all-green `npm test` run without the documented roaming + in-memory MongoDB/server flake. +- [x] Propose a conventional commit message without committing unless asked. + +Verification (2026-08-05): + +- Focused graph, migration, snapshot, ephemeral, virtual, and legacy bundle + specs pass, including exact SHA-256 comparisons against downloaded bundles + and a rerun proving the repair migration is idempotent. +- ESLint and `git diff --check` pass. +- Full-suite attempts reached 1009 passing/3 failures, 1000/5, and repeatedly + 1011/1. Each failure roamed to an unrelated spec as a transient 400/404, + `ECONNRESET`, or socket hangup; every affected spec passes in isolation, + including under the repository-pinned Node 22.14.0 runtime. +- The developer subsequently confirmed a complete all-green test run. +- Proposed commit: `fix(release-tracks): repair deterministic bundle integrity`. + ## Snapshot collection descriptions and bounded release versions - [x] Map each snapshot's user-authored description onto emitted diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index 0fc2986c..f0ad1482 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -71,7 +71,7 @@ surface was simplified | `stixVersion` | **Preserved** (default changed to `2.1`) | | `includeRevoked` / `includeDeprecated` | **Preserved** (default `false`) | | `includeMissingAttackId` | **Renamed** to `includeObjectsWithMissingAttackId` (default `false`) | -| `includeCollectionObject` | **Renamed** to `includeToc` (default `true`). "TOC" (table of contents) describes what the `x-mitre-collection` object actually is, and avoids overloading the term "collection". | +| `includeCollectionObject` | **Renamed** to `includeToc` (default `true`). "TOC" (table of contents) describes what the `x-mitre-collection` object actually is, and avoids overloading the term "collection". It applies only to STIX 2.1; STIX 2.0 always omits the object. | | `collectionObjectVersion` | **Removed** — fixed at `0.1`, signifying an ephemerally generated collection not connected to a release track | | `collectionObjectModified` | **Removed** — fixed at the current timestamp | | `collectionAttackSpecVersion` | **Removed** — fixed at the global default (`config.app.attackSpecVersion`) | @@ -122,13 +122,15 @@ STIX version serialization. The pipeline: bundle envelope is emitted (with `spec_version: "2.0"` only when `stixVersion=2.0` — STIX 2.1 removed `spec_version` from the bundle object). -7. **TOC** — unless `includeToc=false`, an `x-mitre-collection` object is - prepended. Graphless exports derive it from live snapshot metadata. Graph - creation instead freezes it as a `collection` manifest entry and every - member-only replay uses that stored value: +7. **TOC** — for STIX 2.1, unless `includeToc=false`, an + `x-mitre-collection` object is prepended. STIX 2.0 always omits this ATT&CK + extension object. Graphless 2.1 exports derive it from live snapshot + metadata. Graph creation freezes it as a `collection` manifest entry and + every member-only 2.1 replay uses that stored value: - `id`: `x-mitre-collection--` — stable across exports of the same track - - `name`/`created_by_ref`/`object_marking_refs`: from the snapshot metadata + - `created_by_ref`: the configured organization identity's STIX ID + - `name`/`object_marking_refs`: from the snapshot metadata - `description`: from `snapshot_description` when present, otherwise the snapshot's long-lived track `description` - `x_mitre_version`: the snapshot's tagged version, or `0.1` for drafts @@ -146,6 +148,13 @@ STIX version serialization. The pipeline: until the graph is deleted; callers then edit the notes and regenerate the graph and hashes. +The `20260805150000-repair-release-track-bundle-integrity` forward migration +applies these invariants to existing graph manifests. It creates or rewrites +each frozen collection entry with the track-derived ID and current configured +organization identity, then recomputes both hashes for every linked tagged +snapshot. Historical draft graphs remain live exports and therefore do not +retain deterministic hashes. + ### Canonical domains and the legacy graph renderer Domain membership is object data, not an export projection. A cross-domain diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index f8e8568e..3686193b 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -146,7 +146,7 @@ self-contained. | ----------------------------------- | -------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `format` | `bundle` \| `workbench` \| `filesystemstore` | `bundle` | Output format (`filesystemstore` is not yet implemented) | | `stixVersion` | `2.0` \| `2.1` | `2.1` | STIX version the emitted bundle conforms to (bundle format only) | -| `includeToc` | `true` \| `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) in the bundle. The TOC is generated with `x_mitre_version: "0.1"` (signifying an ephemeral, non-release-track collection), a `modified` of the current timestamp, and the deployment's default ATT&CK spec version. | +| `includeToc` | `true` \| `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) in STIX 2.1. STIX 2.0 always omits it. The TOC uses `x_mitre_version: "0.1"`, the current timestamp, and the deployment's default ATT&CK spec version. | | `includeObjectsWithMissingAttackId` | `true` \| `false` | `false` | Include objects that should have an ATT&CK ID set but do not | | `includeDeprecated` | `true` \| `false` | `false` | Include objects with `x_mitre_deprecated: true` (this also governs deprecated Data Sources) | | `includeRevoked` | `true` \| `false` | `false` | Include objects with `revoked: true` | @@ -368,7 +368,7 @@ Workbench responses return the release-track snapshot shape. Entries in the `mem | `include` | `staged` and/or `candidates` (comma-separated or repeated) | Additional tiers to include in the bundle alongside members. If omitted, only members are included. (Note the different semantics from `workbench` responses.) | | `state` | `work-in-progress` and/or `awaiting-review` (comma-separated or repeated) | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included. Members are unaffected. | | `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`) | -| `includeToc` | `true` \| `false` | Include a table-of-contents object (of type `x-mitre-collection`) derived from the release-track metadata (default: `true`) | +| `includeToc` | `true` \| `false` | Include a table-of-contents object (of type `x-mitre-collection`) in STIX 2.1, derived from release-track metadata (default: `true`). STIX 2.0 always omits it. | See [Output Formats](output-formats.md) for details on the bundle structure. @@ -732,10 +732,12 @@ deterministic. Graph creation also stores one stateful `x-mitre-collection` manifest entry. Its ID is stable for the release track, `created` comes from the track's first -cached collection object, and `modified` is the current manifest creation time. -The graph-backed bundle envelope uses the manifest UUID, so repeated STIX 2.0 -or STIX 2.1 downloads are byte-for-byte stable. The graph-creation response and -snapshot history expose SHA-256 hashes for both exact download files. +cached collection object, `created_by_ref` is the configured organization +identity's STIX ID, and `modified` is the current manifest creation time. The +collection object is emitted only in STIX 2.1. The graph-backed bundle envelope +uses the manifest UUID, so repeated STIX 2.0 or STIX 2.1 downloads are +byte-for-byte stable. The graph-creation response and snapshot history expose +SHA-256 hashes for both exact download files. Administrators may use the separate `/graph/reconstruct` POST for a historical baseline backed by an independently verified source bundle. The request sends diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index fd2bf12b..3641015e 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -127,7 +127,7 @@ Standard STIX bundle format: | `include` | `staged`, `candidates` (comma-separated or repeated) | _(members only)_ | Additional tiers to include in the bundle alongside members | | `state` | `work-in-progress`, `awaiting-review` (comma-separated or repeated) | _(no filter)_ | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included, irrespective of this parameter. Members are unaffected. | | `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to | -| `includeToc` | `true`, `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) as the first object in the bundle | +| `includeToc` | `true`, `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) as the first object in STIX 2.1 bundles. STIX 2.0 bundles never include it. | Examples: @@ -141,16 +141,19 @@ GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=staged # Members + candidates and staged objects that are work-in-progress or reviewed GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged&state=work-in-progress -# STIX 2.0 bundle without a table of contents -GET /api/release-tracks/:id/snapshots/latest?format=bundle&stixVersion=2.0&includeToc=false +# STIX 2.0 bundle (the table of contents is always omitted) +GET /api/release-tracks/:id/snapshots/latest?format=bundle&stixVersion=2.0 ``` **The table of contents (TOC) object** -By default, bundles begin with an `x-mitre-collection` object that acts as a -table of contents. It is derived from the release-track metadata: +By default, STIX 2.1 bundles begin with an `x-mitre-collection` object that +acts as a table of contents. STIX 2.0 bundles omit this ATT&CK extension object +regardless of `includeToc`. The STIX 2.1 object is derived from the +release-track metadata: - `id` — stable per track (reuses the track UUID) +- `created_by_ref` — the deployment's configured organization identity - `name` — from the release track snapshot - `description` — from the snapshot's `snapshot_description`; falls back to the long-lived track `description` when no snapshot-local value is set diff --git a/migrations/20260805150000-repair-release-track-bundle-integrity.js b/migrations/20260805150000-repair-release-track-bundle-integrity.js new file mode 100644 index 00000000..fee6e61a --- /dev/null +++ b/migrations/20260805150000-repair-release-track-bundle-integrity.js @@ -0,0 +1,136 @@ +'use strict'; + +/** + * Repair frozen x-mitre-collection entries for persisted release-track graph + * manifests and recompute the exact STIX 2.0/2.1 download hashes for tagged + * snapshots. Draft snapshots may contain historical baseline manifests, but + * their exports remain live and therefore do not receive deterministic hashes. + */ + +const { isDeepStrictEqual } = require('node:util'); +const mongoose = require('mongoose'); +const logger = require('../app/lib/logger'); + +const MIGRATION_NAME = '20260805150000-repair-release-track-bundle-integrity'; + +function ensureMongooseUsesClient(client) { + if (client && mongoose.connection.readyState === 0) { + mongoose.connection.setClient(client); + } +} + +async function organizationIdentityRef(db) { + const systemConfig = await db + .collection('systemconfigurations') + .findOne({}, { sort: { created_at: -1 }, projection: { organization_identity_ref: 1 } }); + if (!systemConfig?.organization_identity_ref) { + throw new Error( + 'System configuration is missing organization_identity_ref; cannot repair graph bundles.', + ); + } + return systemConfig.organization_identity_ref; +} + +function expectedCollectionId(trackId) { + return `x-mitre-collection--${trackId.split('--')[1]}`; +} + +async function linkedGraphSnapshots(db) { + const manifests = await db + .collection('releaseTrackGraphManifests') + .find({ state: { $in: ['pending', 'active'] } }) + .sort({ track_id: 1, created_at: 1, _id: 1 }) + .toArray(); + const collectionNames = new Set( + (await db.listCollections({}, { nameOnly: true }).toArray()).map((entry) => entry.name), + ); + const linked = []; + + for (const manifest of manifests) { + if (!collectionNames.has(manifest.track_id)) continue; + const snapshot = await db.collection(manifest.track_id).findOne({ + graph_manifest_id: manifest.manifest_id, + modified: manifest.snapshot_modified, + }); + if (snapshot) linked.push({ manifest, snapshot }); + } + return { manifests, linked }; +} + +async function run(db, client, options = {}) { + ensureMongooseUsesClient(client); + const graphManifestService = require('../app/services/release-tracks/graph-manifest-service'); + const bundleHashService = require('../app/services/release-tracks/bundle-hash-service'); + const createdByRef = await organizationIdentityRef(db); + const { manifests, linked } = await linkedGraphSnapshots(db); + const report = { + manifests_scanned: manifests.length, + linked_snapshots: linked.length, + collection_entries_repaired: 0, + bundle_hashes_recomputed: 0, + draft_hashes_cleared: 0, + orphaned_manifests_skipped: manifests.length - linked.length, + dry_run: options.dryRun === true, + }; + + for (const { manifest, snapshot } of linked) { + const collectionEntry = await db.collection('releaseTrackGraphManifestEntries').findOne({ + manifest_id: manifest.manifest_id, + kind: 'collection', + }); + const collectionNeedsRepair = + !collectionEntry || + collectionEntry.object_ref !== expectedCollectionId(manifest.track_id) || + collectionEntry.revision_key !== `${expectedCollectionId(manifest.track_id)}::collection` || + collectionEntry.frozen_stix?.id !== expectedCollectionId(manifest.track_id) || + collectionEntry.frozen_stix?.created_by_ref !== createdByRef; + if (collectionNeedsRepair) report.collection_entries_repaired++; + + if (options.dryRun) { + if (typeof snapshot.version === 'string') report.bundle_hashes_recomputed++; + else if (snapshot.bundle_hashes) report.draft_hashes_cleared++; + continue; + } + + await graphManifestService.refreshCollectionEntry(snapshot, manifest); + + if (typeof snapshot.version !== 'string') { + if (snapshot.bundle_hashes) { + await db + .collection(manifest.track_id) + .updateOne({ _id: snapshot._id }, { $unset: { bundle_hashes: '' } }); + report.draft_hashes_cleared++; + } + continue; + } + + const bundleHashes = await bundleHashService.generateBundleHashes(snapshot); + if (!isDeepStrictEqual(snapshot.bundle_hashes, bundleHashes)) { + report.bundle_hashes_recomputed++; + await db + .collection(manifest.track_id) + .updateOne({ _id: snapshot._id }, { $set: { bundle_hashes: bundleHashes } }); + } + } + + return report; +} + +module.exports = { + async up(db, client) { + const report = await run(db, client); + logger.info(`[${MIGRATION_NAME}] ${JSON.stringify(report)}`); + }, + + async down() { + logger.info( + `[${MIGRATION_NAME}] down migration is a no-op: corrected collection identities and hashes are retained`, + ); + }, + + _private: { + run, + linkedGraphSnapshots, + expectedCollectionId, + }, +}; From b2ea6052775517873d21007d7a1ea29c55f97104 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:38:24 -0400 Subject: [PATCH 55/55] feat(config): expose REST API build information Source release metadata from runtime build variables and return it from the public system-version endpoint. Document Docker and non-Docker provenance and add API and configuration regressions. --- .../components/system-configuration.yml | 23 +++++++- .../paths/system-configuration-paths.yml | 9 ++-- app/config/config.js | 17 ++++++ .../system-configuration-controller.js | 2 +- app/routes/system-configuration-routes.js | 8 +-- .../system/system-configuration-service.js | 3 ++ .../system-configuration.spec.js | 13 ++--- app/tests/config/config.spec.js | 17 ++++++ docs/README.md | 2 + docs/admin/configuration.md | 21 +++++--- docs/developer/TODO.md | 28 ++++++++++ docs/developer/build-information.md | 53 +++++++++++++++++++ docs/user/build-information.md | 35 ++++++++++++ 13 files changed, 206 insertions(+), 25 deletions(-) create mode 100644 docs/developer/build-information.md create mode 100644 docs/user/build-information.md diff --git a/app/api/definitions/components/system-configuration.yml b/app/api/definitions/components/system-configuration.yml index 118eff14..791a0e85 100644 --- a/app/api/definitions/components/system-configuration.yml +++ b/app/api/definitions/components/system-configuration.yml @@ -2,13 +2,32 @@ components: schemas: system-version: type: object + required: + - name + - version + - gitCommit + - buildDate + - attackSpecVersion properties: + name: + type: string + description: Name of the REST API component + example: 'attack-workbench-rest-api' version: type: string - description: Version of the REST API software + description: Release version of the running REST API build + example: '4.20.0-beta.23' + gitCommit: + type: string + description: Git commit used to produce the running REST API build, or `unknown` when unavailable + example: 'c2c017c146fae040caba559333b35536bfbd1189' + buildDate: + type: string + description: RFC 3339 build timestamp, or `unknown` when unavailable + example: '2026-08-05T15:13:49.915Z' attackSpecVersion: type: string - description: ATT&CK spec version of the REST API software + description: ATT&CK specification version supported by the REST API allowed-values: type: object diff --git a/app/api/definitions/paths/system-configuration-paths.yml b/app/api/definitions/paths/system-configuration-paths.yml index cbdc29b9..4afa3b33 100644 --- a/app/api/definitions/paths/system-configuration-paths.yml +++ b/app/api/definitions/paths/system-configuration-paths.yml @@ -1,15 +1,18 @@ paths: /api/config/system-version: get: - summary: 'Get the system version info' + summary: 'Get the REST API build information' operationId: 'config-get-system-version' description: | - This endpoint gets the system version info from the package.json file. + This public endpoint returns the REST API release version, Git commit, + build date, and supported ATT&CK specification version. Container images + source the build fields from the same values used for their OCI labels; + non-container deployments use configured or package defaults. tags: - 'System Configuration' responses: '200': - description: 'System version info' + description: 'REST API build information' content: application/json: schema: diff --git a/app/config/config.js b/app/config/config.js index 4ff7b493..a5c6e941 100644 --- a/app/config/config.js +++ b/app/config/config.js @@ -161,6 +161,8 @@ function loadConfig() { }, app: { name: { + doc: 'Application name reported by the build information endpoint', + format: String, default: 'attack-workbench-rest-api', }, env: { @@ -168,7 +170,22 @@ function loadConfig() { env: 'NODE_ENV', }, version: { + doc: 'Application release version', + format: String, default: packageJson.version, + env: 'APP_VERSION', + }, + gitCommit: { + doc: 'Git commit used to build the application', + format: String, + default: 'unknown', + env: 'GIT_COMMIT', + }, + buildDate: { + doc: 'Timestamp when the application was built', + format: String, + default: 'unknown', + env: 'BUILD_DATE', }, attackSpecVersion: { default: packageJson.attackSpecVersion, diff --git a/app/controllers/system-configuration-controller.js b/app/controllers/system-configuration-controller.js index d968fe7e..9df7ec67 100644 --- a/app/controllers/system-configuration-controller.js +++ b/app/controllers/system-configuration-controller.js @@ -8,7 +8,7 @@ exports.retrieveSystemVersion = function (req, res, next) { try { const systemVersionInfo = SystemConfigurationService.retrieveSystemVersion(); logger.debug( - `Success: Retrieved system version, version: ${systemVersionInfo.version}, attackSpecVersion: ${systemVersionInfo.attackSpecVersion}`, + `Success: Retrieved system version, version: ${systemVersionInfo.version}, gitCommit: ${systemVersionInfo.gitCommit}, buildDate: ${systemVersionInfo.buildDate}, attackSpecVersion: ${systemVersionInfo.attackSpecVersion}`, ); return res.status(200).send(systemVersionInfo); } catch (err) { diff --git a/app/routes/system-configuration-routes.js b/app/routes/system-configuration-routes.js index 77cbe575..3776c0dd 100644 --- a/app/routes/system-configuration-routes.js +++ b/app/routes/system-configuration-routes.js @@ -8,13 +8,7 @@ const authz = require('../lib/authz-middleware'); const router = express.Router(); -router - .route('/config/system-version') - .get( - authn.authenticate, - authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), - systemConfigurationController.retrieveSystemVersion, - ); +router.route('/config/system-version').get(systemConfigurationController.retrieveSystemVersion); router .route('/config/allowed-values') diff --git a/app/services/system/system-configuration-service.js b/app/services/system/system-configuration-service.js index 00ca8002..371fe0b3 100644 --- a/app/services/system/system-configuration-service.js +++ b/app/services/system/system-configuration-service.js @@ -32,7 +32,10 @@ class SystemConfigurationService extends BaseService { */ static retrieveSystemVersion() { return { + name: config.app.name, version: config.app.version, + gitCommit: config.app.gitCommit, + buildDate: config.app.buildDate, attackSpecVersion: config.app.attackSpecVersion, }; } diff --git a/app/tests/api/system-configuration/system-configuration.spec.js b/app/tests/api/system-configuration/system-configuration.spec.js index 62f899f1..f65629ad 100644 --- a/app/tests/api/system-configuration/system-configuration.spec.js +++ b/app/tests/api/system-configuration/system-configuration.spec.js @@ -52,15 +52,16 @@ describe('System Configuration API', function () { const res = await request(app) .get('/api/config/system-version') .set('Accept', 'application/json') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200) .expect('Content-Type', /json/); - // We expect to get the system version info - const systemVersionInfo = res.body; - expect(systemVersionInfo).toBeDefined(); - expect(systemVersionInfo.version).toBeDefined(); - expect(systemVersionInfo.attackSpecVersion).toBeDefined(); + expect(res.body).toEqual({ + name: config.app.name, + version: config.app.version, + gitCommit: config.app.gitCommit, + buildDate: config.app.buildDate, + attackSpecVersion: config.app.attackSpecVersion, + }); }); it('GET /api/config/allowed-values returns the allowed values', async function () { diff --git a/app/tests/config/config.spec.js b/app/tests/config/config.spec.js index 25e2f7c2..9907e349 100644 --- a/app/tests/config/config.spec.js +++ b/app/tests/config/config.spec.js @@ -33,6 +33,23 @@ describe('App Configuration', function () { done(); }); + it('loads build information from runtime environment variables', function () { + process.env.APP_VERSION = '4.20.0-beta.23'; + process.env.GIT_COMMIT = 'c2c017c146fae040caba559333b35536bfbd1189'; + process.env.BUILD_DATE = '2026-08-05T15:13:49.915Z'; + + config.reloadConfig(); + + expect(config.app.version).toBe(process.env.APP_VERSION); + expect(config.app.gitCommit).toBe(process.env.GIT_COMMIT); + expect(config.app.buildDate).toBe(process.env.BUILD_DATE); + + delete process.env.APP_VERSION; + delete process.env.GIT_COMMIT; + delete process.env.BUILD_DATE; + config.reloadConfig(); + }); + describe('CORS Configuration', function () { it('should accept wildcard origin', function () { expect(() => config.reloadConfig()).not.toThrow(); diff --git a/docs/README.md b/docs/README.md index e1f4d9b9..505fdb69 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ This directory contains supplementary technical documentation for the ATT&CK Wor Guides for consumers of the REST API — endpoints, workflows, and terminology. +- [Build Information](user/build-information.md): Inspect the running REST API release and build provenance - [Revoke Workflow](user/revoke-workflow.md): How to revoke ATT&CK objects via the API ### Release Tracks @@ -28,6 +29,7 @@ Guides for consumers of the REST API — endpoints, workflows, and terminology. Architecture, patterns, and implementation details for contributors. +- [Build Information](developer/build-information.md): Build metadata provenance, runtime configuration, and frontend integration - [Data Model](developer/data-model.md): Database schema and STIX object structure - [Event Bus Architecture](developer/event-bus-architecture.md): Event-driven architecture for cross-document dependencies - [Lifecycle Hooks Guide](developer/lifecycle-hooks-guide.md): Service lifecycle hooks pattern diff --git a/docs/admin/configuration.md b/docs/admin/configuration.md index eee484d4..4fa6fede 100644 --- a/docs/admin/configuration.md +++ b/docs/admin/configuration.md @@ -205,19 +205,28 @@ DATABASE_URL=mongodb://attack-workbench-database/attack-workspace General application settings. -| Option | Environment Variable | JSON Path | Type | Default | Description | -|---------------------|----------------------|-------------------------|--------|-----------------------------|-----------------------------------------------------------| -| Name | *(none)* | `app.name` | string | `attack-workbench-rest-api` | Application name | -| Environment | `NODE_ENV` | `app.env` | string | `development` | Environment name (`development`, `production`, `test`) | -| Version | *(none)* | `app.version` | string | *(from package.json)* | Application version | -| ATT&CK Spec Version | *(none)* | `app.attackSpecVersion` | string | *(from package.json)* | ATT&CK specification version | +| Option | Environment Variable | JSON Path | Type | Default | Description | +| ------------------- | -------------------- | ----------------------- | ------ | --------------------------- | ------------------------------------------------------ | +| Name | _(none)_ | `app.name` | string | `attack-workbench-rest-api` | Application name | +| Environment | `NODE_ENV` | `app.env` | string | `development` | Environment name (`development`, `production`, `test`) | +| Version | `APP_VERSION` | `app.version` | string | _(from package.json)_ | Running application release version | +| Git commit | `GIT_COMMIT` | `app.gitCommit` | string | `unknown` | Commit used to produce the running build | +| Build date | `BUILD_DATE` | `app.buildDate` | string | `unknown` | RFC 3339 timestamp when the build was produced | +| ATT&CK Spec Version | _(none)_ | `app.attackSpecVersion` | string | _(from package.json)_ | ATT&CK specification version | **Example:** ```bash NODE_ENV=production +APP_VERSION=4.20.0-beta.23 +GIT_COMMIT=c2c017c146fae040caba559333b35536bfbd1189 +BUILD_DATE=2026-08-05T15:13:49.915Z ``` +The published Docker image sets the three build variables automatically from +the same build arguments used for its OCI image labels. Source deployments can +set them explicitly; omitted commit and date values are reported as `unknown`. + ### Logging Logging configuration using Winston. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 2ef45982..4eb48f11 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,33 @@ # Release Track TODOs +## Frontend and REST API build information + +- [x] Source REST API build metadata from the Docker/runtime build variables, + with package/default fallbacks for non-Docker development. +- [x] Extend the public system-version endpoint and OpenAPI contract with the + REST API name, release version, Git commit, and build date. +- [x] Generate matching frontend metadata into production build artifacts and + display frontend plus REST API versions in the navigation footer. +- [x] Add REST API and frontend regressions for metadata loading, fallbacks, + endpoint access, and rendering. +- [x] Update REST API user/developer/admin docs, frontend docs, and the Bruno + collection for the expanded API response. +- [x] Run focused checks followed by the complete REST API and frontend test + suites, then propose conventional commit messages without committing. + +Verification (2026-08-07): + +- REST API focused system-version (19), configuration (22), and OpenAPI (2) + regressions pass; ESLint and whitespace checks also pass. +- The clean complete REST API suite passes: OpenAPI 2, configuration 22, API + 1012, middleware 29, and scheduler 10. An earlier run's documented roaming + `ECONNRESET` passed in isolation before the clean rerun. +- The complete frontend suite passes: 166 files and 385 tests. Focused service, + footer, and navigation tests (22), ESLint, Prettier, the metadata generator, + and a production build with release-like metadata also pass. +- Proposed REST API commit: `feat(config): expose REST API build information`. + Proposed frontend commit: `feat(shell): display component build versions`. + ## Deterministic graph collection identity repair - [x] Reproduce the incorrect graph collection creator, STIX 2.0 TOC diff --git a/docs/developer/build-information.md b/docs/developer/build-information.md new file mode 100644 index 00000000..8d8045ed --- /dev/null +++ b/docs/developer/build-information.md @@ -0,0 +1,53 @@ +# Build Information Architecture + +Build metadata follows the artifact from semantic release to the user-facing +Workbench navigation without requiring a release process to modify tracked +source files. + +| Meaning | Docker build argument | OCI image label | Runtime variable | API/asset field | +| --------------- | --------------------- | ----------------------------------- | ---------------- | --------------- | +| Release version | `VERSION` | `org.opencontainers.image.version` | `APP_VERSION` | `version` | +| Source commit | `REVISION` | `org.opencontainers.image.revision` | `GIT_COMMIT` | `gitCommit` | +| Build timestamp | `BUILDTIME` | `org.opencontainers.image.created` | `BUILD_DATE` | `buildDate` | + +## REST API + +`app/config/config.js` maps the three runtime variables into `config.app`. +`SystemConfigurationService.retrieveSystemVersion()` returns them with the +component name and supported ATT&CK specification version from the existing +public `GET /api/config/system-version` endpoint. + +The Dockerfile already receives all three values from +`@codedependant/semantic-release-docker` and exposes them as both labels and +environment variables. OCI labels cannot be read portably from inside a +running container, so the service uses the environment-variable copy. + +When no build environment is present, `version` falls back to `package.json`; +`gitCommit` and `buildDate` fall back to `unknown`. Operators of non-container +artifacts can set `APP_VERSION`, `GIT_COMMIT`, and `BUILD_DATE` when launching +Node. A JSON configuration file can also set `app.version`, `app.gitCommit`, +and `app.buildDate` under the repository's normal configuration precedence. + +## Frontend + +The frontend is a static Angular application, so environment variables on its +Nginx process are not visible in browser JavaScript. Both `npm run build` and +`npm run build-prod` therefore run `scripts/write-build-info.mjs` as a +post-build step. It writes: + +```text +dist/app/browser/assets/build-info.json +``` + +The frontend Dockerfile exposes `VERSION`, `REVISION`, and `BUILDTIME` to the +Angular build stage. The generated asset consequently matches the image's OCI +labels. A source build uses the frontend package version and `unknown` +provenance values unless the same three runtime variables are supplied to the +build command. `ng serve` uses the checked-in development asset under +`src/assets/build-info.json`. + +`BuildInfoService` loads that local asset and the REST API system-version +endpoint in parallel, caches the completed result, and substitutes safe +fallbacks if either component is unavailable. The navigation footer displays +both versions; native title text exposes commit and build-date details without +adding visual noise to the navigation. diff --git a/docs/user/build-information.md b/docs/user/build-information.md new file mode 100644 index 00000000..31436293 --- /dev/null +++ b/docs/user/build-information.md @@ -0,0 +1,35 @@ +# Build Information + +The REST API exposes the running component's build information through a +public endpoint: + +```http +GET /api/config/system-version +``` + +No login or service credential is required. A response has this shape: + +```json +{ + "name": "attack-workbench-rest-api", + "version": "4.20.0-beta.23", + "gitCommit": "c2c017c146fae040caba559333b35536bfbd1189", + "buildDate": "2026-08-05T15:13:49.915Z", + "attackSpecVersion": "3.3.0" +} +``` + +`version`, `gitCommit`, and `buildDate` identify the deployed REST API +artifact. `attackSpecVersion` is separate: it identifies the ATT&CK +specification version supported by that API build. + +Published Docker images populate the build fields from the same values used +for the `org.opencontainers.image.version`, +`org.opencontainers.image.revision`, and `org.opencontainers.image.created` +labels. A non-container source deployment falls back to the package version +and reports unavailable commit or date values as `unknown` unless its operator +sets the corresponding runtime configuration. + +The Workbench frontend shows its own version and the REST API version at the +bottom of the primary navigation. Hover over either value to see its commit +and build date.