From 362613e1c2bfb3c473bb767e9152d33a952cf982 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 21:21:27 +0800 Subject: [PATCH 01/22] feat(storage): establish durable message admission Generated-by: Codex --- .../sqlite-session-metadata-store.test.ts | 53 ++++- packages/storage/src/message-receipt-store.ts | 70 +++++++ .../src/sqlite-session-metadata-schema.ts | 29 ++- .../src/sqlite-session-metadata-store.ts | 192 ++++++++++++++++++ 4 files changed, 342 insertions(+), 2 deletions(-) diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 124f1e7d8a..6f796e9a54 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -45,6 +45,7 @@ import { type SessionConfigurationMetadataUpdate, type SqliteSessionMetadataStoreFailpoint, } from '../sqlite-session-metadata-store.js'; +import type { PendingMessageAdmission } from '../message-receipt-store.js'; import { createSqliteRuntimeStore, SQLITE_RUNTIME_SCHEMA_VERSION, @@ -84,7 +85,7 @@ describe('SqliteSessionMetadataStore', () => { const migrated = createSqliteSessionMetadataStore(path); try { - assert.equal(migrated.schemaVersion(), 29); + assert.equal(migrated.schemaVersion(), SQLITE_SESSION_METADATA_SCHEMA_VERSION); assert.equal((await migrated.read(legacyHeader.id)).header.externalOrigin, undefined); } finally { migrated.close(); @@ -237,6 +238,56 @@ describe('SqliteSessionMetadataStore', () => { } }); + test('atomically accepts a steering message and its canonical transcript', async () => { + const store = createSqliteSessionMetadataStore(':memory:'); + try { + await store.create(fullHeader({ id: 'session-1' })); + const admission: PendingMessageAdmission = { + sessionId: 'session-1', + turnId: 'turn-1', + runId: 'run-1', + messageId: 'message-1', + content: { text: 'submitted', displayText: 'submitted' }, + modelContent: { text: 'submitted', displayText: 'submitted' }, + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt: 10, + }; + + const normalizedAdmission = { + ...admission, + content: { text: 'submitted' }, + modelContent: { text: 'submitted' }, + }; + assert.deepEqual(await store.commitMessageAdmission(admission), normalizedAdmission); + assert.deepEqual( + await store.readMessageAdmission('session-1', 'message-1'), + normalizedAdmission, + ); + assert.deepEqual( + (await store.readMessages('session-1')).map((message) => ({ + id: message.id, + type: message.type, + turnId: message.turnId, + text: message.type === 'user' ? message.text : undefined, + steeringEventId: message.type === 'user' ? message.steeringEventId : undefined, + })), + [ + { + id: 'message-1', + type: 'user', + turnId: 'turn-1', + text: 'submitted', + steeringEventId: 'message-1', + }, + ], + ); + } finally { + store.close(); + } + }); + test('migrates v24 legacy session statuses to active exactly once', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-status-v24-')); const path = join(root, 'state.sqlite'); diff --git a/packages/storage/src/message-receipt-store.ts b/packages/storage/src/message-receipt-store.ts index 82bcd27277..b9dfeb320e 100644 --- a/packages/storage/src/message-receipt-store.ts +++ b/packages/storage/src/message-receipt-store.ts @@ -20,6 +20,7 @@ import { resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import type { DatabaseSync } from 'node:sqlite'; +import { normalizeMessageContent, type MessageContent } from '@maka/core/events'; import { acquireOperationalStateDatabase, type OperationalStateDatabaseLease, @@ -29,6 +30,75 @@ const SAFE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; const RECEIPT_SCHEMA_VERSION = 1 as const; const RECEIPT_MAX_BYTES = 64 * 1024; +export type MessageLifecycleState = 'accepted' | 'handed_off' | 'executed' | 'cancelled'; + +export interface PendingMessageAdmission { + readonly sessionId: string; + readonly turnId: string; + readonly runId: string; + readonly messageId: string; + readonly content: MessageContent; + readonly modelContent: MessageContent; + readonly submittedPlacement: 'current_turn' | 'next_turn'; + readonly placement: 'current_turn' | 'next_turn'; + readonly disposition: 'steering' | 'followup'; + readonly admittedAt: number; +} + +export function normalizePendingMessageAdmission( + admission: PendingMessageAdmission, +): PendingMessageAdmission { + for (const [name, value] of [ + ['Session', admission.sessionId], + ['Turn', admission.turnId], + ['Run', admission.runId], + ['Message', admission.messageId], + ] as const) { + assertSafeId(value, `Invalid ${name} identity`); + } + if ( + (admission.submittedPlacement !== 'current_turn' && + admission.submittedPlacement !== 'next_turn') || + (admission.placement !== 'current_turn' && admission.placement !== 'next_turn') || + (admission.disposition !== 'steering' && admission.disposition !== 'followup') || + (admission.placement === 'current_turn') !== (admission.disposition === 'steering') + ) { + throw new Error('Invalid pending Message placement'); + } + if (!Number.isSafeInteger(admission.admittedAt) || admission.admittedAt < 0) { + throw new Error('Invalid message admission timestamp'); + } + const normalized = Object.freeze({ + ...admission, + content: normalizeMessageContent(admission.content), + modelContent: normalizeMessageContent(admission.modelContent), + }); + if (Buffer.byteLength(JSON.stringify(normalized), 'utf8') > RECEIPT_MAX_BYTES) { + throw new Error('Pending message admission exceeds size limit'); + } + return normalized; +} + +export function samePendingMessageAdmission( + left: PendingMessageAdmission, + right: PendingMessageAdmission, +): boolean { + const a = normalizePendingMessageAdmission(left); + const b = normalizePendingMessageAdmission(right); + return ( + a.sessionId === b.sessionId && + a.turnId === b.turnId && + a.runId === b.runId && + a.messageId === b.messageId && + a.submittedPlacement === b.submittedPlacement && + a.placement === b.placement && + a.disposition === b.disposition && + a.admittedAt === b.admittedAt && + isDeepStrictEqual(a.content, b.content) && + isDeepStrictEqual(a.modelContent, b.modelContent) + ); +} + export type MessageReceiptOperation = | 'submit' | 'retract' diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index c7b92e8e1a..ef08df2717 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -19,7 +19,7 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 29; +export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 30; export const SQLITE_SESSION_MESSAGE_CHUNK_BYTES = 64 * 1024; export const SQLITE_SESSION_MESSAGE_CHUNK_MARKER = '{"$maka":"session-message-chunks-v1"}'; @@ -820,6 +820,33 @@ const MIGRATIONS: ReadonlyMap = new Map([ ON session_messages(session_id, message_ts, sequence); `, ], + [ + 30, + ` + CREATE TABLE IF NOT EXISTS message_admissions ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + run_id TEXT NOT NULL, + message_id TEXT NOT NULL, + content_json TEXT NOT NULL, + model_content_json TEXT NOT NULL, + submitted_placement TEXT NOT NULL + CHECK (submitted_placement IN ('current_turn', 'next_turn')), + placement TEXT NOT NULL CHECK (placement IN ('current_turn', 'next_turn')), + disposition TEXT NOT NULL CHECK (disposition IN ('steering', 'followup')), + lifecycle_state TEXT NOT NULL + CHECK (lifecycle_state IN ('accepted', 'handed_off', 'executed', 'cancelled')), + queue_order INTEGER NOT NULL CHECK (queue_order >= 0), + admitted_at INTEGER NOT NULL CHECK (admitted_at >= 0), + UNIQUE (session_id, message_id), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS message_admissions_by_session_order + ON message_admissions(session_id, lifecycle_state, queue_order, sequence); + `, + ], [ 21, ` diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 3e13d67460..03395bb0ae 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -96,6 +96,12 @@ import { decodeStoredMessage as decodePersistedStoredMessage, } from '@maka/core/session'; import { markPersisted } from '@maka/core/persisted-value'; +import { + normalizePendingMessageAdmission, + samePendingMessageAdmission, + type MessageLifecycleState, + type PendingMessageAdmission, +} from './message-receipt-store.js'; import { type AgentGraphIntentAdmissionSnapshot, type AgentGraphTimelineMetadataSnapshot, @@ -219,6 +225,59 @@ export interface SessionCatalogMessageProjection { readonly lastMessagePreview?: string; } +interface MessageAdmissionRow { + readonly turn_id?: unknown; + readonly run_id?: unknown; + readonly message_id?: unknown; + readonly content_json?: unknown; + readonly model_content_json?: unknown; + readonly submitted_placement?: unknown; + readonly placement?: unknown; + readonly disposition?: unknown; + readonly lifecycle_state?: unknown; + readonly queue_order?: unknown; + readonly admitted_at?: unknown; +} + +function decodeMessageAdmissionRow( + sessionId: string, + row: MessageAdmissionRow, +): { readonly admission: PendingMessageAdmission; readonly lifecycleState: MessageLifecycleState } { + if ( + typeof row.turn_id !== 'string' || + typeof row.run_id !== 'string' || + typeof row.message_id !== 'string' || + typeof row.content_json !== 'string' || + typeof row.model_content_json !== 'string' || + (row.submitted_placement !== 'current_turn' && row.submitted_placement !== 'next_turn') || + (row.placement !== 'current_turn' && row.placement !== 'next_turn') || + (row.disposition !== 'steering' && row.disposition !== 'followup') || + (row.lifecycle_state !== 'accepted' && + row.lifecycle_state !== 'handed_off' && + row.lifecycle_state !== 'executed' && + row.lifecycle_state !== 'cancelled') || + typeof row.queue_order !== 'number' || + !Number.isSafeInteger(row.queue_order) || + row.queue_order < 0 || + typeof row.admitted_at !== 'number' + ) { + throw new SessionMetadataConflictError(`Invalid Message admission row for ${sessionId}`); + } + const admission = normalizePendingMessageAdmission({ + sessionId, + turnId: row.turn_id, + runId: row.run_id, + messageId: row.message_id, + content: JSON.parse(row.content_json) as PendingMessageAdmission['content'], + modelContent: JSON.parse(row.model_content_json) as PendingMessageAdmission['modelContent'], + submittedPlacement: row.submitted_placement, + placement: row.placement, + disposition: row.disposition, + admittedAt: row.admitted_at, + }); + return { admission, lifecycleState: row.lifecycle_state }; +} + export interface SessionAuthoritySnapshot { record: SessionMetadataRecord; boundary: ExecutionBoundary; @@ -1478,6 +1537,139 @@ export class SqliteSessionMetadataStore { }); } + async commitMessageAdmission( + admission: PendingMessageAdmission, + ): Promise { + this.assertOpen(); + const stored = normalizePendingMessageAdmission(admission); + return this.transaction(() => { + const record = this.readRecordSync(stored.sessionId); + if (!record) throw new SessionNotFoundError(stored.sessionId); + const existingRow = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, model_content_json, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(stored.sessionId, stored.messageId) as MessageAdmissionRow | undefined; + if (existingRow) { + const existing = decodeMessageAdmissionRow(stored.sessionId, existingRow); + if (!samePendingMessageAdmission(existing.admission, stored)) { + throw new SessionMetadataConflictError('Message admission identity conflict'); + } + if (existing.lifecycleState !== 'accepted') { + throw new SessionMetadataConflictError('Message admission identity is already settled'); + } + return existing.admission; + } + const orderRow = this.db + .prepare( + ` + SELECT COALESCE(MAX(queue_order), -1) + 1 AS next_order + FROM message_admissions + WHERE session_id = ? AND lifecycle_state = 'accepted' + `, + ) + .get(stored.sessionId) as { next_order?: unknown }; + if (typeof orderRow.next_order !== 'number' || !Number.isSafeInteger(orderRow.next_order)) { + throw new SessionMetadataConflictError('Invalid message admission order'); + } + this.db + .prepare( + ` + INSERT INTO message_admissions( + session_id, turn_id, run_id, message_id, content_json, model_content_json, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'accepted', ?, ?) + `, + ) + .run( + stored.sessionId, + stored.turnId, + stored.runId, + stored.messageId, + JSON.stringify(stored.content), + JSON.stringify(stored.modelContent), + stored.submittedPlacement, + stored.placement, + stored.disposition, + orderRow.next_order, + stored.admittedAt, + ); + + if (stored.disposition === 'steering') { + const message = decodeCanonicalMessage({ + type: 'user', + id: stored.messageId, + turnId: stored.turnId, + ts: stored.admittedAt, + ...stored.content, + steeringEventId: stored.messageId, + }); + const existingMessages = this.readMessagesWith(stored.sessionId, decodeStoredMessage).filter( + (candidate) => candidate.id === stored.messageId, + ); + if (existingMessages.length > 1) { + throw new SessionMetadataConflictError('Message admission transcript identity is ambiguous'); + } + const existingMessage = existingMessages[0]; + if (existingMessage && !isDeepStrictEqual(existingMessage, message)) { + throw new SessionMetadataConflictError('Message admission transcript identity conflict'); + } + if (!existingMessage) { + const row = this.db + .prepare( + 'SELECT COALESCE(MAX(sequence), -1) AS last_sequence FROM session_messages WHERE session_id = ?', + ) + .get(stored.sessionId) as { last_sequence?: unknown }; + if (typeof row.last_sequence !== 'number' || !Number.isSafeInteger(row.last_sequence)) { + throw new SessionMetadataConflictError('Invalid Session message sequence'); + } + const json = JSON.stringify(message); + this.insertSessionMessagesSync(stored.sessionId, row.last_sequence + 1, [ + { message, json }, + ]); + this.updateCatalogProjectionSync( + stored.sessionId, + { + lastMessageAt: stored.admittedAt, + lastMessagePreview: message.type === 'user' ? message.displayText : undefined, + }, + false, + !record.header.connectionLocked, + ); + } + } + return stored; + }); + } + + async readMessageAdmission( + sessionId: string, + messageId: string, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + assertSafeSessionId(messageId); + return this.readTransaction(() => { + const row = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, model_content_json, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + AND lifecycle_state = 'accepted' + `, + ) + .get(sessionId, messageId) as MessageAdmissionRow | undefined; + return row ? decodeMessageAdmissionRow(sessionId, row).admission : undefined; + }); + } + async readMessages(sessionId: string): Promise { return this.readMessagesWith(sessionId, decodeStoredMessage); } From a1011c7317198cd8e6b583312cf304ca3aec838d Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 21:41:14 +0800 Subject: [PATCH 02/22] feat(runtime-host): wire durable message lifecycle Generated-by: Codex --- .../__tests__/execution-host-queue.test.ts | 80 ++++++ .../fixtures/execution-host-suite.ts | 12 + .../src/server/execution-composition.ts | 4 + .../src/server/message-coordinator.ts | 194 ++++++++++++++ .../src/server/root-turn-coordinator.ts | 128 ++++++++- packages/runtime/src/agent-run.ts | 26 +- .../sqlite-session-metadata-store.test.ts | 14 + packages/storage/src/execution-stores.ts | 24 ++ packages/storage/src/message-receipt-store.ts | 18 ++ packages/storage/src/session-store.ts | 60 ++++- .../src/sqlite-session-metadata-store.ts | 253 ++++++++++++++++++ 11 files changed, 799 insertions(+), 14 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index 6cc97ee966..77703f95e5 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -205,6 +205,7 @@ test('subscribed Clients share one canonical queue and ordered root handoff', as await waitForTerminalTurn(tui, fixture.sessionId, successor.snapshot.rootTurn.turnId); await tui.close(); await fixture.stopHost(host); + assert.equal(await fixture.readMessageLifecycleState(followupId), 'handed_off'); const chain = await fixture.readAdmissionChain(); assert.deepEqual( @@ -215,6 +216,85 @@ test('subscribed Clients share one canonical queue and ordered root handoff', as }); }); +test('production UDS admission commits one transcript before the root handoff', async () => { + await withExecutionRoot(async (fixture) => { + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const messageId = randomUUID(); + const started = await client.request('turn.message.submit', { + originHostEpoch: host.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: FAKE_ASK_USER_QUESTION_PROMPT }, + placement: 'current_turn', + }); + assert.equal(started.disposition, 'turn_started'); + if (started.disposition !== 'turn_started') return; + const active = await client.queryTurn({ sessionId: fixture.sessionId, turnId: started.turnId }); + await client.stopTurn({ + sessionId: fixture.sessionId, + turnId: started.turnId, + runId: active.runId, + }); + await client.close(); + await fixture.stopHost(host); + const ledger = await fixture.readTurn(started.turnId); + assert.deepEqual( + ledger.userMessages.filter((message) => message.id === messageId).map((message) => message.id), + [messageId], + ); + assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); + }); +}); + +test('a Host crash after queue admission recovers the durable successor once', async () => { + await withExecutionRoot(async (fixture) => { + const firstHost = await fixture.startHost(); + const first = await connectClient(fixture.root); + const started = requireStartedTurn( + await first.startTurn({ + sessionId: fixture.sessionId, + turnId: randomUUID(), + content: { text: FAKE_ASK_USER_QUESTION_PROMPT }, + }), + ); + const messageId = randomUUID(); + const queued = await first.request('turn.message.submit', { + originHostEpoch: firstHost.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: 'recover this accepted successor' }, + placement: 'next_turn', + }); + assert.equal(queued.disposition, 'followup'); + await fixture.killHost(firstHost); + await first.closed; + + const secondHost = await fixture.startHost(); + const second = await connectClient(fixture.root); + const subscription = await second.openSessionSubscription({ + sessionId: fixture.sessionId, + transcript: { kind: 'none' }, + }); + const probe = new SubscriptionProbe(subscription); + const successor = await probe.waitFor( + (frame) => + frame.kind === 'subscription.session_projection' && + frame.snapshot.rootTurn !== null && + frame.snapshot.rootTurn.turnId !== started.turnId, + 'durable successor was not recovered after the Host crash', + ); + assert.equal(successor.kind, 'subscription.session_projection'); + if (successor.kind !== 'subscription.session_projection' || !successor.snapshot.rootTurn) return; + await waitForTerminalTurn(second, fixture.sessionId, successor.snapshot.rootTurn.turnId); + await subscription.close(); + await probe.done; + await second.close(); + await fixture.stopHost(secondHost); + assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); + }); +}); + test('concurrent root admission for one Session has a single winner', async () => { await withExecutionRoot(async (fixture) => { const host = await fixture.startHost(); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index f44e88b8fa..23d8b942a3 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -957,6 +957,18 @@ export class ExecutionFixture { } } + async readMessageLifecycleState(messageId: string) { + const reader = await acquireReader(this.capability); + let stores: Awaited> | undefined; + try { + stores = await openInteractiveExecutionStoresForRead(reader.lease); + return await stores.sessionStore.readMessageLifecycleState(this.sessionId, messageId); + } finally { + await stores?.sessionStore.close?.(); + await reader.close(); + } + } + async readTurnFootprint(turnId: string): Promise<{ admitted: boolean; runCount: number; diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 639a8beb7e..c19fc0e4d2 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -468,6 +468,8 @@ export async function createExecutionRuntimeHostComposition( requireRootCoordinator(rootCoordinator).claimStopFence(input, commitQueueFence, admission), startFromMessage: (input, admission) => requireRootCoordinator(rootCoordinator).startFromMessage(input, admission), + startRecoveredMessages: (input, admission) => + requireRootCoordinator(rootCoordinator).startRecoveredMessages(input, admission), prepareMessage: (input) => requireRootCoordinator(rootCoordinator).prepareMessage(input), claimStop: (input, commitQueueFence, admission) => requireRootCoordinator(rootCoordinator).claimStop(input, commitQueueFence, admission), @@ -482,6 +484,7 @@ export async function createExecutionRuntimeHostComposition( stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission, acquireResidency: () => context.acquireResidency('message-queue'), requestDrain: context.requestDrain, @@ -1485,6 +1488,7 @@ export async function createExecutionRuntimeHostComposition( ), ); await coordinator.recover(); + await messages.recoverPendingAfterHostRestart(recoverySessions.map((session) => session.id)); rootRecoveryCompleted = true; }, }, diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index 8d8ac47e9b..dab6ddc675 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -36,8 +36,10 @@ import { import { normalizeRootTurnAdmissionPayload, type ImmutableSteeringMessageProof, + type MessageLifecycleStore, type MessageReceiptOperation, type MessageReceiptStore, + type PendingMessageAdmission, type RootTurnSourceMessage, type RootTurnSourceMessageReceipt, } from '@maka/storage/execution-stores'; @@ -104,6 +106,15 @@ export interface HostMessageStartInput { readonly content: MessageContent; readonly sourceMessage: RootTurnSourceMessage; readonly initiatingConnectionId: string; + readonly turnId?: string; + readonly runId?: string; +} + +export interface HostMessageRecoveryBatch { + readonly sessionId: string; + readonly content: MessageContent; + readonly submittedContent: MessageContent; + readonly sources: readonly RootTurnSourceMessage[]; } export interface HostMessagePreparationInput { @@ -137,6 +148,10 @@ export interface HostMessageRootPort { input: HostMessageStartInput, admission: SessionAdmissionLease, ): Promise<{ readonly turnId: string } | { readonly error: string }>; + startRecoveredMessages?( + input: HostMessageRecoveryBatch, + admission: SessionAdmissionLease, + ): Promise<{ readonly turnId: string } | { readonly error: string }>; prepareMessage( input: HostMessagePreparationInput, ): Promise< @@ -167,6 +182,7 @@ export interface HostMessageCoordinatorOptions { readonly root: HostMessageRootPort; readonly durableProof: HostMessageDurableProofReader; readonly receipts: MessageReceiptStore; + readonly lifecycle?: MessageLifecycleStore; readonly sessionAdmission: SessionAdmissionGate; readonly acquireResidency: () => RuntimeHostResidency; readonly requestDrain?: () => void; @@ -186,6 +202,9 @@ export type CandidateSnapshotPreflight = ( interface LiveEntry { readonly entryId: string; readonly messageId: string; + readonly turnId: string; + readonly runId: string; + readonly admittedAt: number; content: MessageContent; modelContent: MessageContent; readonly initiatingConnectionId: string; @@ -310,6 +329,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { readonly #root: HostMessageRootPort; readonly #durableProof: HostMessageDurableProofReader; readonly #receipts: MessageReceiptStore; + readonly #lifecycle?: MessageLifecycleStore; readonly #sessionAdmission: SessionAdmissionGate; readonly #acquireResidency: () => RuntimeHostResidency; readonly #requestDrain: () => void; @@ -330,6 +350,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { this.#root = options.root; this.#durableProof = options.durableProof; this.#receipts = options.receipts; + this.#lifecycle = options.lifecycle; this.#sessionAdmission = options.sessionAdmission; this.#acquireResidency = options.acquireResidency; this.#requestDrain = options.requestDrain ?? (() => undefined); @@ -503,6 +524,90 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { this.#draining = true; } + async markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise { + await this.#lifecycle?.markMessagesHandedOff(sessionId, messageIds); + } + + async markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise { + await this.#lifecycle?.markMessagesExecuted(sessionId, messageIds); + } + + async cancelMessages(sessionId: string, messageIds: readonly string[]): Promise { + await this.#lifecycle?.cancelMessageAdmissions(sessionId, messageIds); + } + + async recoverPendingAfterHostRestart(sessionIds: readonly string[]): Promise { + if (!this.#lifecycle) return; + for (const sessionId of sessionIds) { + const admissions = await this.#lifecycle.listMessageAdmissions(sessionId); + if (admissions.length === 0) continue; + const rootState = await this.#root.readRootState(sessionId); + const pending = [] as PendingMessageAdmission[]; + for (const admission of admissions) { + const source = await this.#durableProof.readRootTurnSourceMessageReceipt( + sessionId, + admission.messageId, + ); + if (source) { + await this.#lifecycle.markMessagesHandedOff(sessionId, [admission.messageId]); + } else { + pending.push(admission); + } + } + if (pending.length === 0) continue; + if (rootState.kind !== 'active') { + if (rootState.kind !== 'idle') continue; + if (!this.#root.startRecoveredMessages) { + throw new RuntimeMessageAuthorityInvariantError( + 'Message recovery authority is unavailable', + ); + } + await this.#sessionAdmission.run(sessionId, (admission) => + this.#root.startRecoveredMessages!( + { + sessionId, + content: aggregateMessageContents(pending.map((entry) => entry.modelContent)), + submittedContent: aggregateMessageContents(pending.map((entry) => entry.content)), + sources: pending.map(pendingMessageSource), + }, + admission, + ), + ); + continue; + } + if (!this.#sessions.has(sessionId)) this.#state(sessionId); + const state = this.#requireState(sessionId); + if (!state.reservedRoot) this.reserveRootTurn(rootState); + if (!sameRun(state.reservedRoot!, rootState)) continue; + for (const admission of admissions) { + if (admission.turnId !== rootState.turnId || admission.runId !== rootState.runId) continue; + const existing = allLiveEntries(state).find( + (entry) => entry.messageId === admission.messageId, + ); + if (existing) continue; + const residency = this.#acquireResidency(); + const entry: LiveEntry = { + entryId: this.#createId(), + messageId: admission.messageId, + turnId: admission.turnId, + runId: admission.runId, + admittedAt: admission.admittedAt, + content: admission.content, + modelContent: admission.modelContent, + initiatingConnectionId: '', + placement: admission.placement, + disposition: admission.disposition, + generation: state.generation, + residency, + state: 'queued', + }; + if (entry.disposition === 'steering') state.steering.push(entry); + else state.followup.push(entry); + this.#mutated(state); + } + } + } + commitStopFence(identity: RuntimeMessageRunIdentity): QueueFenceResult { return this.#commitQueueFence(identity); } @@ -621,16 +726,45 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { placement: input.placement, disposition: 'turn_started', }; + const pendingAdmission = await this.#lifecycle?.readMessageAdmission( + input.sessionId, + input.messageId, + ); + if ( + pendingAdmission && + (!messageContentsEqual(pendingAdmission.content, payload.content) || + pendingAdmission.submittedPlacement !== input.placement) + ) { + return failure('operation_conflict', 'Message admission has a different payload'); + } + const turnId = pendingAdmission?.turnId ?? this.#createId(); + const runId = pendingAdmission?.runId ?? this.#createId(); + const messageAdmission: PendingMessageAdmission = { + sessionId: input.sessionId, + turnId, + runId, + messageId: input.messageId, + content: payload.content, + modelContent: payload.content, + submittedPlacement: input.placement, + placement: 'current_turn', + disposition: 'steering', + admittedAt: pendingAdmission?.admittedAt ?? Date.now(), + }; + await this.#lifecycle?.commitMessageAdmission(messageAdmission); const started = await this.#root.startFromMessage( { sessionId: input.sessionId, content: payload.content, sourceMessage, initiatingConnectionId, + turnId, + runId, }, admission, ); if ('error' in started) { + await this.#lifecycle?.cancelMessageAdmissions(input.sessionId, [input.messageId]); return failure('operation_conflict', started.error); } if (!isEntityId(started.turnId)) { @@ -638,6 +772,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { 'Started Turn identity is not encodable', ); } + await this.#lifecycle?.markMessagesHandedOff(input.sessionId, [input.messageId]); const result = { disposition: 'turn_started', turnId: started.turnId } as const; return success(result); } @@ -734,10 +869,26 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { continue; } const result = { disposition, queueRevision: candidateRevision + 1 } as const; + const messageAdmission: PendingMessageAdmission = { + sessionId: input.sessionId, + turnId: rootState.turnId, + runId: rootState.runId, + messageId: input.messageId, + content: payload.content, + modelContent: prepared.content, + submittedPlacement: input.placement, + placement: input.placement, + disposition, + admittedAt: Date.now(), + }; + await this.#lifecycle?.commitMessageAdmission(messageAdmission); const residency = this.#acquireResidency(); const entry: LiveEntry = { entryId, messageId: input.messageId, + turnId: rootState.turnId, + runId: rootState.runId, + admittedAt: messageAdmission.admittedAt, content: payload.content, modelContent: prepared.content, initiatingConnectionId, @@ -794,6 +945,10 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { queueRevision: state.revision + (queued.length > 0 ? 1 : 0), retracted: queued.map(retractedSnapshot), }; + await this.#lifecycle?.cancelMessageAdmissions( + input.sessionId, + queued.map((entry) => entry.messageId), + ); const retracted = this.#retractQueued(state); if (retracted.length > 0) this.#mutated(state); if (!isDeepStrictEqual(result, { queueRevision: state.revision, retracted })) { @@ -970,6 +1125,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } return failure('not_found', 'Message queue entry does not exist'); } + await this.#lifecycle?.cancelMessageAdmissions(input.sessionId, [queued.entry.messageId]); queued.remove(); this.#releaseEntry(queued.entry); this.#mutated(state); @@ -1023,6 +1179,18 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } return failure('not_found', 'Message queue entry does not exist'); } + await this.#lifecycle?.updateMessageAdmission({ + sessionId: input.sessionId, + turnId: entry.turnId, + runId: entry.runId, + messageId: entry.messageId, + content: entry.content, + modelContent: entry.modelContent, + submittedPlacement: 'next_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt: entry.admittedAt, + }); state.followup.splice(index, 1); state.steering.push({ ...entry, placement: 'current_turn', disposition: 'steering' }); this.#mutated(state); @@ -1113,6 +1281,18 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { ) { return failure('session_busy', 'Message queue changed during update'); } + await this.#lifecycle?.updateMessageAdmission({ + sessionId: input.sessionId, + turnId: queued.entry.turnId, + runId: queued.entry.runId, + messageId: queued.entry.messageId, + content, + modelContent, + submittedPlacement: queued.entry.placement, + placement: queued.entry.placement, + disposition: queued.entry.disposition, + admittedAt: queued.entry.admittedAt, + }); queued.entry.content = content; queued.entry.modelContent = modelContent; this.#mutated(state); @@ -1153,6 +1333,10 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { reordered.push(entry); } if (reordered.some((entry, index) => current[index] !== entry)) { + await this.#lifecycle?.reorderMessageAdmissions( + input.sessionId, + reordered.map((entry) => entry.messageId), + ); state.followup = reordered; this.#mutated(state); } @@ -1839,6 +2023,16 @@ function sourceFromEntry(entry: LiveEntry): RootFollowupSource { }; } +function pendingMessageSource(admission: PendingMessageAdmission): RootTurnSourceMessage { + return { + messageId: admission.messageId, + content: normalizeMessageContent(admission.modelContent), + submittedContentDigest: messageContentDigest(admission.content), + placement: admission.placement, + disposition: admission.disposition, + }; +} + function queuedSnapshot(entry: LiveEntry): QueuedMessageSnapshot { return { entryId: entry.entryId, diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index c05a8a019d..ffd3ed7f20 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -79,6 +79,7 @@ import type { HostInteractionCoordinator } from './interaction-coordinator.js'; import { type HostMessageRootState, type HostMessagePreparationInput, + type HostMessageRecoveryBatch, type HostMessageSessionHeader, type HostMessageStartInput, type HostMessageStopClaim, @@ -1003,7 +1004,8 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { const header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId); const unavailableReason = runtimeHostExternalTurnUnavailableReason(header); if (unavailableReason) return { error: unavailableReason }; - const turnId = randomUUID(); + const turnId = input.turnId ?? randomUUID(); + const runId = input.runId ?? randomUUID(); const hasSkillInvocation = parseSkillInvocationTokens(content.text).length > 0; const prepared = hasSkillInvocation ? await this.prepareHostedSkillInvocationContent( @@ -1040,7 +1042,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { const admitted = await this.rootAdmissionOwner.admitRootTurn({ sessionId: input.sessionId, turnId, - proposedRunId: randomUUID(), + proposedRunId: runId, proposedUserMessageId: input.sourceMessage.messageId, execution: { kind: 'external_message', @@ -1085,6 +1087,61 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { }); } + startRecoveredMessages( + input: HostMessageRecoveryBatch, + admissionLease: SessionAdmissionLease, + ): Promise<{ readonly turnId: string } | { readonly error: string }> { + return this.runCommand(async () => { + if (this.#executions.has(input.sessionId)) { + return { error: 'A root Turn is still active' }; + } + const header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId); + const unavailableReason = runtimeHostExternalTurnUnavailableReason(header); + if (unavailableReason) return { error: unavailableReason }; + const reservation = this.reserveRootTurn(input.sessionId); + if (!reservation) return { error: 'Another root Turn is being admitted' }; + try { + const turnId = randomUUID(); + const admitted = await this.rootAdmissionOwner.admitRootTurn({ + sessionId: input.sessionId, + turnId, + proposedRunId: randomUUID(), + proposedUserMessageId: randomUUID(), + execution: { + kind: 'external_message', + inputDigest: messageContentDigest(input.submittedContent), + }, + normalizedInput: input.content, + sourceMessages: input.sources, + admittedAt: Date.now(), + }); + if (admitted.kind !== 'admitted') { + return { error: 'Recovered Message root identity already existed' }; + } + const disposition = await this.prepareAdmittedTurn( + { sessionId: input.sessionId, turnId, content: input.content }, + admitted.admission, + this.acquireRecoveryResidency, + admissionLease, + undefined, + undefined, + reservation, + ); + if (disposition.kind !== 'await_start') { + return { error: 'Recovered Message root did not reserve execution' }; + } + await this.messages.markMessagesHandedOff( + input.sessionId, + input.sources.map((source) => source.messageId), + ); + return { turnId }; + } catch (error) { + this.#admissions.release(reservation); + throw error; + } + }); + } + prepareMessage( input: HostMessagePreparationInput, ): Promise< @@ -1810,7 +1867,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (active.startSettled.phase === 'rejected') { return { active, deliverStop: () => Promise.resolve() }; } - commitQueueFence(); + const fence = commitQueueFence(); + await this.messages.cancelMessages( + input.sessionId, + fence.retracted.map((message) => message.messageId), + ); await this.interactions.claimRunClosure(input, 'turn_stopped', admission); const shouldDeliverStop = !active.stopRequested; active.stopRequested = true; @@ -1845,7 +1906,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { const active = this.#executions.get(input.sessionId); if (isTerminalSnapshot(snapshot)) { if (active?.turnId === input.turnId && active.runId === input.runId) { - commitQueueFence(); + const fence = commitQueueFence(); + await this.messages.cancelMessages( + input.sessionId, + fence.retracted.map((message) => message.messageId), + ); active.stopRequested = true; return { kind: 'await_terminal', active }; } @@ -1870,7 +1935,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { }; } - commitQueueFence(); + const fence = commitQueueFence(); + await this.messages.cancelMessages( + input.sessionId, + fence.retracted.map((message) => message.messageId), + ); await this.interactions.claimRunClosure(input, 'turn_stopped', admissionLease); const shouldRequestStop = !active.stopRequested; active.stopRequested = true; @@ -2137,6 +2206,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } this.observeExecutionCompletion(active, { kind: 'terminal', snapshot }); await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status); + await this.settleExecutedMessageSources(active); terminalTransitionStarted = true; await this.completeTerminalTransition(input.sessionId, active); } catch (error) { @@ -2160,6 +2230,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { snapshot, }); await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status); + await this.settleExecutedMessageSources(active); terminalTransitionStarted = true; await this.completeTerminalTransition(input.sessionId, active); containedRunFailure = @@ -2215,6 +2286,37 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } } + private async settleExecutedMessageSources(active: ActiveRootTurn): Promise { + const admission = await this.stores.agentRunStore.readRootTurnAdmission( + active.sessionId, + active.turnId, + ); + if (!admission || admission.sourceMessages.length === 0) return; + const events = await this.stores.agentRunStore.readEvents(active.sessionId, active.runId); + if ( + !events.some( + (event) => + event.type === 'provider_request_captured' || + event.type === 'provider_request_attempt_recorded' || + event.type === 'model_call_attempt_recorded', + ) + ) { + return; + } + const executed = [] as string[]; + for (const source of admission.sourceMessages) { + if ( + (await this.stores.sessionStore.readMessageLifecycleState( + active.sessionId, + source.messageId, + )) === 'handed_off' + ) { + executed.push(source.messageId); + } + } + await this.messages.markMessagesExecuted(active.sessionId, executed); + } + private observeExecutionCompletion( active: ActiveRootTurn, completion: HostedExecutionCompletion, @@ -2276,15 +2378,15 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { admissionLease: SessionAdmissionLease, ): Promise { const initiatingConnectionId = batch.initiatingConnectionId; - if (!initiatingConnectionId) { - throw new RuntimeMessageAuthorityInvariantError( - 'Follow-up batch lost its initiating Client identity', - ); - } // A confirmed follow-up must become a durable root even when a Session // provider is unavailable. Lost tools are omitted while ephemeral // capabilities bind to the Client that submitted this follow-up. - await this.clientCapabilities?.bindConfirmedFollowup(batch.sessionId, initiatingConnectionId); + if (initiatingConnectionId) { + await this.clientCapabilities?.bindConfirmedFollowup( + batch.sessionId, + initiatingConnectionId, + ); + } const turnId = randomUUID(); const header = await this.stores.sessionStore.readHeaderSnapshot(batch.sessionId); @@ -2307,6 +2409,10 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { 'Fresh follow-up root Turn identity already existed', ); } + await this.messages.markMessagesHandedOff( + batch.sessionId, + batch.sources.map((source) => source.messageId), + ); const nextIdentity = { sessionId: batch.sessionId, diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index fcbf0102ad..a5a2ef633b 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -58,7 +58,7 @@ import { resolveEffectiveOrchestration, type EffectiveOrchestration, } from '@maka/core/orchestration'; -import type { SessionEvent } from '@maka/core/events'; +import { messageContentsEqual, type SessionEvent } from '@maka/core/events'; import type { AgentBackend, BackendSendInput } from '@maka/core/backend-types'; import type { RunTraceEvent } from './run-trace.js'; import type { StopSessionInput } from './session-manager.js'; @@ -667,7 +667,7 @@ export class AgentRun { : {}), ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), }); - await this.input.store.appendMessage(this.sessionId, userMsg); + await appendUserMessageOnce(this.input.store, this.sessionId, userMsg); await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage); this.lastTs = userMessageTs; } else { @@ -1896,6 +1896,28 @@ function redactTraceString(value: string): string { function errorMessage(error: unknown): string { return redactTraceString(error instanceof Error ? error.message : String(error)); } + +async function appendUserMessageOnce( + store: AgentRunSessionStore, + sessionId: string, + message: UserMessage, +): Promise { + const existing = (await store.readMessages(sessionId)).find( + (candidate) => candidate.id === message.id, + ); + if (!existing) { + await store.appendMessage(sessionId, message); + return; + } + if ( + existing.type !== 'user' || + existing.turnId !== message.turnId || + !messageContentsEqual(existing, message) + ) { + throw new Error(`Durable UserMessage identity ${message.id} has conflicting content`); + } +} + function isInteractionResumeAck(event: SessionEvent): boolean { return ( event.type === 'sandbox_boundary_decision_ack' || event.type === 'user_question_answer_ack' diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 6f796e9a54..5f52a4ad4a 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -283,6 +283,20 @@ describe('SqliteSessionMetadataStore', () => { }, ], ); + assert.equal( + await store.readMessageLifecycleState('session-1', 'message-1'), + 'accepted', + ); + await store.markMessagesHandedOff('session-1', ['message-1']); + assert.equal( + await store.readMessageLifecycleState('session-1', 'message-1'), + 'handed_off', + ); + await store.markMessagesExecuted('session-1', ['message-1']); + assert.equal( + await store.readMessageLifecycleState('session-1', 'message-1'), + 'executed', + ); } finally { store.close(); } diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index ea8e49bcda..e0b134120f 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -115,9 +115,12 @@ export type { RuntimeEventScanResult, } from './agent-run-store.js'; export type { + MessageLifecycleState, + MessageLifecycleStore, MessageOperationReceipt, MessageReceiptOperation, MessageReceiptStore, + PendingMessageAdmission, } from './message-receipt-store.js'; export type { ProbeSessionRemovalResult, @@ -173,6 +176,10 @@ export interface ExecutionSessionReader { list(filter?: SessionListFilter): Promise; readHeader(sessionId: string): Promise; readMessages(sessionId: string): Promise; + readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise; listTurns(sessionId: string): Promise; close?(): Promise; } @@ -421,6 +428,21 @@ async function createExecutionStoresForWrite sessionStore.appendMessage(sessionId, message)), appendMessages: (sessionId, messages) => run(() => sessionStore.appendMessages(sessionId, messages)), + commitMessageAdmission: (admission) => run(() => sessionStore.commitMessageAdmission(admission)), + readMessageAdmission: (sessionId, messageId) => + run(() => sessionStore.readMessageAdmission(sessionId, messageId)), + readMessageLifecycleState: (sessionId, messageId) => + run(() => sessionStore.readMessageLifecycleState(sessionId, messageId)), + listMessageAdmissions: (sessionId) => run(() => sessionStore.listMessageAdmissions(sessionId)), + updateMessageAdmission: (admission) => run(() => sessionStore.updateMessageAdmission(admission)), + reorderMessageAdmissions: (sessionId, messageIds) => + run(() => sessionStore.reorderMessageAdmissions(sessionId, messageIds)), + cancelMessageAdmissions: (sessionId, messageIds) => + run(() => sessionStore.cancelMessageAdmissions(sessionId, messageIds)), + markMessagesHandedOff: (sessionId, messageIds) => + run(() => sessionStore.markMessagesHandedOff(sessionId, messageIds)), + markMessagesExecuted: (sessionId, messageIds) => + run(() => sessionStore.markMessagesExecuted(sessionId, messageIds)), subscribeTranscriptChanges: (listener) => sessionStore.subscribeTranscriptChanges(listener), updateHeader: (sessionId, patch) => run(() => sessionStore.updateHeader(sessionId, patch)), updateHeaderVersioned: (sessionId, patch, expectedRevision) => @@ -608,6 +630,8 @@ async function openExecutionStoresForRead run(() => sessionStore.list(filter)), readHeader: (sessionId) => run(() => sessionStore.readHeaderSnapshot(sessionId)), readMessages: (sessionId) => run(() => sessionStore.readMessagesSnapshot(sessionId)), + readMessageLifecycleState: (sessionId, messageId) => + run(() => sessionStore.readMessageLifecycleState(sessionId, messageId)), listTurns: (sessionId) => run(() => sessionStore.listTurnsSnapshot(sessionId)), close: () => closeExecutionStorePersistence(sessionStore, runtimePersistence, { diff --git a/packages/storage/src/message-receipt-store.ts b/packages/storage/src/message-receipt-store.ts index b9dfeb320e..d0b5c40a11 100644 --- a/packages/storage/src/message-receipt-store.ts +++ b/packages/storage/src/message-receipt-store.ts @@ -45,6 +45,24 @@ export interface PendingMessageAdmission { readonly admittedAt: number; } +export interface MessageLifecycleStore { + commitMessageAdmission(admission: PendingMessageAdmission): Promise; + readMessageAdmission( + sessionId: string, + messageId: string, + ): Promise; + readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise; + listMessageAdmissions(sessionId: string): Promise; + updateMessageAdmission(admission: PendingMessageAdmission): Promise; + reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; + cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; + markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise; + markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise; +} + export function normalizePendingMessageAdmission( admission: PendingMessageAdmission, ): PendingMessageAdmission { diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index a731808e92..5b310c0768 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -80,6 +80,10 @@ import { type TurnStateMessage, type UserMessage, } from '@maka/core/session'; +import type { + MessageLifecycleStore, + PendingMessageAdmission, +} from './message-receipt-store.js'; const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -299,7 +303,7 @@ export interface SessionStore { close?(): Promise; } -export interface SessionAuthorityStore extends SessionStore { +export interface SessionAuthorityStore extends SessionStore, MessageLifecycleStore { /** Read a bounded set of durable messages at an inclusive transcript watermark. */ readTranscriptMessagesSnapshot( sessionId: string, @@ -857,6 +861,60 @@ class SqliteSessionStore implements SessionAuthorityStore { for (const listener of this.transcriptChangeListeners) listener(sessionId); } + async commitMessageAdmission(admission: PendingMessageAdmission): Promise { + await this.ensureReady(); + const committed = await this.metadata.commitMessageAdmission(admission); + for (const listener of this.transcriptChangeListeners) listener(admission.sessionId); + return committed; + } + + async readMessageAdmission( + sessionId: string, + messageId: string, + ): Promise { + await this.ensureReady(); + return this.metadata.readMessageAdmission(sessionId, messageId); + } + + async listMessageAdmissions(sessionId: string): Promise { + await this.ensureReady(); + return this.metadata.listMessageAdmissions(sessionId); + } + + async readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise { + await this.ensureReady(); + return this.metadata.readMessageLifecycleState(sessionId, messageId); + } + + async updateMessageAdmission(admission: PendingMessageAdmission): Promise { + await this.ensureReady(); + await this.metadata.updateMessageAdmission(admission); + for (const listener of this.transcriptChangeListeners) listener(admission.sessionId); + } + + async reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.reorderMessageAdmissions(sessionId, messageIds); + } + + async cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.cancelMessageAdmissions(sessionId, messageIds); + } + + async markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.markMessagesHandedOff(sessionId, messageIds); + } + + async markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.markMessagesExecuted(sessionId, messageIds); + } + subscribeTranscriptChanges(listener: (sessionId: string) => void): () => void { this.transcriptChangeListeners.add(listener); return () => this.transcriptChangeListeners.delete(listener); diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 03395bb0ae..99437bdf2a 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1670,6 +1670,259 @@ export class SqliteSessionMetadataStore { }); } + async listMessageAdmissions(sessionId: string): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + return this.readTransaction(() => { + const rows = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, model_content_json, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND lifecycle_state = 'accepted' + ORDER BY queue_order, sequence + `, + ) + .all(sessionId) as MessageAdmissionRow[]; + return rows.map((row) => decodeMessageAdmissionRow(sessionId, row).admission); + }); + } + + async readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + assertSafeSessionId(messageId); + return this.readTransaction(() => { + const row = this.db + .prepare( + ` + SELECT lifecycle_state + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(sessionId, messageId) as { lifecycle_state?: unknown } | undefined; + if (!row) return undefined; + if ( + row.lifecycle_state !== 'accepted' && + row.lifecycle_state !== 'handed_off' && + row.lifecycle_state !== 'executed' && + row.lifecycle_state !== 'cancelled' + ) { + throw new SessionMetadataConflictError('Invalid Message admission lifecycle state'); + } + return row.lifecycle_state; + }); + } + + async updateMessageAdmission(admission: PendingMessageAdmission): Promise { + this.assertOpen(); + const stored = normalizePendingMessageAdmission(admission); + this.transaction(() => { + const currentRow = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, model_content_json, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(stored.sessionId, stored.messageId) as MessageAdmissionRow | undefined; + if (!currentRow) throw new SessionMetadataConflictError('Message admission does not exist'); + const current = decodeMessageAdmissionRow(stored.sessionId, currentRow); + if (current.lifecycleState !== 'accepted') { + throw new SessionMetadataConflictError('Message admission is already settled'); + } + if ( + current.admission.turnId !== stored.turnId || + current.admission.runId !== stored.runId || + current.admission.submittedPlacement !== stored.submittedPlacement || + current.admission.admittedAt !== stored.admittedAt + ) { + throw new SessionMetadataConflictError('Message admission update identity conflict'); + } + this.db + .prepare( + ` + UPDATE message_admissions + SET content_json = ?, model_content_json = ?, placement = ?, disposition = ? + WHERE session_id = ? AND message_id = ? AND lifecycle_state = 'accepted' + `, + ) + .run( + JSON.stringify(stored.content), + JSON.stringify(stored.modelContent), + stored.placement, + stored.disposition, + stored.sessionId, + stored.messageId, + ); + if (stored.disposition !== 'steering') return; + const message = decodeCanonicalMessage({ + type: 'user', + id: stored.messageId, + turnId: stored.turnId, + ts: stored.admittedAt, + ...stored.content, + steeringEventId: stored.messageId, + }); + const rows = this.db + .prepare( + ` + SELECT sequence, record_json + FROM session_messages + WHERE session_id = ? AND message_id = ? + `, + ) + .all(stored.sessionId, stored.messageId) as Array<{ + sequence?: unknown; + record_json?: unknown; + }>; + if (rows.length > 1) { + throw new SessionMetadataConflictError('Message admission transcript identity is ambiguous'); + } + const json = JSON.stringify(message); + if (rows.length === 0) { + const row = this.db + .prepare( + 'SELECT COALESCE(MAX(sequence), -1) AS last_sequence FROM session_messages WHERE session_id = ?', + ) + .get(stored.sessionId) as { last_sequence?: unknown }; + if (typeof row.last_sequence !== 'number' || !Number.isSafeInteger(row.last_sequence)) { + throw new SessionMetadataConflictError('Invalid Session message sequence'); + } + this.insertSessionMessagesSync(stored.sessionId, row.last_sequence + 1, [ + { message, json }, + ]); + } else { + const sequence = rows[0]?.sequence; + if (typeof sequence !== 'number' || !Number.isSafeInteger(sequence)) { + throw new SessionMetadataConflictError('Invalid Message transcript sequence'); + } + this.db + .prepare( + 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', + ) + .run(json, stored.sessionId, sequence); + } + this.updateCatalogProjectionSync( + stored.sessionId, + { + lastMessageAt: stored.admittedAt, + lastMessagePreview: message.type === 'user' ? message.displayText : undefined, + }, + true, + ); + }); + } + + async cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + const unique = [...new Set(messageIds)]; + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + const statement = this.db.prepare( + ` + UPDATE message_admissions + SET lifecycle_state = 'cancelled' + WHERE session_id = ? AND message_id = ? AND lifecycle_state = 'accepted' + `, + ); + for (const messageId of unique) { + const result = statement.run(sessionId, messageId); + if (result.changes !== 1) { + throw new SessionMetadataConflictError('Message admission cancellation identity conflict'); + } + } + }); + } + + async reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + const unique = [...new Set(messageIds)]; + if (unique.length !== messageIds.length) { + throw new SessionMetadataConflictError('Message admission reorder contains duplicate identities'); + } + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + const rows = this.db + .prepare( + ` + SELECT message_id + FROM message_admissions + WHERE session_id = ? AND lifecycle_state = 'accepted' AND disposition = 'followup' + ORDER BY queue_order, sequence + `, + ) + .all(sessionId) as Array<{ message_id?: unknown }>; + const current = rows.map((row) => row.message_id); + if ( + current.length !== unique.length || + current.some((messageId, index) => messageId !== unique[index]) + ) { + throw new SessionMetadataConflictError('Message admission reorder identity conflict'); + } + const update = this.db.prepare( + ` + UPDATE message_admissions + SET queue_order = ? + WHERE session_id = ? AND message_id = ? AND lifecycle_state = 'accepted' + `, + ); + unique.forEach((messageId, index) => update.run(index, sessionId, messageId)); + }); + } + + async markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise { + return this.markMessageLifecycle(sessionId, messageIds, 'handed_off'); + } + + async markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise { + return this.markMessageLifecycle(sessionId, messageIds, 'executed'); + } + + private markMessageLifecycle( + sessionId: string, + messageIds: readonly string[], + state: 'handed_off' | 'executed', + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + const unique = [...new Set(messageIds)]; + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + const statement = this.db.prepare( + ` + UPDATE message_admissions + SET lifecycle_state = ? + WHERE session_id = ? AND message_id = ? + AND lifecycle_state IN ('accepted', 'handed_off') + `, + ); + for (const messageId of unique) { + const result = statement.run(state, sessionId, messageId); + if (result.changes !== 1) { + const existing = this.db + .prepare( + 'SELECT lifecycle_state FROM message_admissions WHERE session_id = ? AND message_id = ?', + ) + .get(sessionId, messageId) as { lifecycle_state?: unknown } | undefined; + if (existing?.lifecycle_state !== state) { + throw new SessionMetadataConflictError('Message admission lifecycle identity conflict'); + } + } + } + }); + return Promise.resolve(); + } + async readMessages(sessionId: string): Promise { return this.readMessagesWith(sessionId, decodeStoredMessage); } From d37b31bde9b6c0c75326c27dcb43ddbb41d7d01e Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 21:48:59 +0800 Subject: [PATCH 03/22] refactor(runtime): remove embedded message queue authority Generated-by: Codex --- .../src/server/root-turn-coordinator.ts | 16 + packages/runtime/src/agent-run.ts | 5 +- packages/runtime/src/runtime-kernel.ts | 346 ++++-------------- packages/runtime/src/session-manager.ts | 15 + 4 files changed, 102 insertions(+), 280 deletions(-) diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index ffd3ed7f20..e0a2e53678 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -157,6 +157,7 @@ interface ActiveRootTurn { residency: RuntimeHostResidency; stopRequested: boolean; messageTransitionCommitted: boolean; + initialUserMessagesMaterialized: boolean; } export type TurnStartOutcome = OperationOutcome<'turn.start'>; @@ -1978,6 +1979,19 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (unavailableReason) { return completedStart(operationUnavailable(unavailableReason)); } + const initialUserMessagesMaterialized = admission.sourceMessages.length > 0; + if (initialUserMessagesMaterialized) { + await this.manager.materializeRootSourceMessages({ + sessionId: input.sessionId, + turnId: input.turnId, + previousRootTurnId: admission.previousRootTurnId, + messages: admission.sourceMessages.map((source) => ({ + messageId: source.messageId, + content: source.content, + disposition: source.disposition, + })), + }); + } const { runId } = admission; const existingRun = await this.readRunIfPresent(input.sessionId, runId); if (replacing && existingRun) { @@ -2070,6 +2084,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { residency, stopRequested: false, messageTransitionCommitted: false, + initialUserMessagesMaterialized, }; if (replacing && this.#executions.get(input.sessionId) !== replacing) { residency.release(); @@ -2167,6 +2182,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { { runId: active.runId, userMessageId: active.userMessageId ?? undefined, + recordInitialUserMessage: !active.initialUserMessagesMaterialized, durability: 'required', onRunStarted: async (startedRunId) => { if (startedRunId !== active.runId) { diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index a5a2ef633b..0c63293c45 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -161,6 +161,7 @@ export interface AgentRunInput { commitContinuationStart?: (startedAt: number) => Promise<{ startEventId: string; created: true }>; hooks: AgentRunHooks; recordSessionMessages?: boolean; + recordInitialUserMessage?: boolean; invocationId?: string; /** Pre-resolved snapshot used by continuations; normal turns derive it from header + input. */ effectiveOrchestration?: EffectiveOrchestration; @@ -667,7 +668,9 @@ export class AgentRun { : {}), ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), }); - await appendUserMessageOnce(this.input.store, this.sessionId, userMsg); + if (this.input.recordInitialUserMessage !== false) { + await appendUserMessageOnce(this.input.store, this.sessionId, userMsg); + } await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage); this.lastTs = userMessageTs; } else { diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index a9c0c9fefc..570a780566 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -33,13 +33,15 @@ import type { RuntimeEventStore, } from '@maka/core/runtime-event-store'; import { isSessionInlineRun } from '@maka/core/agent-run'; -import type { - ActiveInteractionRequestEvent, - CompleteEvent, - QueueEnqueueOutcome, - QueueUpdateEvent, - SessionEvent, - TokenUsageEvent, +import { + messageContentsEqual, + normalizeMessageContent, + type ActiveInteractionRequestEvent, + type CompleteEvent, + type MessageContent, + type QueueEnqueueOutcome, + type SessionEvent, + type TokenUsageEvent, } from '@maka/core/events'; import type { SessionBlockedReason, @@ -185,13 +187,20 @@ export interface RuntimeKernelLike { respondToSandboxBoundary(sessionId: string, response: SandboxBoundaryResponse): Promise; listActiveInteractions?(sessionId: string): ActiveInteractionRequestEvent[]; respondToUserQuestion?(sessionId: string, response: UserQuestionResponse): Promise; - /** Queue a user message for mid-turn injection at the next step boundary. */ + materializeRootSourceMessages?(input: { + sessionId: string; + turnId: string; + previousRootTurnId: string | null; + messages: readonly { + messageId: string; + content: MessageContent; + disposition: 'steering' | 'followup' | 'turn_started'; + }[]; + }): Promise; + /** Compatibility surface; durable message admission belongs to Runtime Host. */ steer(sessionId: string, text: string): QueueEnqueueOutcome; - /** Queue a user message to open the turn after the current one finishes. */ queueMessage(sessionId: string, text: string): QueueEnqueueOutcome; - /** Drain the followup queue into one `\n\n`-joined prompt, or null if empty. */ drainFollowup(sessionId: string): string | null; - /** Take back every queued message (both queues) as one `\n\n`-joined string. */ retractQueue(sessionId: string): string; hasActiveRuns(sessionId: string): boolean; /** @@ -233,6 +242,7 @@ export class RuntimeContextCompactError extends Error { export interface TurnStartOptions { runId?: string; userMessageId?: string; + recordInitialUserMessage?: boolean; durability?: AgentRunDurability; /** * Resolve turn admission after this Session has registered a pending start @@ -271,44 +281,6 @@ export interface ChildAgentRetryInput { onRunStarted?: () => void | Promise; } -/** - * An embedded session's authoritative pending-message queues plus its event - * sink. Hosted composition never creates this state; its Host owns admission, - * snapshots, leases, and follow-up drain. - */ -interface PendingSteeringMessage extends SteeringLease {} - -/** - * A pulled lease is bound to the turn that pulled it: only the issuing turn's - * backend can settle it (ack/nack stay valid even after ownership moved to an - * overlapping turn — invalidating a delivered lease would leave it in-flight - * and redeliver an already-executed message), and no other turn's retract/ - * clear/release may reclaim it while its delivery is still undetermined. - */ -interface LeasedSteeringMessage extends PendingSteeringMessage { - issuingTurnId: string; -} - -interface SessionSteeringState { - /** Messages waiting to be injected into the running turn at a step boundary. */ - steering: PendingSteeringMessage[]; - /** - * Leased to the running turn's backend but not yet settled. pull() is the - * single atomic commit point: an in-flight lease is committed to that - * turn's delivery — retract/clear reclaim only QUEUED messages — and it - * settles exactly once, decided solely by the persistence fact: ack when - * the steering event is durably consumed (even under abort), nack when it - * provably never persisted. Snapshots count in-flight as still pending so - * the UI keeps showing the message until it lands in the transcript. - */ - inFlight: LeasedSteeringMessage[]; - /** Messages waiting to open the next turn. */ - followup: PendingSteeringMessage[]; - /** Pushes a `queue_update` into the active turn's stream; unset when idle. */ - sink?: (event: QueueUpdateEvent) => void; - activeTurnId?: string; -} - export type BackendActivationBoundary = (operation: () => Promise | T) => Promise; interface ChildToolActivation { @@ -458,7 +430,6 @@ export class RuntimeKernel implements RuntimeKernelLike { private readonly historyCompactCoordinator: HistoryCompactCheckpointCoordinator; private readonly pendingContinuationClaims = new Set(); private readonly pendingContinuationSessions = new Set(); - private readonly steeringBySession = new Map(); private readonly backendInvalidations = new Map(); private readonly interactionRequestOwners = new Map(); private nextBackendGeneration = 0; @@ -722,6 +693,7 @@ export class RuntimeKernel implements RuntimeKernelLike { userInput: input, runId: options.runId, userMessageId: options.userMessageId, + recordInitialUserMessage: options.recordInitialUserMessage, durability: options.durability, store: this.deps.store, runStore: this.deps.runStore, @@ -1560,11 +1532,6 @@ export class RuntimeKernel implements RuntimeKernelLike { const interactionRun = owners.interactionRun; const messageOwner = owners.messageOwner; - // Steering is a top-level-turn affordance only; child agent turns run - // without a queue. Hosted ownership is bound before begin so a pre-start - // cancellation can release the exact admitted owner. The pull hook still - // re-checks this run's turnId so stale or overlapping runs cannot drain - // messages queued for the current owner. let pullSteering: (() => readonly SteeringLease[]) | undefined; let ackSteering: ((leaseIds: readonly string[]) => void) | undefined; let nackSteering: ((leaseIds: readonly string[]) => void) | undefined; @@ -1572,71 +1539,6 @@ export class RuntimeKernel implements RuntimeKernelLike { pullSteering = () => messageOwner?.pull() ?? []; ackSteering = (leaseIds) => messageOwner?.ack(leaseIds); nackSteering = (leaseIds) => messageOwner?.nack(leaseIds); - } else if (steering) { - const state = this.ensureSteering(sessionId); - state.sink = (event) => { - void sessionEvents.push(event).catch(() => {}); - }; - state.activeTurnId = run.turnId; - // Lease, don't consume: pulled messages move to in-flight and only an - // ack (durable + injected) removes them; a nack or a retract/clear/ - // release reclaims them, so an abort window can never drop text. - pullSteering = () => { - const current = this.steeringBySession.get(sessionId); - if (!current || current.activeTurnId !== run.turnId) return []; - if (current.steering.length === 0) return []; - const leased = current.steering.splice(0); - current.inFlight.push( - ...leased.map((message) => ({ ...message, issuingTurnId: run.turnId })), - ); - return leased.map((message) => ({ ...message })); - }; - // Settlement is keyed by lease id + issuing turn, NOT by current - // ownership: an overlapping turn that takes the owner slot must not - // invalidate the issuer's ack (the message was delivered to ITS - // provider) or intercept its nack. A late settle for a reclaimed lease - // finds no match and is a no-op. - ackSteering = (leaseIds) => { - const current = this.steeringBySession.get(sessionId); - if (!current) return; - const ids = new Set(leaseIds); - const before = current.inFlight.length; - current.inFlight = current.inFlight.filter( - (message) => !(ids.has(message.id) && message.issuingTurnId === run.turnId), - ); - if (current.inFlight.length !== before) this.emitQueueUpdate(sessionId, current); - }; - nackSteering = (leaseIds) => { - const current = this.steeringBySession.get(sessionId); - if (!current) return; - const ids = new Set(leaseIds); - const returned = current.inFlight.filter( - (message) => ids.has(message.id) && message.issuingTurnId === run.turnId, - ); - if (returned.length === 0) return; - current.inFlight = current.inFlight.filter( - (message) => !(ids.has(message.id) && message.issuingTurnId === run.turnId), - ); - if (current.activeTurnId === run.turnId) { - // Back to the FRONT of the queue: a re-pull at the next step - // boundary preserves the user's original ordering. - current.steering = [ - ...returned.map(({ id, messageId, content }) => ({ id, messageId, content })), - ...current.steering, - ]; - } else { - // The issuer no longer owns the queue (an overlapping turn took - // over and possibly released): it will never pull again, so the - // steering queue would strand the text ownerless. The followup - // queue is its only safe home — the same direction a release-time - // fold takes. - current.followup = [ - ...returned.map(({ id, messageId, content }) => ({ id, messageId, content })), - ...current.followup, - ]; - } - this.emitQueueUpdate(sessionId, current); - }; } const stopBackend = this.stopBackendFor(begin.backend); @@ -1688,7 +1590,6 @@ export class RuntimeKernel implements RuntimeKernelLike { // under its Session admission gate. The outer finally remains an // idempotent backstop for paths that never reach this hook. if (messageOwner) owners.releaseMessage(); - else if (steering) this.releaseSteeringTurn(sessionId, run.turnId); sessionEvents.close(); } catch (error) { sessionEvents.fail(error); @@ -1734,7 +1635,6 @@ export class RuntimeKernel implements RuntimeKernelLike { finalizeRun: () => owners.finalize(), releaseOwner: () => { if (messageOwner) owners.releaseMessage(); - else if (steering) this.releaseSteeringTurn(sessionId, run.turnId); }, }); } finally { @@ -2189,10 +2089,6 @@ export class RuntimeKernel implements RuntimeKernelLike { } private async stopSessionAttempt(sessionId: string, intent: SessionStopIntent): Promise { - // Interrupt clears both queues before the abort lands; the emitted empty - // snapshot lets the UI collapse its pending bar, and callers refill their - // editor from the mirror captured before the clear. - this.clearSteering(sessionId); const failures: unknown[] = []; let operation = this.stopOperations.get(sessionId); try { @@ -2492,170 +2388,63 @@ export class RuntimeKernel implements RuntimeKernelLike { // -------------------------------------------------------------------------- steer(sessionId: string, text: string): QueueEnqueueOutcome { - this.assertEmbeddedMessageQueue('steer'); - // Steering's delivery contract is anchored to the runtime event ledger - // (fail-closed persist + durable-consume ack). Without a RuntimeEventStore - // that anchor does not exist — same condition as requireTerminalWrite — - // so fall back to a fresh turn, whose user message the SessionStore - // persists with the ordinary turn-open guarantee. - if (!this.deps.runtimeEventStore) return { kind: 'fallback' }; - // Double responsibility (codex): with no live steering owner to inject - // into — the turn just ended, begin() failed, or only child/compact runs - // are active (they never consume this queue) — tell the caller to open a - // fresh turn instead so the message is never dropped. - const state = this.liveSteeringState(sessionId); - if (!state) return { kind: 'fallback' }; - const messageId = this.deps.newId(); - state.steering.push({ id: messageId, messageId, content: { text } }); - this.emitQueueUpdate(sessionId, state); - return { kind: 'queued' }; + void sessionId; + void text; + return { kind: 'fallback' }; } queueMessage(sessionId: string, text: string): QueueEnqueueOutcome { - this.assertEmbeddedMessageQueue('queueMessage'); - const state = this.liveSteeringState(sessionId); - if (!state) return { kind: 'fallback' }; - const messageId = this.deps.newId(); - state.followup.push({ id: messageId, messageId, content: { text } }); - this.emitQueueUpdate(sessionId, state); - return { kind: 'queued' }; + void sessionId; + void text; + return { kind: 'fallback' }; } drainFollowup(sessionId: string): string | null { - this.assertEmbeddedMessageQueue('drainFollowup'); - const state = this.steeringBySession.get(sessionId); - if (!state || state.followup.length === 0) return null; - const drained = state.followup.splice(0); - this.emitQueueUpdate(sessionId, state); - return drained.map((message) => message.content.text).join('\n\n'); + void sessionId; + return null; } retractQueue(sessionId: string): string { - this.assertEmbeddedMessageQueue('retractQueue'); - const state = this.steeringBySession.get(sessionId); - if (!state) return ''; - // Retract reclaims QUEUED messages only. pull() is the single atomic - // commit point of delivery: an in-flight lease is already committed to - // the running turn — its durable append may land at any moment, so - // handing its text back to the user here would refill AND execute the - // same directive. An in-flight lease settles only by the persistence - // fact (ack when the ledger owns it, nack back to a queue otherwise). - const all = [ - ...state.steering.map((message) => message.content.text), - ...state.followup.map((message) => message.content.text), - ]; - state.steering = []; - state.followup = []; - this.emitQueueUpdate(sessionId, state); - return all.join('\n\n'); + void sessionId; + return ''; } - private ensureSteering(sessionId: string): SessionSteeringState { - const existing = this.steeringBySession.get(sessionId); - if (existing) return existing; - const created: SessionSteeringState = { steering: [], inFlight: [], followup: [] }; - this.steeringBySession.set(sessionId, created); - return created; - } - - private assertEmbeddedMessageQueue(operation: string): void { - if (this.deps.messageAuthority) { - throw new RuntimeMessageAuthorityInvariantError( - `Hosted Runtime cannot ${operation}; the Runtime Host owns message admission and queues`, - ); - } - } - - /** - * The session's steering state only while a steering-capable top-level run - * owns it (sink registered after begin() succeeded and not yet released). - * Child agent and compact runs never establish ownership, so their activity - * alone yields undefined — enqueue must fall back rather than strand text. - */ - private liveSteeringState(sessionId: string): SessionSteeringState | undefined { - const state = this.steeringBySession.get(sessionId); - return state?.sink ? state : undefined; - } - - private emitQueueUpdate(sessionId: string, state: SessionSteeringState): void { - state.sink?.({ - type: 'queue_update', - id: this.deps.newId(), - turnId: state.activeTurnId ?? '', - ts: this.deps.now(), - steering: [ - ...state.inFlight.map((message) => message.content.text), - ...state.steering.map((message) => message.content.text), - ], - followup: state.followup.map((message) => message.content.text), - steeringEntries: [ - ...state.inFlight.map((message) => ({ - entryId: message.id, - messageId: message.messageId, - content: message.content, - placement: 'current_turn' as const, - state: 'in_flight' as const, - })), - ...state.steering.map((message) => ({ - entryId: message.id, - messageId: message.messageId, - content: message.content, - placement: 'current_turn' as const, - state: 'queued' as const, - })), - ], - followupEntries: state.followup.map((message) => ({ - entryId: message.id, - messageId: message.messageId, - content: message.content, - placement: 'next_turn' as const, - state: 'queued' as const, - })), - }); - } - - private clearSteering(sessionId: string): void { - const state = this.steeringBySession.get(sessionId); - if (!state) return; - // Same commit-point rule as retractQueue: only QUEUED messages are - // clearable. An in-flight lease is already committed to the running - // turn's delivery and settles only by the persistence fact. - if (state.steering.length === 0 && state.followup.length === 0) return; - state.steering = []; - state.followup = []; - this.emitQueueUpdate(sessionId, state); - } - - private releaseSteeringTurn(sessionId: string, turnId: string): void { - const state = this.steeringBySession.get(sessionId); - if (!state) return; - // A release folds only the leases THIS turn issued; an overlapping turn's - // in-flight lease stays for its issuer to settle (acked = delivered, so - // folding it into followup would redeliver an already-executed message). - const own = state.inFlight.filter((message) => message.issuingTurnId === turnId); - if (state.activeTurnId !== turnId) { - // Not (or no longer) the owner. The issuer's backend settles every - // lease before its turn ends, so `own` is normally empty; this is a - // backstop that keeps a never-settled lease from stranding invisibly. - if (own.length === 0) return; - state.inFlight = state.inFlight.filter((message) => message.issuingTurnId !== turnId); - state.followup = [...own, ...state.followup]; - this.emitQueueUpdate(sessionId, state); - return; - } - // Stranded steering (arrived after the final step boundary, so no step is - // left to consume it) becomes the head of the followup queue instead of - // vanishing — the next turn opens with it first (grok-build safety). The - // migration is a queue change, so emit the final snapshot BEFORE the sink - // is cleared; otherwise observers stay on the stale pre-fold snapshot. - if (state.steering.length > 0 || own.length > 0) { - state.followup = [...own, ...state.steering, ...state.followup]; - state.inFlight = state.inFlight.filter((message) => message.issuingTurnId !== turnId); - state.steering = []; - this.emitQueueUpdate(sessionId, state); + async materializeRootSourceMessages(input: { + sessionId: string; + turnId: string; + previousRootTurnId: string | null; + messages: readonly { + messageId: string; + content: MessageContent; + disposition: 'steering' | 'followup' | 'turn_started'; + }[]; + }): Promise { + const existingById = new Map( + (await this.deps.store.readMessages(input.sessionId)).map((message) => [message.id, message]), + ); + for (const message of input.messages) { + const existing = existingById.get(message.messageId); + if (existing) { + if ( + existing.type !== 'user' || + !messageContentsEqual(normalizeMessageContent(existing), message.content) || + (existing.turnId !== input.turnId && + (message.disposition !== 'steering' || existing.turnId !== input.previousRootTurnId)) + ) { + throw new Error(`Queued root source ${message.messageId} conflicts with its transcript`); + } + continue; + } + const materialized = { + type: 'user' as const, + id: message.messageId, + turnId: input.turnId, + ts: this.deps.now(), + ...structuredClone(message.content), + }; + await this.deps.store.appendMessage(input.sessionId, materialized); + existingById.set(message.messageId, materialized); } - state.sink = undefined; - state.activeTurnId = undefined; } hasActiveRuns(sessionId: string): boolean { @@ -2720,7 +2509,6 @@ export class RuntimeKernel implements RuntimeKernelLike { private async disposeBackendNow(sessionId: string): Promise { const generations = this.backendGenerationsFor(sessionId); - this.steeringBySession.delete(sessionId); this.historyCompactCoordinator.clear(sessionId); let disposalError: unknown; for (const active of generations) { diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 7693840d64..e351c721d0 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -4809,6 +4809,21 @@ export class SessionManager { : this.runtimeKernel.stopSession(identity.sessionId, input); } + materializeRootSourceMessages(input: { + sessionId: string; + turnId: string; + previousRootTurnId: string | null; + messages: readonly { + messageId: string; + content: import('@maka/core/events').MessageContent; + disposition: 'steering' | 'followup' | 'turn_started'; + }[]; + }): Promise { + const materialize = this.runtimeKernel.materializeRootSourceMessages; + if (!materialize) throw new Error('Runtime root message materialization is unavailable'); + return materialize.call(this.runtimeKernel, input); + } + /** Queue a user message for mid-turn injection at the next step boundary. */ steer(sessionId: string, text: string): QueueEnqueueOutcome { return this.runtimeKernel.steer(sessionId, text); From feca1d6b562b1536f19a813da8bd26a2ea197f6f Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 21:57:53 +0800 Subject: [PATCH 04/22] feat(storage): persist every accepted message transcript Generated-by: Codex --- packages/runtime/src/runtime-kernel.ts | 3 +- .../sqlite-session-metadata-store.test.ts | 55 +++++++++++++++++++ .../src/sqlite-session-metadata-store.ts | 4 +- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 570a780566..d3c51fe1f7 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -2429,7 +2429,8 @@ export class RuntimeKernel implements RuntimeKernelLike { existing.type !== 'user' || !messageContentsEqual(normalizeMessageContent(existing), message.content) || (existing.turnId !== input.turnId && - (message.disposition !== 'steering' || existing.turnId !== input.previousRootTurnId)) + (message.disposition !== 'steering' && message.disposition !== 'followup' || + existing.turnId !== input.previousRootTurnId)) ) { throw new Error(`Queued root source ${message.messageId} conflicts with its transcript`); } diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 5f52a4ad4a..82e13e40cf 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -302,6 +302,61 @@ describe('SqliteSessionMetadataStore', () => { } }); + test('atomically accepts a follow-up message and its canonical transcript', async () => { + const store = createSqliteSessionMetadataStore(':memory:'); + try { + await store.create(fullHeader({ id: 'session-followup-admission' })); + const admission = await store.commitMessageAdmission({ + sessionId: 'session-followup-admission', + turnId: 'turn-current', + runId: 'run-current', + messageId: 'message-followup', + content: { text: 'queued before the successor root' }, + modelContent: { text: 'queued before the successor root' }, + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: 11, + }); + assert.equal(admission.disposition, 'followup'); + assert.deepEqual( + (await store.readMessages('session-followup-admission')).map((message) => ({ + id: message.id, + turnId: message.turnId, + })), + [{ id: 'message-followup', turnId: 'turn-current' }], + ); + } finally { + store.close(); + } + }); + + test('rejects an oversized durable Message admission before transcript mutation', async () => { + const store = createSqliteSessionMetadataStore(':memory:'); + try { + await store.create(fullHeader({ id: 'session-oversized' })); + await assert.rejects( + () => + store.commitMessageAdmission({ + sessionId: 'session-oversized', + turnId: 'turn-oversized', + runId: 'run-oversized', + messageId: 'message-oversized', + content: { text: 'x'.repeat(70_000) }, + modelContent: { text: 'x'.repeat(70_000) }, + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt: 10, + }), + /exceeds size limit/, + ); + assert.deepEqual(await store.readMessages('session-oversized'), []); + } finally { + store.close(); + } + }); + test('migrates v24 legacy session statuses to active exactly once', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-status-v24-')); const path = join(root, 'state.sqlite'); diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 99437bdf2a..dffc85b13b 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1600,7 +1600,7 @@ export class SqliteSessionMetadataStore { stored.admittedAt, ); - if (stored.disposition === 'steering') { + if (stored.disposition === 'steering' || stored.disposition === 'followup') { const message = decodeCanonicalMessage({ type: 'user', id: stored.messageId, @@ -1762,7 +1762,7 @@ export class SqliteSessionMetadataStore { stored.sessionId, stored.messageId, ); - if (stored.disposition !== 'steering') return; + if (stored.disposition !== 'steering' && stored.disposition !== 'followup') return; const message = decodeCanonicalMessage({ type: 'user', id: stored.messageId, From 1f3313a19ce335e5ade3929687158dfdcbe87c73 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 21:57:58 +0800 Subject: [PATCH 05/22] feat(runtime-host): unify durable message settlement Generated-by: Codex --- .../server/client-capability-coordinator.ts | 16 ++++++ .../src/server/execution-composition.ts | 9 ++++ .../src/server/message-coordinator.ts | 49 ++++++++++++++++++- .../src/server/root-turn-coordinator.ts | 35 +++++-------- 4 files changed, 85 insertions(+), 24 deletions(-) diff --git a/packages/runtime-host/src/server/client-capability-coordinator.ts b/packages/runtime-host/src/server/client-capability-coordinator.ts index 40813af1b8..5841132bbd 100644 --- a/packages/runtime-host/src/server/client-capability-coordinator.ts +++ b/packages/runtime-host/src/server/client-capability-coordinator.ts @@ -21,6 +21,7 @@ import { createHash } from 'node:crypto'; import { AsyncLocalStorage } from 'node:async_hooks'; import { buildMcpTools, mcpProxyToolName, type McpToolProvider } from '@maka/runtime/mcp-tools'; import { type MakaTool } from '@maka/runtime/tool-runtime'; +import type { RootExecutionDescriptor } from '@maka/core/agent-run'; import { type ToolGroup } from '@maka/runtime/tool-availability'; import { type ClientCapabilityOffer, @@ -241,6 +242,21 @@ export class HostClientCapabilityCoordinator implements ClientCapabilityService } } + /** Rebuild Session-scoped capability bindings from a durable root contract. */ + async bindDurableRoot(input: { + sessionId: string; + userMessageId: string | null; + execution: RootExecutionDescriptor; + }): Promise { + if (input.execution.kind !== 'external_message' || input.userMessageId === null) return; + await this.#activation.runMutation(async () => { + const selection = this.#selectSessionState(input.sessionId, '', 'degrade'); + if (!selection.ok) throw new Error(selection.message); + this.#storeSessionState(input.sessionId, selection.state); + if (selection.modelToolsChanged) this.#onModelToolsChanged(); + }); + } + async #bindSession( sessionId: string, initiatingConnectionId: string, diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index c19fc0e4d2..51a5bba45d 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -482,6 +482,15 @@ export async function createExecutionRuntimeHostComposition( stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async ({ sessionId, turnId, runId, admittedAt }) => { + const events = await stores.agentRunStore.readEvents(sessionId, runId); + return events.some( + (event) => + event.turnId === turnId && + event.ts >= admittedAt && + event.type === 'model_call_attempt_recorded', + ); + }, }, receipts: stores.messageReceiptStore, lifecycle: stores.sessionStore, diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index dab6ddc675..f0b2916d93 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -175,6 +175,13 @@ export interface HostMessageDurableProofReader { sessionId: string, messageId: string, ): Promise; + /** True only when the admitted root has a durable downstream provider proof. */ + readProviderRequestProof?(input: { + sessionId: string; + turnId: string; + runId: string; + admittedAt: number; + }): Promise; } export interface HostMessageCoordinatorOptions { @@ -532,6 +539,34 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { await this.#lifecycle?.markMessagesExecuted(sessionId, messageIds); } + /** + * One settlement owner for both the normal terminal path and Host recovery. + * A queued message becomes Executed only after the durable root has recorded + * a provider request downstream of its admitted root contract. + */ + async settleMessagesAfterRoot(input: { + sessionId: string; + turnId: string; + runId: string; + admittedAt: number; + messageIds: readonly string[]; + }): Promise { + if (!this.#lifecycle || input.messageIds.length === 0) return; + if (!this.#durableProof.readProviderRequestProof) return; + const proved = await this.#durableProof.readProviderRequestProof(input); + if (!proved) return; + const executed: string[] = []; + for (const messageId of input.messageIds) { + if ( + (await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId)) === + 'handed_off' + ) { + executed.push(messageId); + } + } + await this.#lifecycle.markMessagesExecuted(input.sessionId, executed); + } + async cancelMessages(sessionId: string, messageIds: readonly string[]): Promise { await this.#lifecycle?.cancelMessageAdmissions(sessionId, messageIds); } @@ -550,6 +585,13 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { ); if (source) { await this.#lifecycle.markMessagesHandedOff(sessionId, [admission.messageId]); + await this.settleMessagesAfterRoot({ + sessionId, + turnId: source.admission.turnId, + runId: source.admission.runId, + admittedAt: source.admission.admittedAt, + messageIds: [admission.messageId], + }); } else { pending.push(admission); } @@ -562,7 +604,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { 'Message recovery authority is unavailable', ); } - await this.#sessionAdmission.run(sessionId, (admission) => + const started = await this.#sessionAdmission.run(sessionId, (admission) => this.#root.startRecoveredMessages!( { sessionId, @@ -573,6 +615,11 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { admission, ), ); + if ('error' in started) { + throw new RuntimeMessageAuthorityInvariantError( + `Durable Message recovery failed: ${started.error}`, + ); + } continue; } if (!this.#sessions.has(sessionId)) this.#state(sessionId); diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index e0a2e53678..213a07076f 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -1979,6 +1979,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (unavailableReason) { return completedStart(operationUnavailable(unavailableReason)); } + await this.clientCapabilities?.bindDurableRoot({ + sessionId: admission.sessionId, + userMessageId: admission.userMessageId, + execution: admission.execution, + }); const initialUserMessagesMaterialized = admission.sourceMessages.length > 0; if (initialUserMessagesMaterialized) { await this.manager.materializeRootSourceMessages({ @@ -2308,29 +2313,13 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { active.turnId, ); if (!admission || admission.sourceMessages.length === 0) return; - const events = await this.stores.agentRunStore.readEvents(active.sessionId, active.runId); - if ( - !events.some( - (event) => - event.type === 'provider_request_captured' || - event.type === 'provider_request_attempt_recorded' || - event.type === 'model_call_attempt_recorded', - ) - ) { - return; - } - const executed = [] as string[]; - for (const source of admission.sourceMessages) { - if ( - (await this.stores.sessionStore.readMessageLifecycleState( - active.sessionId, - source.messageId, - )) === 'handed_off' - ) { - executed.push(source.messageId); - } - } - await this.messages.markMessagesExecuted(active.sessionId, executed); + await this.messages.settleMessagesAfterRoot({ + sessionId: active.sessionId, + turnId: active.turnId, + runId: active.runId, + admittedAt: admission.admittedAt, + messageIds: admission.sourceMessages.map((source) => source.messageId), + }); } private observeExecutionCompletion( From 5f432d050e9c39a8cc286e49f6aa6b410b58d9e1 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 21:58:01 +0800 Subject: [PATCH 06/22] test(runtime): remove obsolete embedded queue coverage Generated-by: Codex --- .../src/__tests__/session-manager.test.ts | 924 ------------------ 1 file changed, 924 deletions(-) diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index a3c83a20ea..d140ce9415 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -14801,930 +14801,6 @@ describe('SessionManager permission mode updates', () => { }); }); -describe('SessionManager steering and followup queues', () => { - function steeringManager() { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - return { manager, store }; - } - - // Run a turn and invoke `duringFirstDelta` synchronously the first time the - // turn streams text — the point at which a real user would type while the - // agent works. Returns every streamed event. - async function runTurnWith( - manager: SessionManager, - sessionId: string, - turnId: string, - duringFirstDelta: () => void, - ): Promise { - const events: SessionEvent[] = []; - let fired = false; - for await (const event of manager.sendMessage(sessionId, { turnId, text: 'hello' })) { - events.push(event); - if (!fired && event.type === 'text_delta') { - fired = true; - duringFirstDelta(); - } - } - return events; - } - - test('hosted root runs consume the Host owner and release it exactly once', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const identities: RuntimeMessageRunIdentity[] = []; - const acked: string[] = []; - const nacked: string[] = []; - let pulled = false; - let releases = 0; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - messageAuthority: { - bindRun: (identity) => { - identities.push(identity); - return { - ...identity, - pull: () => { - if (pulled) return []; - pulled = true; - return [ - { - id: 'host-lease-1', - messageId: 'host-message-1', - content: { text: 'host steer', displayText: 'visible host steer' }, - }, - ]; - }, - ack: (leaseIds) => acked.push(...leaseIds), - nack: (leaseIds) => nacked.push(...leaseIds), - release: () => { - releases += 1; - }, - }; - }, - }, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const events = await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-host-message', text: 'start' }), - ); - const [run] = await runStore.listSessionRuns(session.id); - expect(identities).toEqual([ - { sessionId: session.id, turnId: 'turn-host-message', runId: run?.runId }, - ]); - expect(acked).toEqual(['host-lease-1']); - expect(nacked).toEqual([]); - expect(releases).toBe(1); - expect( - events.some( - (event) => - event.type === 'steering_message' && - event.messageId === 'host-message-1' && - event.content.displayText === 'visible host steer', - ), - ).toBe(true); - expect(events.some((event) => event.type === 'queue_update')).toBe(false); - for (const operation of [ - () => manager.steer(session.id, 'runtime mirror'), - () => manager.queueMessage(session.id, 'runtime mirror'), - () => manager.drainFollowup(session.id), - () => manager.retractQueue(session.id), - ]) { - let error: unknown; - try { - operation(); - } catch (caught) { - error = caught; - } - expect(error instanceof RuntimeMessageAuthorityInvariantError).toBe(true); - } - }); - - test('hosted Interaction binds the durable Run identity and closes before release', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const identities: RuntimeInteractionRunIdentity[] = []; - const lifecycle: string[] = []; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - interactionAuthority: { - bindRun: (identity) => { - identities.push(identity); - return { - ...identity, - acceptSandboxBoundaryRequest: async () => {}, - acceptUserQuestionRequest: async () => {}, - close: async (reason) => { - lifecycle.push(`close:${reason}`); - }, - release: () => lifecycle.push('release'), - }; - }, - }, - canonicalPermissionOutcomes: noCanonicalPermissionOutcomes, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-host-interaction', text: 'go' }), - ); - const [run] = await runStore.listSessionRuns(session.id); - expect(identities).toEqual([ - { sessionId: session.id, turnId: 'turn-host-interaction', runId: run?.runId }, - ]); - expect(lifecycle).toEqual(['close:turn_terminal', 'release']); - }); - - test('hosted stopped question abandonment preserves stop closure before release', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const lifecycle: string[] = []; - let question: RuntimeUserQuestionContinuation | undefined; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - interactionAuthority: { - bindRun: (identity) => ({ - ...identity, - acceptSandboxBoundaryRequest: async () => {}, - acceptUserQuestionRequest: async ({ continuation }) => { - question = continuation; - }, - close: async (reason) => { - lifecycle.push(`close:${reason}`); - await question?.applyClosure(reason); - lifecycle.push('local-settled'); - }, - release: () => lifecycle.push('release'), - }), - }, - canonicalPermissionOutcomes: noCanonicalPermissionOutcomes, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'ask' })); - const iterator = manager - .sendMessage(session.id, { - turnId: 'turn-host-question-abandoned', - text: FAKE_ASK_USER_QUESTION_PROMPT, - }) - [Symbol.asyncIterator](); - - let request: SessionEvent | undefined; - while (request?.type !== 'user_question_request') { - const next = await iterator.next(); - if (next.done) break; - request = next.value; - } - expect(request?.type).toBe('user_question_request'); - await manager.stopSession(session.id, { source: 'stop_button' }); - expect(lifecycle).toEqual(['close:turn_stopped', 'local-settled']); - await iterator.return?.(undefined); - expect(lifecycle).toEqual(['close:turn_stopped', 'local-settled', 'release']); - }); - - test('hosted RuntimeKernel rejects a backend request without an admission receipt', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new UnadmittedQuestionBackend(ctx)); - let releases = 0; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - interactionAuthority: { - bindRun: (identity) => ({ - ...identity, - acceptSandboxBoundaryRequest: async () => {}, - acceptUserQuestionRequest: async () => {}, - close: async () => {}, - release: () => { - releases += 1; - }, - }), - }, - canonicalPermissionOutcomes: noCanonicalPermissionOutcomes, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'ask' })); - - let failure: unknown; - try { - await drainAll(manager.sendMessage(session.id, { turnId: 'turn-forged', text: 'start' })); - } catch (error) { - failure = error; - } - expect(failure instanceof RuntimeInteractionInvariantError).toBe(true); - expect(releases).toBe(1); - }); - - test('hosted owner is released when backend execution fails', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new ThrowBeforeTerminalBackend(ctx)); - let releases = 0; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - messageAuthority: { - bindRun: (identity) => ({ - ...identity, - pull: () => [], - ack: () => {}, - nack: () => {}, - release: () => { - releases += 1; - }, - }), - }, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - let failure: unknown; - try { - await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-host-failed', text: 'start' }), - ); - } catch (error) { - failure = error; - } - expect((failure as Error).message).toBe('backend failed before terminal'); - expect(releases).toBe(1); - }); - - test('a failed turn begin never leaks a steering owner', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let failBuilds = 1; - backends.register('ai-sdk', (ctx) => { - if (failBuilds > 0) { - failBuilds -= 1; - throw new Error('backend build failed'); - } - return new FakeBackend(ctx); - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - let failed: unknown; - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-fail', - text: 'hello', - })) { - // drain - } - } catch (error) { - failed = error; - } - expect((failed as Error).message).toBe('backend build failed'); - - // The failed begin must not have left a live owner: steering falls back - // instead of queueing a message no run will ever consume. - expect(manager.steer(session.id, 'orphaned')).toEqual({ kind: 'fallback' }); - expect(manager.queueMessage(session.id, 'orphaned too')).toEqual({ kind: 'fallback' }); - - // A later successful turn establishes ownership normally. - let outcome: QueueEnqueueOutcome | undefined; - const events = await runTurnWith(manager, session.id, 'turn-2', () => { - outcome = manager.steer(session.id, 'now consumed'); - }); - expect(outcome?.kind).toBe('queued'); - expect( - events.some( - (event) => event.type === 'steering_message' && event.content.text === 'now consumed', - ), - ).toBe(true); - }); - - test('an overlapping turn cannot drain steering queued for the current owner', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let backend: GatedSteeringBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new GatedSteeringBackend(ctx); - return backend; - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const first = drainAll(manager.sendMessage(session.id, { turnId: 'turn-a', text: 'first' })); - await waitUntil(() => backend?.gates.has('turn-a') === true); - const second = drainAll(manager.sendMessage(session.id, { turnId: 'turn-b', text: 'second' })); - await waitUntil(() => backend?.gates.has('turn-b') === true); - - // turn-b established ownership last, so the steer targets it. - expect(manager.steer(session.id, 'for the owner').kind).toBe('queued'); - - // The stale turn's pull hook fails the identity check and drains nothing. - backend?.release('turn-a'); - const firstEvents = await first; - expect(backend?.pulls.get('turn-a')).toEqual([[]]); - expect(firstEvents.some((event) => event.type === 'steering_message')).toBe(false); - - // The owner drains exactly the queued message. - backend?.release('turn-b'); - const secondEvents = await second; - expect(backend?.pulls.get('turn-b')).toEqual([['for the owner']]); - expect( - secondEvents.some( - (event) => event.type === 'steering_message' && event.content.text === 'for the owner', - ), - ).toBe(true); - }); - - test('a pulled lease is past the retract point: retract excludes it and it delivers exactly once', async () => { - // Round-5 F1/D1: pull() is the single atomic commit point. Once leased, - // the message belongs to this turn's delivery — a retract during the - // (slow) durable append returns only still-queued text, never the - // in-flight lease; otherwise the retracted text would ALSO be executed by - // the provider once the append lands (refill + execute = two copies). - const gate = makeGate(); - const parked = makeGate(); - class GatedRuntimeEventStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new GatedRuntimeEventStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - expect(manager.steer(sessionId, 'urgent steer').kind).toBe('queued'); - }, - ); - - const turnEvents: SessionEvent[] = []; - const turn = (async () => { - for await (const event of manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })) { - turnEvents.push(event); - } - })(); - await parked.promise; - // The steering append has not committed: the next provider request must - // not have started while the message is not durable. - await new Promise((resolve) => setTimeout(resolve, 25)); - expect(model.doStreamCalls.length).toBe(1); - // Pulled means committed to this turn: retract returns nothing. - expect(manager.retractQueue(session.id)).toBe(''); - gate.release(); - await turn; - // The message delivered exactly once: in the next provider request… - expect(model.doStreamCalls.length).toBe(2); - expect(JSON.stringify(model.doStreamCalls[1]?.prompt).includes('urgent steer')).toBe(true); - // …echoed once in the stream/ledger… - expect(turnEvents.filter((event) => event.type === 'steering_message').length).toBe(1); - // …and owned by no queue afterwards. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('an abort never converts a durably appended steering message into a redelivery', async () => { - // Round-5 F1/D3: abort does not settle a pushed lease — settlement is - // decided only by the persistence fact. Here the append is parked when - // the stop arrives; once it commits, the message belongs to the ledger - // (history replay presents it to the next turn) and must NOT also be - // nacked into the followup queue, which would put the same directive in - // the account twice. - const gate = makeGate(); - const parked = makeGate(); - class GatedRuntimeEventStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new GatedRuntimeEventStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - expect(manager.steer(sessionId, 'urgent steer').kind).toBe('queued'); - }, - ); - - const turn = (async () => { - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - // drain - } - } catch { - // the abort may end the stream abruptly - } - })(); - await parked.promise; - void manager.stopSession(session.id, { source: 'stop_button' }); - // Let the abort reach the backend's durability wait while the append is - // still parked — the exact window where an abort-settles-the-lease bug - // nacks a message that then also commits to the ledger. - await new Promise((resolve) => setTimeout(resolve, 25)); - gate.release(); - // Teardown converges: the parked append commits, the lease settles, and - // the aborted send terminates without hanging. - await turn; - - // The dying request was never sent… - expect(model.doStreamCalls.length).toBe(1); - // …the ledger owns the message (exactly one durable steering event)… - const runs = await runStore.listSessionRuns(session.id); - const steeringEvents: RuntimeEvent[] = []; - for (const run of runs) { - const events = await runStore.readRuntimeEvents(session.id, run.runId); - steeringEvents.push( - ...events.filter( - (event) => - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true, - ), - ); - } - expect(steeringEvents.length).toBe(1); - // …and no queue redelivers it. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('a nack that lands after the owner released folds into the followup queue, not an ownerless steering queue', async () => { - // Round-5 F3: turn A's append fails only after turn B took over and - // released. A's nack can no longer target A (it will never pull again) — - // the text's only safe home is the followup queue, exactly where a - // release-time fold would have put it. - const gate = makeGate(); - const parked = makeGate(); - class ParkThenFailStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - throw new Error('steering append failed'); - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new ParkThenFailStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - manager.steer(sessionId, 'urgent steer'); - }, - ); - - const turnA = (async () => { - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - // drain - } - } catch { - // the failed append ends the stream abruptly - } - })(); - await parked.promise; - // Turn B takes ownership and releases it while A is parked. - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-2', - text: 'second', - })) { - // drain - } - gate.release(); - await turnA; - - // The failed message is redeliverable exactly once, via followup. - expect(manager.drainFollowup(session.id)).toBe('urgent steer'); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('steer falls back when no RuntimeEventStore is configured', async () => { - // Round-5 F4: without a runtime event ledger, the steering durability ack - // has nothing to anchor to — the fail-closed persist contract cannot be - // honored. The fallback path opens a fresh turn whose user message is - // persisted by the SessionStore, keeping the same durability guarantee. - const store = new MemorySessionStore(); - const backends = new BackendRegistry(); - let backend: GatedSteeringBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new GatedSteeringBackend(ctx); - return backend; - }); - const manager = new SessionManager({ - store, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const turn = drainAll(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })); - await waitUntil(() => backend?.gates.has('turn-1') === true); - // A live turn exists, but steering cannot be made durable: fall back. - expect(manager.steer(session.id, 'no ledger')).toEqual({ kind: 'fallback' }); - // Followups are unaffected — they open a normal turn anyway. - expect(manager.queueMessage(session.id, 'later').kind).toBe('queued'); - backend?.gates.get('turn-1')?.release(); - backend?.pullDone.get('turn-1')?.release(); - await turn; - }); - - test('a failed steering append nacks the lease back to the queue and the request never carries it', async () => { - // Fail-CLOSED persistence: the steering append throws, the ack judgment - // propagates the failure (no fail-open swallow), the lease is nacked back - // to the queue (folded into followup at release), and neither the ledger - // nor the projection carries the undelivered message. - class FailingSteeringStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - throw new Error('steering append failed'); - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new FailingSteeringStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - expect(manager.steer(sessionId, 'urgent steer').kind).toBe('queued'); - }, - ); - - let failed: unknown; - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - // drain - } - } catch (error) { - failed = error; - } - expect(failed instanceof Error).toBe(true); - // The dying request never carried the steering: no second provider call. - expect(model.doStreamCalls.length).toBe(1); - // Nacked back to the queue and folded into followup at release — the - // text is redeliverable, not lost. - expect(manager.drainFollowup(session.id)).toBe('urgent steer'); - // Ledger and projection agree: the message was never persisted. - const messages = await manager.getMessages(session.id); - expect( - messages.some((message) => message.type === 'user' && message.text === 'urgent steer'), - ).toBe(false); - }); - - test('an overlapping turn cannot turn a delivered lease into a followup redelivery', async () => { - // Round-4 V1: turn A leases the steer and parks in the (gated) durable - // append; turn B starts meanwhile and takes the owner slot. A's append - // then commits and A's provider request carries the message — so A's ack - // MUST still settle the lease (it is keyed by issuer, not by the current - // owner), and B's teardown must not fold A's in-flight lease into the - // followup queue, which would redeliver an already-executed directive. - const gate = makeGate(); - const parked = makeGate(); - class GatedRuntimeEventStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new GatedRuntimeEventStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - manager.steer(sessionId, 'urgent steer'); - }, - ); - - const turnAEvents: SessionEvent[] = []; - const turnA = (async () => { - try { - for await (const event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - turnAEvents.push(event); - } - } catch { - // A gated teardown may end the stream abruptly. - } - })(); - await parked.promise; - - // Turn B runs to completion while A is parked mid-lease. - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-2', - text: 'second', - })) { - // drain - } - expect(model.doStreamCalls.length).toBe(2); - - gate.release(); - await turnA; - - // A's post-steer request went out carrying the directive exactly once… - expect(model.doStreamCalls.length).toBe(3); - expect(JSON.stringify(model.doStreamCalls[2]?.prompt).includes('urgent steer')).toBe(true); - // …B's request never did… - expect(JSON.stringify(model.doStreamCalls[1]?.prompt).includes('urgent steer')).toBe(false); - // …the ledger echoes it exactly once… - expect(turnAEvents.filter((event) => event.type === 'steering_message').length).toBe(1); - // …and NOTHING redelivers it: the delivered lease was acked by its - // issuer, so no queue still holds the text. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('a backend-forged queue_update never reaches the ledger or observers', async () => { - // Round-6 R3: the kernel is the only legal producer of queue_update (it - // pushes them directly into the turn stream). A backend that yields one - // is forging authoritative queue state; the flow drops it at the ingress - // — not mapped, not forwarded, not persisted. - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new ForgingQueueBackend(ctx)); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const events = await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' }), - ); - // Nothing was enqueued in this turn, so ANY queue_update in the stream - // is the forged one leaking through. - expect(events.some((event) => event.type === 'queue_update')).toBe(false); - const runs = await runStore.listSessionRuns(session.id); - const runtimeEvents = ( - await Promise.all(runs.map((run) => runStore.readRuntimeEvents(session.id, run.runId))) - ).flat(); - expect( - runtimeEvents.some( - (event) => - (event.actions?.stateDelta as { queueUpdate?: unknown } | undefined)?.queueUpdate !== - undefined, - ), - ).toBe(false); - }); - - test('provider retry progress reaches observers without becoming a durable runtime fact', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new ProviderRetryProgressBackend(ctx)); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const events = await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' }), - ); - expect( - events.filter((event) => event.type === 'provider_retry').map((event) => event.phase), - ).toEqual(['scheduled', 'started']); - - const runs = await runStore.listSessionRuns(session.id); - const runtimeEvents = ( - await Promise.all(runs.map((run) => runStore.readRuntimeEvents(session.id, run.runId))) - ).flat(); - expect( - runtimeEvents.some( - (event) => - (event.actions?.stateDelta as { providerRetry?: unknown } | undefined)?.providerRetry !== - undefined, - ), - ).toBe(false); - }); - - test('an append error after the write landed settles by the ledger read-back, not a duplicate nack', async () => { - // Round-6 R5: appendRuntimeEvent can fail AFTER the bytes landed (e.g. a - // close error). Treating every append error as not-durable would nack a - // message the ledger already owns — history replay plus the followup - // redelivery equals a double. The ambiguous failure is settled by reading - // the ledger back: present ⇒ durable ⇒ ack path. - class WriteThenThrowStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - await super.appendRuntimeEvent(sessionId, runId, event); - throw new Error('close failed after the write landed'); - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new WriteThenThrowStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - manager.steer(sessionId, 'urgent steer'); - }, - ); - - const turnEvents: SessionEvent[] = []; - for await (const event of manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })) { - turnEvents.push(event); - } - - // Delivered exactly once: the next request carries it… - expect(model.doStreamCalls.length).toBe(2); - expect(JSON.stringify(model.doStreamCalls[1]?.prompt).includes('urgent steer')).toBe(true); - // …the ledger owns exactly one copy… - const runs = await runStore.listSessionRuns(session.id); - const steeringEvents: RuntimeEvent[] = []; - for (const run of runs) { - const events = await runStore.readRuntimeEvents(session.id, run.runId); - steeringEvents.push( - ...events.filter( - (event) => - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true, - ), - ); - } - expect(steeringEvents.length).toBe(1); - // …and no queue redelivers it. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('stranded steering emits a final queue snapshot when it folds into the followup queue', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let backend: GatedSteeringBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new GatedSteeringBackend(ctx); - return backend; - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const turn = drainAll(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })); - await waitUntil(() => backend?.gates.has('turn-1') === true); - backend?.gates.get('turn-1')?.release(); - // The turn's only step boundary has already pulled (empty)… - await waitUntil(() => backend?.pulls.has('turn-1') === true); - // …so this steer is stranded: no step is left to consume it. - expect(manager.steer(session.id, 'late').kind).toBe('queued'); - backend?.pullDone.get('turn-1')?.release(); - const events = await turn; - - // The stranded → followup migration is a queue change; the LAST snapshot - // in the stream reflects it, not the stale pre-fold state. - const updates = events.filter( - (event): event is Extract => - event.type === 'queue_update', - ); - expect(updates.at(-1)?.steering).toEqual([]); - expect(updates.at(-1)?.followup).toEqual(['late']); - expect(updates.at(-1)?.steeringEntries).toEqual([]); - expect(updates.at(-1)?.followupEntries).toHaveLength(1); - expect(updates.at(-1)?.followupEntries?.[0]?.content).toEqual({ text: 'late' }); - expect(updates.at(-1)?.followupEntries?.[0]?.placement).toBe('next_turn'); - expect(updates.at(-1)?.followupEntries?.[0]?.state).toBe('queued'); - // And the followup queue is the authoritative owner of the text. - expect(manager.drainFollowup(session.id)).toBe('late'); - }); -}); - async function drainAll(iterable: AsyncIterable): Promise { const events: SessionEvent[] = []; for await (const event of iterable) events.push(event); From e180cbfacc4e28805f6ebb71b559893e81449f01 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 22:07:00 +0800 Subject: [PATCH 07/22] chore: satisfy repository formatting check Generated-by: Codex --- .../__tests__/execution-host-queue.test.ts | 7 ++++-- .../src/server/execution-composition.ts | 4 +++- .../src/server/root-turn-coordinator.ts | 5 +--- packages/runtime/src/runtime-kernel.ts | 2 +- .../sqlite-session-metadata-store.test.ts | 15 +++--------- packages/storage/src/execution-stores.ts | 9 +++++--- packages/storage/src/session-store.ts | 9 ++++---- .../src/sqlite-session-metadata-store.ts | 23 +++++++++++++------ 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index 77703f95e5..92abf63003 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -240,7 +240,9 @@ test('production UDS admission commits one transcript before the root handoff', await fixture.stopHost(host); const ledger = await fixture.readTurn(started.turnId); assert.deepEqual( - ledger.userMessages.filter((message) => message.id === messageId).map((message) => message.id), + ledger.userMessages + .filter((message) => message.id === messageId) + .map((message) => message.id), [messageId], ); assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); @@ -285,7 +287,8 @@ test('a Host crash after queue admission recovers the durable successor once', a 'durable successor was not recovered after the Host crash', ); assert.equal(successor.kind, 'subscription.session_projection'); - if (successor.kind !== 'subscription.session_projection' || !successor.snapshot.rootTurn) return; + if (successor.kind !== 'subscription.session_projection' || !successor.snapshot.rootTurn) + return; await waitForTerminalTurn(second, fixture.sessionId, successor.snapshot.rootTurn.turnId); await subscription.close(); await probe.done; diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 51a5bba45d..c220755f0f 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -1497,7 +1497,9 @@ export async function createExecutionRuntimeHostComposition( ), ); await coordinator.recover(); - await messages.recoverPendingAfterHostRestart(recoverySessions.map((session) => session.id)); + await messages.recoverPendingAfterHostRestart( + recoverySessions.map((session) => session.id), + ); rootRecoveryCompleted = true; }, }, diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 213a07076f..fd34416959 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -2387,10 +2387,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { // provider is unavailable. Lost tools are omitted while ephemeral // capabilities bind to the Client that submitted this follow-up. if (initiatingConnectionId) { - await this.clientCapabilities?.bindConfirmedFollowup( - batch.sessionId, - initiatingConnectionId, - ); + await this.clientCapabilities?.bindConfirmedFollowup(batch.sessionId, initiatingConnectionId); } const turnId = randomUUID(); diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index d3c51fe1f7..05ce8d52e9 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -2429,7 +2429,7 @@ export class RuntimeKernel implements RuntimeKernelLike { existing.type !== 'user' || !messageContentsEqual(normalizeMessageContent(existing), message.content) || (existing.turnId !== input.turnId && - (message.disposition !== 'steering' && message.disposition !== 'followup' || + ((message.disposition !== 'steering' && message.disposition !== 'followup') || existing.turnId !== input.previousRootTurnId)) ) { throw new Error(`Queued root source ${message.messageId} conflicts with its transcript`); diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 82e13e40cf..7e9e6a3c83 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -283,20 +283,11 @@ describe('SqliteSessionMetadataStore', () => { }, ], ); - assert.equal( - await store.readMessageLifecycleState('session-1', 'message-1'), - 'accepted', - ); + assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'accepted'); await store.markMessagesHandedOff('session-1', ['message-1']); - assert.equal( - await store.readMessageLifecycleState('session-1', 'message-1'), - 'handed_off', - ); + assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'handed_off'); await store.markMessagesExecuted('session-1', ['message-1']); - assert.equal( - await store.readMessageLifecycleState('session-1', 'message-1'), - 'executed', - ); + assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'executed'); } finally { store.close(); } diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index e0b134120f..6beccbc293 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -428,13 +428,16 @@ async function createExecutionStoresForWrite sessionStore.appendMessage(sessionId, message)), appendMessages: (sessionId, messages) => run(() => sessionStore.appendMessages(sessionId, messages)), - commitMessageAdmission: (admission) => run(() => sessionStore.commitMessageAdmission(admission)), + commitMessageAdmission: (admission) => + run(() => sessionStore.commitMessageAdmission(admission)), readMessageAdmission: (sessionId, messageId) => run(() => sessionStore.readMessageAdmission(sessionId, messageId)), readMessageLifecycleState: (sessionId, messageId) => run(() => sessionStore.readMessageLifecycleState(sessionId, messageId)), - listMessageAdmissions: (sessionId) => run(() => sessionStore.listMessageAdmissions(sessionId)), - updateMessageAdmission: (admission) => run(() => sessionStore.updateMessageAdmission(admission)), + listMessageAdmissions: (sessionId) => + run(() => sessionStore.listMessageAdmissions(sessionId)), + updateMessageAdmission: (admission) => + run(() => sessionStore.updateMessageAdmission(admission)), reorderMessageAdmissions: (sessionId, messageIds) => run(() => sessionStore.reorderMessageAdmissions(sessionId, messageIds)), cancelMessageAdmissions: (sessionId, messageIds) => diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 5b310c0768..7420840053 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -80,10 +80,7 @@ import { type TurnStateMessage, type UserMessage, } from '@maka/core/session'; -import type { - MessageLifecycleStore, - PendingMessageAdmission, -} from './message-receipt-store.js'; +import type { MessageLifecycleStore, PendingMessageAdmission } from './message-receipt-store.js'; const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -861,7 +858,9 @@ class SqliteSessionStore implements SessionAuthorityStore { for (const listener of this.transcriptChangeListeners) listener(sessionId); } - async commitMessageAdmission(admission: PendingMessageAdmission): Promise { + async commitMessageAdmission( + admission: PendingMessageAdmission, + ): Promise { await this.ensureReady(); const committed = await this.metadata.commitMessageAdmission(admission); for (const listener of this.transcriptChangeListeners) listener(admission.sessionId); diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index dffc85b13b..29f8d77ed0 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1609,11 +1609,14 @@ export class SqliteSessionMetadataStore { ...stored.content, steeringEventId: stored.messageId, }); - const existingMessages = this.readMessagesWith(stored.sessionId, decodeStoredMessage).filter( - (candidate) => candidate.id === stored.messageId, - ); + const existingMessages = this.readMessagesWith( + stored.sessionId, + decodeStoredMessage, + ).filter((candidate) => candidate.id === stored.messageId); if (existingMessages.length > 1) { - throw new SessionMetadataConflictError('Message admission transcript identity is ambiguous'); + throw new SessionMetadataConflictError( + 'Message admission transcript identity is ambiguous', + ); } const existingMessage = existingMessages[0]; if (existingMessage && !isDeepStrictEqual(existingMessage, message)) { @@ -1784,7 +1787,9 @@ export class SqliteSessionMetadataStore { record_json?: unknown; }>; if (rows.length > 1) { - throw new SessionMetadataConflictError('Message admission transcript identity is ambiguous'); + throw new SessionMetadataConflictError( + 'Message admission transcript identity is ambiguous', + ); } const json = JSON.stringify(message); if (rows.length === 0) { @@ -1837,7 +1842,9 @@ export class SqliteSessionMetadataStore { for (const messageId of unique) { const result = statement.run(sessionId, messageId); if (result.changes !== 1) { - throw new SessionMetadataConflictError('Message admission cancellation identity conflict'); + throw new SessionMetadataConflictError( + 'Message admission cancellation identity conflict', + ); } } }); @@ -1848,7 +1855,9 @@ export class SqliteSessionMetadataStore { assertSafeSessionId(sessionId); const unique = [...new Set(messageIds)]; if (unique.length !== messageIds.length) { - throw new SessionMetadataConflictError('Message admission reorder contains duplicate identities'); + throw new SessionMetadataConflictError( + 'Message admission reorder contains duplicate identities', + ); } for (const messageId of unique) assertSafeSessionId(messageId); this.transaction(() => { From 1496b959126cbf6260ade9e803a4efdaa527864a Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 22:42:41 +0800 Subject: [PATCH 08/22] fix(runtime): keep atomic message transcripts recovery-safe Preserve live Client capability bindings, make cancellation retries idempotent, and keep admission-backed transcripts out of compatibility Run synthesis until their root contract owns them. Generated-by: Codex --- .../server/client-capability-coordinator.ts | 5 +++++ .../src/server/hosted-execution-recovery.ts | 2 +- .../src/server/root-turn-coordinator.ts | 4 +++- packages/runtime/src/runtime-ledger-repair.ts | 19 +++++++++++++++++-- packages/runtime/src/session-manager.ts | 11 +++++++++++ .../src/sqlite-session-metadata-store.ts | 13 ++++++++++--- 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/packages/runtime-host/src/server/client-capability-coordinator.ts b/packages/runtime-host/src/server/client-capability-coordinator.ts index 5841132bbd..09e18cd565 100644 --- a/packages/runtime-host/src/server/client-capability-coordinator.ts +++ b/packages/runtime-host/src/server/client-capability-coordinator.ts @@ -249,6 +249,11 @@ export class HostClientCapabilityCoordinator implements ClientCapabilityService execution: RootExecutionDescriptor; }): Promise { if (input.execution.kind !== 'external_message' || input.userMessageId === null) return; + // A live root already selected its Client capabilities at admission. Only + // cold recovery needs to rebuild a missing in-memory binding from the + // durable root contract; reselecting here would discard the active + // connection/turn-affine binding and can make providers ambiguous. + if (this.#sessions.has(input.sessionId)) return; await this.#activation.runMutation(async () => { const selection = this.#selectSessionState(input.sessionId, '', 'degrade'); if (!selection.ok) throw new Error(selection.message); diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 5187583b16..674c92e2d2 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -69,7 +69,7 @@ export async function prepareHostedExecutionRecovery( const run = runsById.get(admission.runId); const rootUserMessages = ( messageIndex.userMessagesByTurnId.get(admission.turnId) ?? [] - ).filter((message) => message.steeringEventId === undefined); + ).filter((message) => message.id === admission.userMessageId); const messageIdOwners = admission.userMessageId ? (messageIndex.messagesById.get(admission.userMessageId) ?? []) : []; diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index fd34416959..9dd00b6928 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -1984,7 +1984,9 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { userMessageId: admission.userMessageId, execution: admission.execution, }); - const initialUserMessagesMaterialized = admission.sourceMessages.length > 0; + const initialUserMessagesMaterialized = + admission.sourceMessages.length > 0 && + admission.sourceMessages.every((source) => source.disposition === 'turn_started'); if (initialUserMessagesMaterialized) { await this.manager.materializeRootSourceMessages({ sessionId: input.sessionId, diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 98e85f4105..6071b73d0b 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -38,6 +38,8 @@ export interface RuntimeLedgerRepairDeps { runStore: AgentRunStore; runtimeEventStore: RuntimeEventStore; readMessages(sessionId: string): Promise; + readPendingMessageIds?(sessionId: string): Promise; + readMessageLifecycleState?(sessionId: string, messageId: string): Promise; appendMessage(sessionId: string, message: StoredMessage): Promise; appendTurnState( sessionId: string, @@ -89,11 +91,24 @@ export class RuntimeLedgerRepair { this.deps.readMessages(sessionId), this.deps.runStore.listSessionRuns(sessionId), ]); + const pendingMessageIds = new Set((await this.deps.readPendingMessageIds?.(sessionId)) ?? []); + if (this.deps.readMessageLifecycleState) { + const lifecycleStates = await Promise.all( + messages.map(async (message) => ({ + messageId: message.id, + state: await this.deps.readMessageLifecycleState!(sessionId, message.id), + })), + ); + for (const { messageId, state } of lifecycleStates) { + if (state !== undefined) pendingMessageIds.add(messageId); + } + } + const ledgerMessages = messages.filter((message) => !pendingMessageIds.has(message.id)); const inlineRunsByTurn = new Map( runs.filter(isSessionInlineRun).map((run) => [run.turnId, run] as const), ); - const messagesByTurn = groupMessagesByTurn(messages); - const turns = deriveTurnRecords(messages).filter((turn) => + const messagesByTurn = groupMessagesByTurn(ledgerMessages); + const turns = deriveTurnRecords(ledgerMessages).filter((turn) => (messagesByTurn.get(turn.turnId) ?? []).some((message) => message.type === 'user'), ); if (turns.length === 0) return; diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index e351c721d0..d02a6f725c 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -671,6 +671,11 @@ export interface SessionStore { list(filter?: SessionListFilter): Promise; readHeader(sessionId: string): Promise; readMessages(sessionId: string): Promise; + listMessageAdmissions?(sessionId: string): Promise; + readMessageLifecycleState?( + sessionId: string, + messageId: string, + ): Promise<'accepted' | 'handed_off' | 'executed' | 'cancelled' | undefined>; readMessagesSnapshot?(sessionId: string): Promise; listTurns(sessionId: string): Promise; appendMessage(sessionId: string, m: StoredMessage): Promise; @@ -938,6 +943,12 @@ export class SessionManager { runStore: deps.runStore, runtimeEventStore: deps.runtimeEventStore, readMessages: (sessionId) => deps.store.readMessages(sessionId), + readPendingMessageIds: async (sessionId) => + (await deps.store.listMessageAdmissions?.(sessionId))?.map( + ({ messageId }) => messageId, + ) ?? [], + readMessageLifecycleState: async (sessionId, messageId) => + deps.store.readMessageLifecycleState?.(sessionId, messageId) ?? undefined, appendMessage: (sessionId, message) => deps.store.appendMessage(sessionId, message), appendTurnState: (sessionId, turnId, status, lineage, options) => this.appendTurnState(sessionId, turnId, status, lineage, options), diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 29f8d77ed0..6fc0e5932e 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1842,9 +1842,16 @@ export class SqliteSessionMetadataStore { for (const messageId of unique) { const result = statement.run(sessionId, messageId); if (result.changes !== 1) { - throw new SessionMetadataConflictError( - 'Message admission cancellation identity conflict', - ); + const existing = this.db + .prepare( + 'SELECT lifecycle_state FROM message_admissions WHERE session_id = ? AND message_id = ?', + ) + .get(sessionId, messageId) as { lifecycle_state?: unknown } | undefined; + if (existing?.lifecycle_state !== 'cancelled') { + throw new SessionMetadataConflictError( + 'Message admission cancellation identity conflict', + ); + } } } }); From ff270059b6ca639b4b4f887b54b30cbffc319921 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 23:47:12 +0800 Subject: [PATCH 09/22] fix(runtime): prove prepared root sources by submitted digest Generated-by: Codex --- packages/core/src/events.ts | 19 ++++++++ .../src/__tests__/message-coordinator.test.ts | 3 +- .../src/server/message-content-digest.ts | 38 ---------------- .../src/server/message-coordinator.ts | 2 +- .../src/server/root-turn-coordinator.ts | 5 ++- .../runtime-kernel-interaction.test.ts | 43 ++++++++++++++++++- packages/runtime/src/runtime-kernel.ts | 8 +++- packages/runtime/src/session-manager.ts | 1 + 8 files changed, 75 insertions(+), 44 deletions(-) delete mode 100644 packages/runtime-host/src/server/message-content-digest.ts diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index 4a58287e49..15c6170fb5 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -26,6 +26,7 @@ * Connection-setup events live in ./connections.ts (separate channel). */ +import * as nodeCrypto from 'node:crypto'; import type { AdditionalPermissionRequest, PermissionMode, @@ -398,6 +399,24 @@ export function messageContentsEqual(left: MessageContent, right: MessageContent ); } +export function messageContentDigest(content: MessageContent): `sha256:${string}` { + return `sha256:${nodeCrypto + .createHash('sha256') + .update(JSON.stringify(canonicalizeMessageContent(normalizeMessageContent(content)))) + .digest('hex')}`; +} + +function canonicalizeMessageContent(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalizeMessageContent); + if (value === null || typeof value !== 'object') return value; + return Object.fromEntries( + Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => [key, canonicalizeMessageContent(entry)]), + ); +} + function inlineReferencesEqual(left: InlineReference, right: InlineReference): boolean { return ( left.kind === right.kind && diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index 934227bd5a..62024d88d7 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -19,7 +19,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import type { MessageContent } from '@maka/core/events'; +import { messageContentDigest, type MessageContent } from '@maka/core/events'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { MessageOperationReceipt, @@ -39,7 +39,6 @@ import { type HostMessageRootPort, type HostMessageRootState, } from '../server/message-coordinator.js'; -import { messageContentDigest } from '../server/message-content-digest.js'; import { SessionAdmissionGate } from '../server/session-admission-gate.js'; const ROOT = { sessionId: 'session-1', turnId: 'turn-1', runId: 'run-1' } as const; diff --git a/packages/runtime-host/src/server/message-content-digest.ts b/packages/runtime-host/src/server/message-content-digest.ts deleted file mode 100644 index 03f88d598a..0000000000 --- a/packages/runtime-host/src/server/message-content-digest.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { createHash } from 'node:crypto'; -import { normalizeMessageContent, type MessageContent } from '@maka/core/events'; - -export function messageContentDigest(content: MessageContent): `sha256:${string}` { - return `sha256:${createHash('sha256') - .update(JSON.stringify(canonicalize(normalizeMessageContent(content)))) - .digest('hex')}`; -} - -function canonicalize(value: unknown): unknown { - if (Array.isArray(value)) return value.map(canonicalize); - if (value === null || typeof value !== 'object') return value; - return Object.fromEntries( - Object.entries(value) - .filter(([, entry]) => entry !== undefined) - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([key, entry]) => [key, canonicalize(entry)]), - ); -} diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index f0b2916d93..32669e814c 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -22,6 +22,7 @@ import { isDeepStrictEqual } from 'node:util'; import type { SteeringLease } from '@maka/core/backend-types'; import { aggregateMessageContents, + messageContentDigest, messageContentsEqual, normalizeMessageContent, type MessageContent, @@ -72,7 +73,6 @@ import type { RuntimeHostResidency } from './host-kernel.js'; import { worstCaseFailedTurnSnapshot } from './canonical-turn-snapshot.js'; import { worstCaseMessageQueueProjection } from './message-queue-capacity.js'; import type { ConnectionContext, MessageOperationHandlerMap } from './operation-dispatcher.js'; -import { messageContentDigest } from './message-content-digest.js'; import { type SessionAdmissionLease, SessionAdmissionGate } from './session-admission-gate.js'; type MessageOperationErrorCode = diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 9dd00b6928..12474ec8ef 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -23,6 +23,7 @@ import type { BackendStopMode } from '@maka/core/backend-types'; import type { AgentRunHeader, RootExecutionDescriptor } from '@maka/core/agent-run'; import { INLINE_REFERENCE_MAX_COUNT, + messageContentDigest, messageContentsEqual, normalizeMessageContent, type AttachmentRef, @@ -88,7 +89,6 @@ import { type QueueFenceResult, type RootFollowupBatch, } from './message-coordinator.js'; -import { messageContentDigest } from './message-content-digest.js'; import type { ConnectionContext, TurnOperationHandlerMap } from './operation-dispatcher.js'; import { RootAdmissionOwner } from './root-admission-owner.js'; import { type SessionAdmissionLease, SessionAdmissionGate } from './session-admission-gate.js'; @@ -1995,6 +1995,9 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { messages: admission.sourceMessages.map((source) => ({ messageId: source.messageId, content: source.content, + ...(source.submittedContentDigest + ? { submittedContentDigest: source.submittedContentDigest } + : {}), disposition: source.disposition, })), }); diff --git a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts index 4eab4a25e4..df2d5ed932 100644 --- a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts +++ b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts @@ -20,7 +20,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { SessionEvent } from '@maka/core/events'; +import { messageContentDigest, type SessionEvent } from '@maka/core/events'; import type { SessionHeader, StoredMessage } from '@maka/core/session'; import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary'; @@ -39,6 +39,47 @@ import { import { BackendRegistry, type SessionStore } from '../session-manager.js'; describe('RuntimeKernel Interaction close cleanup', () => { + test('accepts submitted transcript content when the root source is model-prepared', async () => { + const store = memoryStore(); + const submitted = { text: '/skill:writer inspect' }; + await store.appendMessage(SESSION_ID, { + type: 'user', + id: 'submitted-message', + turnId: 'prepared-turn', + ts: 1, + ...submitted, + }); + const kernel = new RuntimeKernel({ + store, + backends: new BackendRegistry(), + newId: () => 'materialize-id', + now: () => 1, + }); + + await kernel.materializeRootSourceMessages({ + sessionId: SESSION_ID, + turnId: 'prepared-turn', + previousRootTurnId: null, + messages: [ + { + messageId: 'submitted-message', + content: { text: 'inspect' }, + submittedContentDigest: messageContentDigest(submitted), + disposition: 'turn_started', + }, + ], + }); + assert.deepEqual(await store.readMessages(SESSION_ID), [ + { + type: 'user', + id: 'submitted-message', + turnId: 'prepared-turn', + ts: 1, + ...submitted, + }, + ]); + }); + test('reserve followed by begin failure settles a concurrent stop claim', async () => { const store = memoryStore(); const updateHeader = store.updateHeader; diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 05ce8d52e9..25037847e7 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -34,6 +34,7 @@ import type { } from '@maka/core/runtime-event-store'; import { isSessionInlineRun } from '@maka/core/agent-run'; import { + messageContentDigest, messageContentsEqual, normalizeMessageContent, type ActiveInteractionRequestEvent, @@ -194,6 +195,7 @@ export interface RuntimeKernelLike { messages: readonly { messageId: string; content: MessageContent; + submittedContentDigest?: `sha256:${string}`; disposition: 'steering' | 'followup' | 'turn_started'; }[]; }): Promise; @@ -2416,6 +2418,7 @@ export class RuntimeKernel implements RuntimeKernelLike { messages: readonly { messageId: string; content: MessageContent; + submittedContentDigest?: `sha256:${string}`; disposition: 'steering' | 'followup' | 'turn_started'; }[]; }): Promise { @@ -2427,7 +2430,10 @@ export class RuntimeKernel implements RuntimeKernelLike { if (existing) { if ( existing.type !== 'user' || - !messageContentsEqual(normalizeMessageContent(existing), message.content) || + (!messageContentsEqual(normalizeMessageContent(existing), message.content) && + (message.submittedContentDigest === undefined || + messageContentDigest(normalizeMessageContent(existing)) !== + message.submittedContentDigest)) || (existing.turnId !== input.turnId && ((message.disposition !== 'steering' && message.disposition !== 'followup') || existing.turnId !== input.previousRootTurnId)) diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index d02a6f725c..4c47734384 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -4827,6 +4827,7 @@ export class SessionManager { messages: readonly { messageId: string; content: import('@maka/core/events').MessageContent; + submittedContentDigest?: `sha256:${string}`; disposition: 'steering' | 'followup' | 'turn_started'; }[]; }): Promise { From 5b8061ce13925a4f43fd71a87e564ca3e8f0aadb Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 24 Aug 2026 23:50:20 +0800 Subject: [PATCH 10/22] fix(runtime-host): settle handed off messages on terminal stop Generated-by: Codex --- .../__tests__/execution-host-queue.test.ts | 2 +- .../src/server/message-coordinator.ts | 33 ++++++++++++------- .../src/server/root-turn-coordinator.ts | 27 ++++++++++++--- .../src/sqlite-session-metadata-store.ts | 6 ++-- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index 92abf63003..fb0cc029ba 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -245,7 +245,7 @@ test('production UDS admission commits one transcript before the root handoff', .map((message) => message.id), [messageId], ); - assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); + assert.equal(await fixture.readMessageLifecycleState(messageId), 'cancelled'); }); }); diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index 32669e814c..d9b28a5c44 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -550,21 +550,32 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { runId: string; admittedAt: number; messageIds: readonly string[]; + terminalStatus?: 'completed' | 'failed' | 'cancelled'; }): Promise { if (!this.#lifecycle || input.messageIds.length === 0) return; - if (!this.#durableProof.readProviderRequestProof) return; - const proved = await this.#durableProof.readProviderRequestProof(input); - if (!proved) return; - const executed: string[] = []; - for (const messageId of input.messageIds) { - if ( - (await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId)) === - 'handed_off' - ) { - executed.push(messageId); + const proved = this.#durableProof.readProviderRequestProof + ? await this.#durableProof.readProviderRequestProof(input) + : false; + if (proved) { + const executed: string[] = []; + for (const messageId of input.messageIds) { + if ( + (await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId)) === + 'handed_off' + ) { + executed.push(messageId); + } } + await this.#lifecycle.markMessagesExecuted(input.sessionId, executed); + return; + } + if (input.terminalStatus !== 'cancelled') return; + const cancelled: string[] = []; + for (const messageId of input.messageIds) { + const state = await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId); + if (state === 'accepted' || state === 'handed_off') cancelled.push(messageId); } - await this.#lifecycle.markMessagesExecuted(input.sessionId, executed); + await this.#lifecycle.cancelMessageAdmissions(input.sessionId, cancelled); } async cancelMessages(sessionId: string, messageIds: readonly string[]): Promise { diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 12474ec8ef..70a718751f 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -362,7 +362,18 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { admission.runId, run, ); - if (!isTerminalSnapshot(snapshot)) { + if (isTerminalSnapshot(snapshot)) { + if (admission.sourceMessages.length > 0) { + await this.messages.settleMessagesAfterRoot({ + sessionId, + turnId: admission.turnId, + runId: admission.runId, + admittedAt: admission.admittedAt, + messageIds: admission.sourceMessages.map((source) => source.messageId), + terminalStatus: snapshot.status, + }); + } + } else { if (admission.execution.kind !== 'safe_boundary_continuation') { throw new Error(`Startup recovery left Turn ${admission.turnId} non-terminal`); } @@ -2232,7 +2243,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } this.observeExecutionCompletion(active, { kind: 'terminal', snapshot }); await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status); - await this.settleExecutedMessageSources(active); + await this.settleExecutedMessageSources(active, snapshot.status); terminalTransitionStarted = true; await this.completeTerminalTransition(input.sessionId, active); } catch (error) { @@ -2256,7 +2267,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { snapshot, }); await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status); - await this.settleExecutedMessageSources(active); + await this.settleExecutedMessageSources(active, snapshot.status); terminalTransitionStarted = true; await this.completeTerminalTransition(input.sessionId, active); containedRunFailure = @@ -2312,7 +2323,10 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } } - private async settleExecutedMessageSources(active: ActiveRootTurn): Promise { + private async settleExecutedMessageSources( + active: ActiveRootTurn, + terminalStatus: 'completed' | 'failed' | 'cancelled', + ): Promise { const admission = await this.stores.agentRunStore.readRootTurnAdmission( active.sessionId, active.turnId, @@ -2324,6 +2338,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { runId: active.runId, admittedAt: admission.admittedAt, messageIds: admission.sourceMessages.map((source) => source.messageId), + terminalStatus, }); } @@ -2864,7 +2879,9 @@ function throwIfAborted(signal: AbortSignal): void { throw new DOMException('Agent graph supervisor Turn was aborted', 'AbortError'); } -function isTerminalSnapshot(snapshot: TurnSnapshot): boolean { +function isTerminalSnapshot( + snapshot: TurnSnapshot, +): snapshot is Extract { return ( snapshot.status === 'completed' || snapshot.status === 'failed' || diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 6fc0e5932e..1ecf89751d 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1836,7 +1836,7 @@ export class SqliteSessionMetadataStore { ` UPDATE message_admissions SET lifecycle_state = 'cancelled' - WHERE session_id = ? AND message_id = ? AND lifecycle_state = 'accepted' + WHERE session_id = ? AND message_id = ? AND lifecycle_state IN ('accepted', 'handed_off') `, ); for (const messageId of unique) { @@ -1914,12 +1914,14 @@ export class SqliteSessionMetadataStore { const unique = [...new Set(messageIds)]; for (const messageId of unique) assertSafeSessionId(messageId); this.transaction(() => { + const allowedPreviousStates = + state === 'handed_off' ? "lifecycle_state = 'accepted'" : "lifecycle_state = 'handed_off'"; const statement = this.db.prepare( ` UPDATE message_admissions SET lifecycle_state = ? WHERE session_id = ? AND message_id = ? - AND lifecycle_state IN ('accepted', 'handed_off') + AND ${allowedPreviousStates} `, ); for (const messageId of unique) { From 23ac0c5e1c71fb0f92584cde211091e8d504dcf3 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 00:06:21 +0800 Subject: [PATCH 11/22] fix(runtime-host): settle durable message proofs across recovery Generated-by: Codex --- .../canonical-session-projection.test.ts | 2 + .../__tests__/execution-host-message.test.ts | 1 + .../src/__tests__/goal-root-authority.test.ts | 2 + .../src/__tests__/message-coordinator.test.ts | 212 ++++++++++++++++++ .../__tests__/root-turn-coordinator.test.ts | 4 + .../src/server/message-coordinator.ts | 115 +++++++--- .../src/server/root-turn-coordinator.ts | 2 +- 7 files changed, 309 insertions(+), 29 deletions(-) diff --git a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts index bb7a568554..31278d18cd 100644 --- a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts +++ b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts @@ -543,8 +543,10 @@ function createMessages( stores.agentRunStore.readRootTurnSourceMessageReceipt(requestedSessionId, messageId), readImmutableSteeringMessageProof: (requestedSessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(requestedSessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission: new SessionAdmissionGate(), acquireResidency: () => ({ release: () => undefined }), preflightSessionSnapshot: () => true, diff --git a/packages/runtime-host/src/__tests__/execution-host-message.test.ts b/packages/runtime-host/src/__tests__/execution-host-message.test.ts index 8c75c21e5a..01ac3e4af5 100644 --- a/packages/runtime-host/src/__tests__/execution-host-message.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-message.test.ts @@ -195,6 +195,7 @@ test('steering becomes durable and ordered followups automatically start the nex await first.close(); await second.close(); await fixture.stopHost(host); + assert.equal(await fixture.readMessageLifecycleState(steeringId), 'handed_off'); const firstLedger = await fixture.readTurn(firstTurnId); const steeringEvents = firstLedger.runtimeEvents.filter( diff --git a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts index f5f74ebaa0..497a5566a5 100644 --- a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts +++ b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts @@ -575,8 +575,10 @@ async function createFixture(options: { recoverAdmissions?: boolean } = {}): Pro stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission: admission, acquireResidency, requestDrain: () => { diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index 62024d88d7..62c9dc7a83 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -22,8 +22,10 @@ import { test } from 'node:test'; import { messageContentDigest, type MessageContent } from '@maka/core/events'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { + MessageLifecycleStore, MessageOperationReceipt, MessageReceiptStore, + PendingMessageAdmission, RootTurnSourceMessageReceipt, } from '@maka/storage/execution-stores'; import { @@ -262,6 +264,77 @@ test('partitions a mixed-Client follow-up queue across root handoffs', async () await fixture.coordinator.close(); }); +test('recovered followups without a connection owner still form one successor batch', async () => { + const fixture = createFixture(); + await fixture.lifecycle.commitMessageAdmission({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + messageId: 'recovered-followup', + content: { text: 'recover without a connection owner' }, + modelContent: { text: 'recover without a connection owner' }, + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: 1, + }); + + await fixture.coordinator.recoverPendingAfterHostRestart([ROOT.sessionId]); + const owner = fixture.coordinator.bindRun(ROOT); + owner.release(); + const batch = fixture.coordinator.beginTerminalTransition(ROOT); + assert.deepEqual( + batch.sources.map((source) => source.messageId), + ['recovered-followup'], + ); + fixture.coordinator.commitNextRoot(batch, { + sessionId: ROOT.sessionId, + turnId: 'turn-recovered-successor', + runId: 'run-recovered-successor', + }); + const successor = fixture.coordinator.bindRun({ + sessionId: ROOT.sessionId, + turnId: 'turn-recovered-successor', + runId: 'run-recovered-successor', + }); + successor.release(); + fixture.coordinator.completeIdle( + fixture.coordinator.beginTerminalTransition({ + sessionId: ROOT.sessionId, + turnId: 'turn-recovered-successor', + runId: 'run-recovered-successor', + }), + ); +}); + +test('recovery treats a durable steering event as the handoff proof', async () => { + const fixture = createFixture(); + await fixture.lifecycle.commitMessageAdmission({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + messageId: 'recovered-steering', + content: { text: 'recover this steering event' }, + modelContent: { text: 'recover this steering event' }, + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt: 1, + }); + fixture.events.push(steeringEvent('recovered-steering', 'recover this steering event')); + + await fixture.coordinator.recoverPendingAfterHostRestart([ROOT.sessionId]); + + assert.equal(fixture.readMessageLifecycleState('recovered-steering'), 'handed_off'); + assert.deepEqual(fixture.coordinator.projection(ROOT.sessionId), { + hostEpoch: 'epoch-1', + queueRevision: 0, + steering: [], + followup: [], + }); + await fixture.coordinator.close(); +}); + test('binds the exact reserved Run after a pre-bind stop fence', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); @@ -794,6 +867,43 @@ test('entry promote moves a follow-up into the steering queue', async () => { assert.equal(fixture.liveResidencies(), 0); }); +test('editing a promoted entry preserves its original submitted placement', async () => { + const fixture = createFixture(); + fixture.coordinator.reserveRootTurn(ROOT); + + await submit(fixture, 'follow-1', 'first', 'next_turn'); + const promoted = await fixture.coordinator.handlers['queue.entry.promote']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + entryId: 'id-1', + promoteId: 'promote-edit', + }, + operationContext(), + ); + assert.equal(promoted.ok, true); + + const updated = await fixture.coordinator.handlers['queue.entry.update']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + entryId: 'id-1', + updateId: 'update-promoted', + expectedQueueRevision: 2, + text: 'edited after promotion', + }, + operationContext(), + ); + assert.equal(updated.ok, true); + const admission = fixture.readMessageAdmission('follow-1'); + assert.ok(admission); + assert.equal(admission.submittedPlacement, 'next_turn'); + assert.equal(admission.placement, 'current_turn'); + assert.equal(admission.disposition, 'steering'); + assert.deepEqual(admission.content, { text: 'edited after promotion' }); + assert.deepEqual(admission.modelContent, { text: 'edited after promotion' }); +}); + test('entry promote requires an active Turn', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); @@ -1561,6 +1671,37 @@ test('terminal transition atomically folds messages submitted after run release' fixture.coordinator.completeIdle(empty); }); +test('terminal settlement executes only steering admissions with a provider proof', async () => { + const fixture = createFixture(); + fixture.coordinator.reserveRootTurn(ROOT); + const owner = fixture.coordinator.bindRun(ROOT); + await submit(fixture, 'steer-proved', 'provider must see this', 'current_turn'); + const [lease] = owner.pull(); + assert.ok(lease); + owner.ack([lease.id]); + owner.release(); + fixture.events.push(steeringEvent('steer-proved', 'provider must see this')); + let providerProofAfter = -1; + fixture.setProviderRequestProof((admittedAt) => { + providerProofAfter = admittedAt; + return true; + }); + + await fixture.coordinator.settleMessagesAfterRoot({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + admittedAt: 0, + messageIds: [], + terminalStatus: 'completed', + }); + + assert.equal(fixture.readMessageLifecycleState('steer-proved'), 'executed'); + assert.equal(providerProofAfter, 1); + const batch = fixture.coordinator.beginTerminalTransition(ROOT); + fixture.coordinator.completeIdle(batch); +}); + test('administrative drain preserves accepted entries until the terminal stop fence', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); @@ -2091,6 +2232,7 @@ function createFixture( let drainRequests = 0; let receiptReads = 0; let rootReads = 0; + let providerRequestProof: boolean | ((admittedAt: number) => boolean) = false; let stopDeliveryError: Error | undefined; let prepareMessage: NonNullable = async (input) => ({ kind: 'ready', @@ -2106,6 +2248,13 @@ function createFixture( const receipts = new Map(); const events: RuntimeEvent[] = []; const operationReceipts = new Map(); + const messageAdmissions = new Map< + string, + { + admission: PendingMessageAdmission; + state: 'accepted' | 'handed_off' | 'executed' | 'cancelled'; + } + >(); const receiptDelays = new Map< string, { @@ -2114,6 +2263,7 @@ function createFixture( readonly error?: Error; } >(); + const lifecycle = memoryMessageLifecycleStore(messageAdmissions); const stopClaimed = deferred(); const terminal = deferred(); let coordinator: HostMessageCoordinator; @@ -2183,6 +2333,10 @@ function createFixture( ); return event ? { event } : undefined; }, + readProviderRequestProof: async ({ admittedAt }) => + typeof providerRequestProof === 'function' + ? providerRequestProof(admittedAt) + : providerRequestProof, }, receipts: memoryReceiptStore( operationReceipts, @@ -2198,6 +2352,7 @@ function createFixture( receiptReads += 1; }, ), + lifecycle, sessionAdmission: new SessionAdmissionGate(), acquireResidency: () => { liveResidencies += 1; @@ -2220,6 +2375,7 @@ function createFixture( coordinator = new HostMessageCoordinator(options); return { coordinator, + lifecycle, setRootState: (state: HostMessageRootState) => { rootState = state; }, @@ -2229,12 +2385,17 @@ function createFixture( startCalls: () => startCalls, events, receipts, + readMessageAdmission: (messageId: string) => messageAdmissions.get(messageId)?.admission, + readMessageLifecycleState: (messageId: string) => messageAdmissions.get(messageId)?.state, stopClaimed, resolveTerminal: terminal.resolve, liveResidencies: () => liveResidencies, drainRequests: () => drainRequests, receiptReads: () => receiptReads, rootReads: () => rootReads, + setProviderRequestProof: (proved: boolean | ((admittedAt: number) => boolean)) => { + providerRequestProof = proved; + }, failStopDelivery: (error: Error) => { stopDeliveryError = error; }, @@ -2280,6 +2441,57 @@ function memoryReceiptStore( }; } +function memoryMessageLifecycleStore( + admissions: Map< + string, + { + admission: PendingMessageAdmission; + state: 'accepted' | 'handed_off' | 'executed' | 'cancelled'; + } + >, +): MessageLifecycleStore { + return { + commitMessageAdmission: async (admission) => { + const existing = admissions.get(admission.messageId); + if (existing) return existing.admission; + admissions.set(admission.messageId, { admission, state: 'accepted' }); + return admission; + }, + readMessageAdmission: async (_sessionId, messageId) => admissions.get(messageId)?.admission, + readMessageLifecycleState: async (_sessionId, messageId) => admissions.get(messageId)?.state, + listMessageAdmissions: async (sessionId) => + [...admissions.values()] + .filter(({ admission }) => admission.sessionId === sessionId) + .map(({ admission }) => admission), + updateMessageAdmission: async (admission) => { + const existing = admissions.get(admission.messageId); + if (!existing) throw new Error(`Missing admission ${admission.messageId}`); + existing.admission = admission; + }, + reorderMessageAdmissions: async () => undefined, + cancelMessageAdmissions: async (_sessionId, messageIds) => { + for (const messageId of messageIds) { + const existing = admissions.get(messageId); + if (existing && (existing.state === 'accepted' || existing.state === 'handed_off')) { + existing.state = 'cancelled'; + } + } + }, + markMessagesHandedOff: async (_sessionId, messageIds) => { + for (const messageId of messageIds) { + const existing = admissions.get(messageId); + if (existing?.state === 'accepted') existing.state = 'handed_off'; + } + }, + markMessagesExecuted: async (_sessionId, messageIds) => { + for (const messageId of messageIds) { + const existing = admissions.get(messageId); + if (existing?.state === 'handed_off') existing.state = 'executed'; + } + }, + }; +} + function submit( fixture: ReturnType, messageId: string, diff --git a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts index aad3fdb039..4811893236 100644 --- a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts @@ -2175,8 +2175,10 @@ test('hosted linked child roots share admission, message, terminal, and stop aut stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission, acquireResidency, requestDrain: () => { @@ -4799,8 +4801,10 @@ async function createFailureFixture(options: { stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission, acquireResidency, requestDrain, diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index d9b28a5c44..30cbe679c6 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -176,7 +176,7 @@ export interface HostMessageDurableProofReader { messageId: string, ): Promise; /** True only when the admitted root has a durable downstream provider proof. */ - readProviderRequestProof?(input: { + readProviderRequestProof(input: { sessionId: string; turnId: string; runId: string; @@ -189,7 +189,7 @@ export interface HostMessageCoordinatorOptions { readonly root: HostMessageRootPort; readonly durableProof: HostMessageDurableProofReader; readonly receipts: MessageReceiptStore; - readonly lifecycle?: MessageLifecycleStore; + readonly lifecycle: MessageLifecycleStore; readonly sessionAdmission: SessionAdmissionGate; readonly acquireResidency: () => RuntimeHostResidency; readonly requestDrain?: () => void; @@ -336,7 +336,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { readonly #root: HostMessageRootPort; readonly #durableProof: HostMessageDurableProofReader; readonly #receipts: MessageReceiptStore; - readonly #lifecycle?: MessageLifecycleStore; + readonly #lifecycle: MessageLifecycleStore; readonly #sessionAdmission: SessionAdmissionGate; readonly #acquireResidency: () => RuntimeHostResidency; readonly #requestDrain: () => void; @@ -532,11 +532,11 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } async markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise { - await this.#lifecycle?.markMessagesHandedOff(sessionId, messageIds); + await this.#lifecycle.markMessagesHandedOff(sessionId, messageIds); } async markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise { - await this.#lifecycle?.markMessagesExecuted(sessionId, messageIds); + await this.#lifecycle.markMessagesExecuted(sessionId, messageIds); } /** @@ -552,13 +552,49 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { messageIds: readonly string[]; terminalStatus?: 'completed' | 'failed' | 'cancelled'; }): Promise { - if (!this.#lifecycle || input.messageIds.length === 0) return; - const proved = this.#durableProof.readProviderRequestProof - ? await this.#durableProof.readProviderRequestProof(input) - : false; - if (proved) { + const messageIds = new Set(input.messageIds); + const providerProofAfter = new Map(); + const admissions = await this.#lifecycle.listMessageAdmissions(input.sessionId); + for (const admission of admissions) { + if ( + admission.turnId !== input.turnId || + admission.runId !== input.runId || + admission.disposition !== 'steering' + ) { + continue; + } + const proof = await this.#durableProof.readImmutableSteeringMessageProof( + input.sessionId, + admission.messageId, + ); + if (proof?.event.turnId === input.turnId && proof.event.runId === input.runId) { + messageIds.add(admission.messageId); + providerProofAfter.set(admission.messageId, proof.event.ts); + } + } + const handedOff: string[] = []; + for (const messageId of messageIds) { + if ( + (await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId)) === 'accepted' + ) { + handedOff.push(messageId); + } + } + await this.#lifecycle.markMessagesHandedOff(input.sessionId, handedOff); + const proved: string[] = []; + for (const messageId of messageIds) { + if ( + await this.#durableProof.readProviderRequestProof({ + ...input, + admittedAt: providerProofAfter.get(messageId) ?? input.admittedAt, + }) + ) { + proved.push(messageId); + } + } + if (proved.length > 0) { const executed: string[] = []; - for (const messageId of input.messageIds) { + for (const messageId of proved) { if ( (await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId)) === 'handed_off' @@ -567,11 +603,10 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } } await this.#lifecycle.markMessagesExecuted(input.sessionId, executed); - return; } if (input.terminalStatus !== 'cancelled') return; const cancelled: string[] = []; - for (const messageId of input.messageIds) { + for (const messageId of messageIds) { const state = await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId); if (state === 'accepted' || state === 'handed_off') cancelled.push(messageId); } @@ -579,11 +614,10 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } async cancelMessages(sessionId: string, messageIds: readonly string[]): Promise { - await this.#lifecycle?.cancelMessageAdmissions(sessionId, messageIds); + await this.#lifecycle.cancelMessageAdmissions(sessionId, messageIds); } async recoverPendingAfterHostRestart(sessionIds: readonly string[]): Promise { - if (!this.#lifecycle) return; for (const sessionId of sessionIds) { const admissions = await this.#lifecycle.listMessageAdmissions(sessionId); if (admissions.length === 0) continue; @@ -604,7 +638,25 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { messageIds: [admission.messageId], }); } else { - pending.push(admission); + const steering = await this.#durableProof.readImmutableSteeringMessageProof( + sessionId, + admission.messageId, + ); + if ( + steering?.event.turnId === admission.turnId && + steering.event.runId === admission.runId + ) { + await this.#lifecycle.markMessagesHandedOff(sessionId, [admission.messageId]); + await this.settleMessagesAfterRoot({ + sessionId, + turnId: admission.turnId, + runId: admission.runId, + admittedAt: admission.admittedAt, + messageIds: [admission.messageId], + }); + } else { + pending.push(admission); + } } } if (pending.length === 0) continue; @@ -784,7 +836,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { placement: input.placement, disposition: 'turn_started', }; - const pendingAdmission = await this.#lifecycle?.readMessageAdmission( + const pendingAdmission = await this.#lifecycle.readMessageAdmission( input.sessionId, input.messageId, ); @@ -809,7 +861,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { disposition: 'steering', admittedAt: pendingAdmission?.admittedAt ?? Date.now(), }; - await this.#lifecycle?.commitMessageAdmission(messageAdmission); + await this.#lifecycle.commitMessageAdmission(messageAdmission); const started = await this.#root.startFromMessage( { sessionId: input.sessionId, @@ -822,7 +874,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { admission, ); if ('error' in started) { - await this.#lifecycle?.cancelMessageAdmissions(input.sessionId, [input.messageId]); + await this.#lifecycle.cancelMessageAdmissions(input.sessionId, [input.messageId]); return failure('operation_conflict', started.error); } if (!isEntityId(started.turnId)) { @@ -830,7 +882,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { 'Started Turn identity is not encodable', ); } - await this.#lifecycle?.markMessagesHandedOff(input.sessionId, [input.messageId]); + await this.#lifecycle.markMessagesHandedOff(input.sessionId, [input.messageId]); const result = { disposition: 'turn_started', turnId: started.turnId } as const; return success(result); } @@ -939,7 +991,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { disposition, admittedAt: Date.now(), }; - await this.#lifecycle?.commitMessageAdmission(messageAdmission); + await this.#lifecycle.commitMessageAdmission(messageAdmission); const residency = this.#acquireResidency(); const entry: LiveEntry = { entryId, @@ -1003,7 +1055,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { queueRevision: state.revision + (queued.length > 0 ? 1 : 0), retracted: queued.map(retractedSnapshot), }; - await this.#lifecycle?.cancelMessageAdmissions( + await this.#lifecycle.cancelMessageAdmissions( input.sessionId, queued.map((entry) => entry.messageId), ); @@ -1183,7 +1235,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } return failure('not_found', 'Message queue entry does not exist'); } - await this.#lifecycle?.cancelMessageAdmissions(input.sessionId, [queued.entry.messageId]); + await this.#lifecycle.cancelMessageAdmissions(input.sessionId, [queued.entry.messageId]); queued.remove(); this.#releaseEntry(queued.entry); this.#mutated(state); @@ -1237,7 +1289,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } return failure('not_found', 'Message queue entry does not exist'); } - await this.#lifecycle?.updateMessageAdmission({ + await this.#lifecycle.updateMessageAdmission({ sessionId: input.sessionId, turnId: entry.turnId, runId: entry.runId, @@ -1339,14 +1391,18 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { ) { return failure('session_busy', 'Message queue changed during update'); } - await this.#lifecycle?.updateMessageAdmission({ + const admission = await this.#lifecycle.readMessageAdmission( + input.sessionId, + queued.entry.messageId, + ); + await this.#lifecycle.updateMessageAdmission({ sessionId: input.sessionId, turnId: queued.entry.turnId, runId: queued.entry.runId, messageId: queued.entry.messageId, content, modelContent, - submittedPlacement: queued.entry.placement, + submittedPlacement: admission?.submittedPlacement ?? queued.entry.placement, placement: queued.entry.placement, disposition: queued.entry.disposition, admittedAt: queued.entry.admittedAt, @@ -1391,7 +1447,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { reordered.push(entry); } if (reordered.some((entry, index) => current[index] !== entry)) { - await this.#lifecycle?.reorderMessageAdmissions( + await this.#lifecycle.reorderMessageAdmissions( input.sessionId, reordered.map((entry) => entry.messageId), ); @@ -2231,7 +2287,10 @@ function canonicalFollowupBatch(entries: readonly LiveEntry[]): { function sameInitiatingClientPrefix(entries: readonly LiveEntry[]): LiveEntry[] { const initiatingConnectionId = entries[0]?.initiatingConnectionId; - if (!initiatingConnectionId) return []; + if (!initiatingConnectionId) { + const boundary = entries.findIndex((entry) => entry.initiatingConnectionId !== ''); + return entries.slice(0, boundary === -1 ? entries.length : boundary); + } const boundary = entries.findIndex( (entry) => entry.initiatingConnectionId !== initiatingConnectionId, ); diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 70a718751f..9f380c3988 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -2331,7 +2331,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { active.sessionId, active.turnId, ); - if (!admission || admission.sourceMessages.length === 0) return; + if (!admission) return; await this.messages.settleMessagesAfterRoot({ sessionId: active.sessionId, turnId: active.turnId, From e1ea4205920b2d15f3a09a6d397d77db2d1dede5 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 01:26:22 +0800 Subject: [PATCH 12/22] fix(runtime-host): replay admitted roots from durable contracts Generated-by: Codex --- .../__tests__/execution-host-queue.test.ts | 23 ++++++++ .../fixtures/execution-host-suite.ts | 59 ++++++++++++++++++- .../src/server/hosted-execution-recovery.ts | 13 +++- 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index fb0cc029ba..fe60562855 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -298,6 +298,29 @@ test('a Host crash after queue admission recovers the durable successor once', a }); }); +test('restart replays an atomically admitted root without duplicating its transcript', async () => { + await withExecutionRoot(async (fixture) => { + const turnId = randomUUID(); + const messageId = randomUUID(); + const content = { text: 'recover the root after admission before Run creation' }; + await fixture.seedAtomicRootAdmissionWithoutRun({ turnId, messageId, content }); + + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const terminal = await waitForTerminalTurn(client, fixture.sessionId, turnId); + assert.equal(terminal.status, 'completed'); + await client.close(); + await fixture.stopHost(host); + + const ledger = await fixture.readTurn(turnId); + assert.deepEqual( + ledger.userMessages.filter((message) => message.id === messageId).map((message) => message.id), + [messageId], + ); + assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); + }); +}); + test('concurrent root admission for one Session has a single winner', async () => { await withExecutionRoot(async (fixture) => { const host = await fixture.startHost(); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index 23d8b942a3..cc2c70ad3d 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -38,7 +38,11 @@ import { test } from 'node:test'; import { TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import type { AgentRunHeader } from '@maka/core/agent-run'; -import { normalizeMessageContent, type MessageContent } from '@maka/core/events'; +import { + messageContentDigest, + normalizeMessageContent, + type MessageContent, +} from '@maka/core/events'; import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; import type { StoredMessage } from '@maka/core/session'; import type { Task } from '@maka/core/task-ledger'; @@ -672,6 +676,59 @@ export class ExecutionFixture { return this.seedTurnState(turnId, content, false, false); } + async seedAtomicRootAdmissionWithoutRun(input: { + turnId: string; + messageId: string; + content: MessageContent; + }): Promise { + const owner = await tryAcquireInteractiveRootOwner(this.capability); + assert.ok(owner); + if (!owner) throw new Error('Unable to acquire execution root for atomic root setup'); + let stores: Awaited> | undefined; + try { + stores = await openInteractiveExecutionStoresForWrite(owner.lease); + const admittedAt = Date.now(); + const content = normalizeMessageContent(input.content); + const contentDigest = messageContentDigest(content); + const runId = randomUUID(); + await stores.sessionStore.commitMessageAdmission({ + sessionId: this.sessionId, + turnId: input.turnId, + runId, + messageId: input.messageId, + content, + modelContent: content, + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt, + }); + const result = await stores.agentRunStore.admitRootTurn({ + sessionId: this.sessionId, + turnId: input.turnId, + proposedRunId: runId, + proposedUserMessageId: input.messageId, + execution: { kind: 'external_message', inputDigest: contentDigest }, + previousRootTurnId: null, + normalizedInput: content, + sourceMessages: [ + { + messageId: input.messageId, + content, + submittedContentDigest: contentDigest, + placement: 'current_turn', + disposition: 'turn_started', + }, + ], + admittedAt, + }); + assert.equal(result.kind, 'admitted'); + } finally { + await stores?.sessionStore.close?.(); + await owner.close(); + } + } + async archiveSession(): Promise { const owner = await tryAcquireInteractiveRootOwner(this.capability); assert.ok(owner); diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 674c92e2d2..032f745ff2 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -145,7 +145,16 @@ export async function prepareHostedExecutionRecovery( continue; } if (!run) { - if (rootUserMessages.length > 0 || messageIdOwner) { + if (executionContract.pendingWithoutRun === 'root_replay') { + verifyOrRecoverUserMessage( + admission, + rootUserMessages, + messageIdOwner, + missingMessages, + messageIndex, + false, + ); + } else if (rootUserMessages.length > 0 || messageIdOwner) { throw new Error(`Admitted Turn ${admission.turnId} has a UserMessage but no Run`); } replayAdmissions.push(admission); @@ -263,6 +272,7 @@ function verifyOrRecoverUserMessage( messageIdOwner: StoredMessage | undefined, missingMessages: RecoveryUserMessage[], index: RecoveryMessageIndex, + materializeMissing = true, ): void { if (rootUserMessages.length > 1) { throw new Error(`Admitted Turn ${admission.turnId} has multiple UserMessages`); @@ -285,6 +295,7 @@ function verifyOrRecoverUserMessage( if (messageIdOwner) { throw new Error(`Admitted Turn ${admission.turnId} reuses another message identity`); } + if (!materializeMissing) return; const recoveredMessage = recoveryUserMessage(admission); missingMessages.push(recoveredMessage); indexRecoveryMessage(index, recoveredMessage); From 17500194c7253c87ca0a2057d1fca31c9d2aff58 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 01:30:35 +0800 Subject: [PATCH 13/22] fix(runtime-host): own durable message handoff transitions Generated-by: Codex --- .../src/__tests__/message-coordinator.test.ts | 13 +++- .../src/server/message-coordinator.ts | 78 +++++++++++++++---- .../src/server/root-turn-coordinator.ts | 32 ++++++-- .../sqlite-session-metadata-store.test.ts | 10 +++ packages/storage/src/execution-stores.ts | 2 + packages/storage/src/message-receipt-store.ts | 1 + packages/storage/src/session-store.ts | 7 ++ .../src/sqlite-session-metadata-store.ts | 21 +++++ 8 files changed, 139 insertions(+), 25 deletions(-) diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index 62c9dc7a83..6982e9e069 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -2461,7 +2461,18 @@ function memoryMessageLifecycleStore( readMessageLifecycleState: async (_sessionId, messageId) => admissions.get(messageId)?.state, listMessageAdmissions: async (sessionId) => [...admissions.values()] - .filter(({ admission }) => admission.sessionId === sessionId) + .filter( + ({ admission, state }) => + admission.sessionId === sessionId && state === 'accepted', + ) + .map(({ admission }) => admission), + listUnsettledMessageAdmissions: async (sessionId) => + [...admissions.values()] + .filter( + ({ admission, state }) => + admission.sessionId === sessionId && + (state === 'accepted' || state === 'handed_off'), + ) .map(({ admission }) => admission), updateMessageAdmission: async (admission) => { const existing = admissions.get(admission.messageId); diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index 30cbe679c6..31299af8f1 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -531,12 +531,42 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { this.#draining = true; } - async markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise { - await this.#lifecycle.markMessagesHandedOff(sessionId, messageIds); - } - - async markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise { - await this.#lifecycle.markMessagesExecuted(sessionId, messageIds); + /** + * Commit the root-admission proof before Runtime activation. The in-memory + * queue never owns this transition: it only projects the durable result. + */ + async handoffRootSources(input: { + sessionId: string; + turnId: string; + runId: string; + messageIds: readonly string[]; + }): Promise { + const handoff: string[] = []; + for (const messageId of new Set(input.messageIds)) { + const proof = await this.#durableProof.readRootTurnSourceMessageReceipt( + input.sessionId, + messageId, + ); + if ( + !proof || + proof.admission.turnId !== input.turnId || + proof.admission.runId !== input.runId || + proof.sourceMessage.messageId !== messageId + ) { + throw new RuntimeMessageAuthorityInvariantError( + `Root admission does not prove Message handoff ${messageId}`, + ); + } + const state = await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId); + if (state === 'accepted') handoff.push(messageId); + else if (state === 'handed_off' || state === 'executed') continue; + else { + throw new RuntimeMessageAuthorityInvariantError( + `Message ${messageId} cannot be handed off from lifecycle state ${state ?? 'missing'}`, + ); + } + } + await this.#lifecycle.markMessagesHandedOff(input.sessionId, handoff); } /** @@ -552,9 +582,22 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { messageIds: readonly string[]; terminalStatus?: 'completed' | 'failed' | 'cancelled'; }): Promise { - const messageIds = new Set(input.messageIds); + const messageIds = new Set(); const providerProofAfter = new Map(); - const admissions = await this.#lifecycle.listMessageAdmissions(input.sessionId); + const admissions = await this.#lifecycle.listUnsettledMessageAdmissions(input.sessionId); + for (const messageId of new Set(input.messageIds)) { + const proof = await this.#durableProof.readRootTurnSourceMessageReceipt( + input.sessionId, + messageId, + ); + if ( + proof?.admission.turnId === input.turnId && + proof.admission.runId === input.runId && + proof.sourceMessage.messageId === messageId + ) { + messageIds.add(messageId); + } + } for (const admission of admissions) { if ( admission.turnId !== input.turnId || @@ -620,16 +663,20 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { async recoverPendingAfterHostRestart(sessionIds: readonly string[]): Promise { for (const sessionId of sessionIds) { const admissions = await this.#lifecycle.listMessageAdmissions(sessionId); - if (admissions.length === 0) continue; - const rootState = await this.#root.readRootState(sessionId); + const unsettled = await this.#lifecycle.listUnsettledMessageAdmissions(sessionId); + if (unsettled.length === 0) continue; + const acceptedIds = new Set(admissions.map((admission) => admission.messageId)); const pending = [] as PendingMessageAdmission[]; - for (const admission of admissions) { + for (const admission of unsettled) { const source = await this.#durableProof.readRootTurnSourceMessageReceipt( sessionId, admission.messageId, ); - if (source) { - await this.#lifecycle.markMessagesHandedOff(sessionId, [admission.messageId]); + if ( + source?.admission.turnId === admission.turnId && + source.admission.runId === admission.runId && + source.sourceMessage.messageId === admission.messageId + ) { await this.settleMessagesAfterRoot({ sessionId, turnId: source.admission.turnId, @@ -646,7 +693,6 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { steering?.event.turnId === admission.turnId && steering.event.runId === admission.runId ) { - await this.#lifecycle.markMessagesHandedOff(sessionId, [admission.messageId]); await this.settleMessagesAfterRoot({ sessionId, turnId: admission.turnId, @@ -654,12 +700,13 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { admittedAt: admission.admittedAt, messageIds: [admission.messageId], }); - } else { + } else if (acceptedIds.has(admission.messageId)) { pending.push(admission); } } } if (pending.length === 0) continue; + const rootState = await this.#root.readRootState(sessionId); if (rootState.kind !== 'active') { if (rootState.kind !== 'idle') continue; if (!this.#root.startRecoveredMessages) { @@ -882,7 +929,6 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { 'Started Turn identity is not encodable', ); } - await this.#lifecycle.markMessagesHandedOff(input.sessionId, [input.messageId]); const result = { disposition: 'turn_started', turnId: started.turnId } as const; return success(result); } diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 9f380c3988..6bb9241ee1 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -409,6 +409,12 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { `Unable to recover admitted Turn ${admission.turnId}: ${continuation.plan.reason}`, ); } + await this.messages.handoffRootSources({ + sessionId, + turnId: admission.turnId, + runId: admission.runId, + messageIds: admission.sourceMessages.map((source) => source.messageId), + }); return this.prepareAdmittedTurn( input, admission, @@ -1074,6 +1080,12 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { 'Fresh Message root Turn identity already existed', ); } + await this.messages.handoffRootSources({ + sessionId: input.sessionId, + turnId, + runId, + messageIds: [input.sourceMessage.messageId], + }); const disposition = await this.prepareAdmittedTurn( { sessionId: input.sessionId, @@ -1130,6 +1142,12 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (admitted.kind !== 'admitted') { return { error: 'Recovered Message root identity already existed' }; } + await this.messages.handoffRootSources({ + sessionId: input.sessionId, + turnId, + runId: admitted.admission.runId, + messageIds: input.sources.map((source) => source.messageId), + }); const disposition = await this.prepareAdmittedTurn( { sessionId: input.sessionId, turnId, content: input.content }, admitted.admission, @@ -1142,10 +1160,6 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (disposition.kind !== 'await_start') { return { error: 'Recovered Message root did not reserve execution' }; } - await this.messages.markMessagesHandedOff( - input.sessionId, - input.sources.map((source) => source.messageId), - ); return { turnId }; } catch (error) { this.#admissions.release(reservation); @@ -2431,10 +2445,12 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { 'Fresh follow-up root Turn identity already existed', ); } - await this.messages.markMessagesHandedOff( - batch.sessionId, - batch.sources.map((source) => source.messageId), - ); + await this.messages.handoffRootSources({ + sessionId: batch.sessionId, + turnId, + runId: admitted.admission.runId, + messageIds: batch.sources.map((source) => source.messageId), + }); const nextIdentity = { sessionId: batch.sessionId, diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 7e9e6a3c83..6b61abc965 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -284,10 +284,20 @@ describe('SqliteSessionMetadataStore', () => { ], ); assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'accepted'); + assert.deepEqual( + (await store.listMessageAdmissions('session-1')).map((entry) => entry.messageId), + ['message-1'], + ); await store.markMessagesHandedOff('session-1', ['message-1']); assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'handed_off'); + assert.deepEqual(await store.listMessageAdmissions('session-1'), []); + assert.deepEqual( + (await store.listUnsettledMessageAdmissions('session-1')).map((entry) => entry.messageId), + ['message-1'], + ); await store.markMessagesExecuted('session-1', ['message-1']); assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'executed'); + assert.deepEqual(await store.listUnsettledMessageAdmissions('session-1'), []); } finally { store.close(); } diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 6beccbc293..e01a5ec5d4 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -436,6 +436,8 @@ async function createExecutionStoresForWrite sessionStore.readMessageLifecycleState(sessionId, messageId)), listMessageAdmissions: (sessionId) => run(() => sessionStore.listMessageAdmissions(sessionId)), + listUnsettledMessageAdmissions: (sessionId) => + run(() => sessionStore.listUnsettledMessageAdmissions(sessionId)), updateMessageAdmission: (admission) => run(() => sessionStore.updateMessageAdmission(admission)), reorderMessageAdmissions: (sessionId, messageIds) => diff --git a/packages/storage/src/message-receipt-store.ts b/packages/storage/src/message-receipt-store.ts index d0b5c40a11..cb8de52325 100644 --- a/packages/storage/src/message-receipt-store.ts +++ b/packages/storage/src/message-receipt-store.ts @@ -56,6 +56,7 @@ export interface MessageLifecycleStore { messageId: string, ): Promise; listMessageAdmissions(sessionId: string): Promise; + listUnsettledMessageAdmissions(sessionId: string): Promise; updateMessageAdmission(admission: PendingMessageAdmission): Promise; reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 7420840053..c968312b7d 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -880,6 +880,13 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.metadata.listMessageAdmissions(sessionId); } + async listUnsettledMessageAdmissions( + sessionId: string, + ): Promise { + await this.ensureReady(); + return this.metadata.listUnsettledMessageAdmissions(sessionId); + } + async readMessageLifecycleState( sessionId: string, messageId: string, diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 1ecf89751d..1ed55331d5 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1692,6 +1692,27 @@ export class SqliteSessionMetadataStore { }); } + async listUnsettledMessageAdmissions( + sessionId: string, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + return this.readTransaction(() => { + const rows = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, model_content_json, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND lifecycle_state IN ('accepted', 'handed_off') + ORDER BY queue_order, sequence + `, + ) + .all(sessionId) as MessageAdmissionRow[]; + return rows.map((row) => decodeMessageAdmissionRow(sessionId, row).admission); + }); + } + async readMessageLifecycleState( sessionId: string, messageId: string, From 65056c453e1d53dcf8bebd63e1e53b54c1889c98 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 01:36:27 +0800 Subject: [PATCH 14/22] fix(runtime-host): keep one canonical follow-up transcript Generated-by: Codex --- .../__tests__/execution-host-message.test.ts | 18 +++- .../fixtures/execution-host-suite.ts | 14 +++ .../src/__tests__/message-coordinator.test.ts | 1 + .../src/server/message-coordinator.ts | 7 ++ .../src/server/root-turn-coordinator.ts | 8 +- .../sqlite-session-metadata-store.test.ts | 19 ++++ packages/storage/src/execution-stores.ts | 2 + packages/storage/src/message-receipt-store.ts | 6 ++ packages/storage/src/session-store.ts | 11 +++ .../src/sqlite-session-metadata-store.ts | 92 +++++++++++++++++++ 10 files changed, 173 insertions(+), 5 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-message.test.ts b/packages/runtime-host/src/__tests__/execution-host-message.test.ts index 01ac3e4af5..61c1ee357e 100644 --- a/packages/runtime-host/src/__tests__/execution-host-message.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-message.test.ts @@ -237,9 +237,23 @@ test('steering becomes durable and ordered followups automatically start the nex assert.ok(followupTurnId); const followupLedger = await fixture.readTurn(followupTurnId); const expectedQuotes = followupSources.flatMap((source) => source.content.quotes ?? []); - assert.equal(followupLedger.userMessages.length, 1); - assert.deepEqual(followupLedger.userMessages[0]?.quotes, expectedQuotes); + assert.equal(followupLedger.userMessages.length, followupSources.length); + assert.deepEqual( + followupLedger.userMessages.flatMap((message) => message.quotes ?? []), + expectedQuotes, + ); assert.deepEqual(userRuntimeContent(followupLedger.runtimeEvents)?.quotes, expectedQuotes); + const sessionUserMessages = await fixture.readSessionUserMessages(); + for (const source of followupSources) { + assert.equal( + sessionUserMessages.filter((message) => message.id === source.messageId).length, + 1, + ); + } + assert.equal( + sessionUserMessages.filter((message) => message.turnId === followupTurnId).length, + followupSources.length, + ); }); }); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index cc2c70ad3d..324c89d478 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -1000,6 +1000,20 @@ export class ExecutionFixture { } } + async readSessionUserMessages(): Promise>> { + const reader = await acquireReader(this.capability); + let stores: Awaited> | undefined; + try { + stores = await openInteractiveExecutionStoresForRead(reader.lease); + return (await stores.sessionStore.readMessages(this.sessionId)).filter( + (message): message is Extract => message.type === 'user', + ); + } finally { + await stores?.sessionStore.close?.(); + await reader.close(); + } + } + async readAdmissionChain() { const owner = await tryAcquireInteractiveRootOwner(this.capability); assert.ok(owner); diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index 6982e9e069..b3a39b368e 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -2474,6 +2474,7 @@ function memoryMessageLifecycleStore( (state === 'accepted' || state === 'handed_off'), ) .map(({ admission }) => admission), + rebindMessageAdmissionTranscript: async () => undefined, updateMessageAdmission: async (admission) => { const existing = admissions.get(admission.messageId); if (!existing) throw new Error(`Missing admission ${admission.messageId}`); diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index 31299af8f1..f9db15b100 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -539,6 +539,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { sessionId: string; turnId: string; runId: string; + previousRootTurnId: string | null; messageIds: readonly string[]; }): Promise { const handoff: string[] = []; @@ -566,6 +567,12 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { ); } } + await this.#lifecycle.rebindMessageAdmissionTranscript({ + sessionId: input.sessionId, + messageIds: [...new Set(input.messageIds)], + turnId: input.turnId, + previousRootTurnId: input.previousRootTurnId, + }); await this.#lifecycle.markMessagesHandedOff(input.sessionId, handoff); } diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 6bb9241ee1..c2614ad3c1 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -413,6 +413,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId, turnId: admission.turnId, runId: admission.runId, + previousRootTurnId: admission.previousRootTurnId, messageIds: admission.sourceMessages.map((source) => source.messageId), }); return this.prepareAdmittedTurn( @@ -1084,6 +1085,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId: input.sessionId, turnId, runId, + previousRootTurnId: admitted.admission.previousRootTurnId, messageIds: [input.sourceMessage.messageId], }); const disposition = await this.prepareAdmittedTurn( @@ -1146,6 +1148,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId: input.sessionId, turnId, runId: admitted.admission.runId, + previousRootTurnId: admitted.admission.previousRootTurnId, messageIds: input.sources.map((source) => source.messageId), }); const disposition = await this.prepareAdmittedTurn( @@ -2009,9 +2012,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { userMessageId: admission.userMessageId, execution: admission.execution, }); - const initialUserMessagesMaterialized = - admission.sourceMessages.length > 0 && - admission.sourceMessages.every((source) => source.disposition === 'turn_started'); + const initialUserMessagesMaterialized = admission.sourceMessages.length > 0; if (initialUserMessagesMaterialized) { await this.manager.materializeRootSourceMessages({ sessionId: input.sessionId, @@ -2449,6 +2450,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId: batch.sessionId, turnId, runId: admitted.admission.runId, + previousRootTurnId: admitted.admission.previousRootTurnId, messageIds: batch.sources.map((source) => source.messageId), }); diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 6b61abc965..e15b352ebe 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -327,6 +327,25 @@ describe('SqliteSessionMetadataStore', () => { })), [{ id: 'message-followup', turnId: 'turn-current' }], ); + await store.rebindMessageAdmissionTranscript({ + sessionId: 'session-followup-admission', + messageIds: ['message-followup'], + turnId: 'turn-successor', + previousRootTurnId: 'turn-current', + }); + await store.rebindMessageAdmissionTranscript({ + sessionId: 'session-followup-admission', + messageIds: ['message-followup'], + turnId: 'turn-successor', + previousRootTurnId: 'turn-current', + }); + assert.deepEqual( + (await store.readMessages('session-followup-admission')).map((message) => ({ + id: message.id, + turnId: message.turnId, + })), + [{ id: 'message-followup', turnId: 'turn-successor' }], + ); } finally { store.close(); } diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index e01a5ec5d4..fcbae1a961 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -438,6 +438,8 @@ async function createExecutionStoresForWrite sessionStore.listMessageAdmissions(sessionId)), listUnsettledMessageAdmissions: (sessionId) => run(() => sessionStore.listUnsettledMessageAdmissions(sessionId)), + rebindMessageAdmissionTranscript: (input) => + run(() => sessionStore.rebindMessageAdmissionTranscript(input)), updateMessageAdmission: (admission) => run(() => sessionStore.updateMessageAdmission(admission)), reorderMessageAdmissions: (sessionId, messageIds) => diff --git a/packages/storage/src/message-receipt-store.ts b/packages/storage/src/message-receipt-store.ts index cb8de52325..42cd1375fe 100644 --- a/packages/storage/src/message-receipt-store.ts +++ b/packages/storage/src/message-receipt-store.ts @@ -57,6 +57,12 @@ export interface MessageLifecycleStore { ): Promise; listMessageAdmissions(sessionId: string): Promise; listUnsettledMessageAdmissions(sessionId: string): Promise; + rebindMessageAdmissionTranscript(input: { + sessionId: string; + messageIds: readonly string[]; + turnId: string; + previousRootTurnId: string | null; + }): Promise; updateMessageAdmission(admission: PendingMessageAdmission): Promise; reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index c968312b7d..c6bc1a59f8 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -887,6 +887,17 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.metadata.listUnsettledMessageAdmissions(sessionId); } + async rebindMessageAdmissionTranscript(input: { + sessionId: string; + messageIds: readonly string[]; + turnId: string; + previousRootTurnId: string | null; + }): Promise { + await this.ensureReady(); + await this.metadata.rebindMessageAdmissionTranscript(input); + for (const listener of this.transcriptChangeListeners) listener(input.sessionId); + } + async readMessageLifecycleState( sessionId: string, messageId: string, diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 1ed55331d5..d2370a01d2 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -102,6 +102,7 @@ import { type MessageLifecycleState, type PendingMessageAdmission, } from './message-receipt-store.js'; +import { messageContentsEqual, normalizeMessageContent } from '@maka/core/events'; import { type AgentGraphIntentAdmissionSnapshot, type AgentGraphTimelineMetadataSnapshot, @@ -1743,6 +1744,97 @@ export class SqliteSessionMetadataStore { }); } + async rebindMessageAdmissionTranscript(input: { + sessionId: string; + messageIds: readonly string[]; + turnId: string; + previousRootTurnId: string | null; + }): Promise { + this.assertOpen(); + assertSafeSessionId(input.sessionId); + assertSafeSessionId(input.turnId); + if (input.previousRootTurnId !== null) { + assertSafeSessionId(input.previousRootTurnId); + } + const unique = [...new Set(input.messageIds)]; + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + for (const messageId of unique) { + const admissionRow = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, model_content_json, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(input.sessionId, messageId) as MessageAdmissionRow | undefined; + if (!admissionRow) { + throw new SessionMetadataConflictError('Message admission does not exist'); + } + const admission = decodeMessageAdmissionRow(input.sessionId, admissionRow); + if ( + admission.lifecycleState !== 'accepted' && + admission.lifecycleState !== 'handed_off' && + admission.lifecycleState !== 'executed' + ) { + throw new SessionMetadataConflictError('Message admission is already cancelled'); + } + const rows = this.db + .prepare( + ` + SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 + FROM session_messages AS message + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE message.session_id = ? AND message.message_id = ? + `, + ) + .all(input.sessionId, messageId) as Array<{ + sequence?: unknown; + record_json?: unknown; + record_bytes?: unknown; + sha256?: unknown; + }>; + if (rows.length !== 1) { + throw new SessionMetadataConflictError( + rows.length === 0 + ? 'Message admission transcript is missing' + : 'Message admission transcript identity is ambiguous', + ); + } + const sequence = rows[0]?.sequence; + if (typeof sequence !== 'number' || !Number.isSafeInteger(sequence)) { + throw new SessionMetadataConflictError('Invalid Message transcript sequence'); + } + const row = rows[0]!; + const recordJson = readStoredMessageRecordJson(this.db, input.sessionId, sequence, row); + const message = decodeStoredMessage(JSON.parse(recordJson) as unknown); + if ( + message.type !== 'user' || + message.id !== messageId || + !messageContentsEqual(normalizeMessageContent(message), admission.admission.content) + ) { + throw new SessionMetadataConflictError('Message admission transcript identity conflict'); + } + if (message.turnId === input.turnId) continue; + if ( + input.previousRootTurnId === null || + message.turnId !== input.previousRootTurnId + ) { + throw new SessionMetadataConflictError('Message admission transcript Turn conflict'); + } + const rebound = decodeCanonicalMessage({ ...message, turnId: input.turnId }); + this.db + .prepare( + 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', + ) + .run(JSON.stringify(rebound), input.sessionId, sequence); + } + }); + } + async updateMessageAdmission(admission: PendingMessageAdmission): Promise { this.assertOpen(); const stored = normalizePendingMessageAdmission(admission); From 1fbeec336bbcc517f2d81c9bac1d7795e8fd7787 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 01:38:33 +0800 Subject: [PATCH 15/22] fix(storage): preserve transcript chunks during rebinding Generated-by: Codex --- .../src/sqlite-session-metadata-store.ts | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index d2370a01d2..d66c948d8d 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1826,11 +1826,7 @@ export class SqliteSessionMetadataStore { throw new SessionMetadataConflictError('Message admission transcript Turn conflict'); } const rebound = decodeCanonicalMessage({ ...message, turnId: input.turnId }); - this.db - .prepare( - 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', - ) - .run(JSON.stringify(rebound), input.sessionId, sequence); + this.replaceSessionMessageSync(input.sessionId, sequence, rebound); } }); } @@ -1922,11 +1918,7 @@ export class SqliteSessionMetadataStore { if (typeof sequence !== 'number' || !Number.isSafeInteger(sequence)) { throw new SessionMetadataConflictError('Invalid Message transcript sequence'); } - this.db - .prepare( - 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', - ) - .run(json, stored.sessionId, sequence); + this.replaceSessionMessageSync(stored.sessionId, sequence, message, json); } this.updateCatalogProjectionSync( stored.sessionId, @@ -4765,6 +4757,62 @@ export class SqliteSessionMetadataStore { } } + private replaceSessionMessageSync( + sessionId: string, + sequence: number, + message: StoredMessage, + json = JSON.stringify(message), + ): void { + const encoded = Buffer.from(json, 'utf8'); + this.db + .prepare('DELETE FROM session_message_chunks WHERE session_id = ? AND sequence = ?') + .run(sessionId, sequence); + this.db + .prepare('DELETE FROM session_message_payloads WHERE session_id = ? AND sequence = ?') + .run(sessionId, sequence); + if (encoded.byteLength <= SQLITE_SESSION_MESSAGE_CHUNK_BYTES) { + this.db + .prepare( + 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', + ) + .run(json, sessionId, sequence); + return; + } + this.db + .prepare( + 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', + ) + .run(SQLITE_SESSION_MESSAGE_CHUNK_MARKER, sessionId, sequence); + this.db + .prepare( + 'INSERT INTO session_message_payloads(session_id, sequence, record_bytes, sha256) VALUES (?, ?, ?, ?)', + ) + .run( + sessionId, + sequence, + encoded.byteLength, + createHash('sha256').update(encoded).digest('hex'), + ); + for ( + let offset = 0; + offset < encoded.byteLength; + offset += SQLITE_SESSION_MESSAGE_CHUNK_BYTES + ) { + const chunk = encoded.subarray(offset, offset + SQLITE_SESSION_MESSAGE_CHUNK_BYTES); + this.db + .prepare( + 'INSERT INTO session_message_chunks(session_id, sequence, chunk_index, data, sha256) VALUES (?, ?, ?, ?, ?)', + ) + .run( + sessionId, + sequence, + offset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES, + chunk, + createHash('sha256').update(chunk).digest('hex'), + ); + } + } + private readMessagesWith( sessionId: string, decode: (value: unknown) => StoredMessage, From fe40f9b828e1040229d1fd292cd831c28cc4d2b0 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 01:39:53 +0800 Subject: [PATCH 16/22] refactor(runtime): remove previous-root transcript fallback Generated-by: Codex --- packages/runtime-host/src/server/root-turn-coordinator.ts | 1 - .../src/__tests__/runtime-kernel-interaction.test.ts | 1 - packages/runtime/src/runtime-kernel.ts | 6 +----- packages/runtime/src/session-manager.ts | 1 - 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index c2614ad3c1..0fceafc755 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -2017,7 +2017,6 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { await this.manager.materializeRootSourceMessages({ sessionId: input.sessionId, turnId: input.turnId, - previousRootTurnId: admission.previousRootTurnId, messages: admission.sourceMessages.map((source) => ({ messageId: source.messageId, content: source.content, diff --git a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts index df2d5ed932..8f65b55009 100644 --- a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts +++ b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts @@ -59,7 +59,6 @@ describe('RuntimeKernel Interaction close cleanup', () => { await kernel.materializeRootSourceMessages({ sessionId: SESSION_ID, turnId: 'prepared-turn', - previousRootTurnId: null, messages: [ { messageId: 'submitted-message', diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 25037847e7..c388139360 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -191,7 +191,6 @@ export interface RuntimeKernelLike { materializeRootSourceMessages?(input: { sessionId: string; turnId: string; - previousRootTurnId: string | null; messages: readonly { messageId: string; content: MessageContent; @@ -2414,7 +2413,6 @@ export class RuntimeKernel implements RuntimeKernelLike { async materializeRootSourceMessages(input: { sessionId: string; turnId: string; - previousRootTurnId: string | null; messages: readonly { messageId: string; content: MessageContent; @@ -2434,9 +2432,7 @@ export class RuntimeKernel implements RuntimeKernelLike { (message.submittedContentDigest === undefined || messageContentDigest(normalizeMessageContent(existing)) !== message.submittedContentDigest)) || - (existing.turnId !== input.turnId && - ((message.disposition !== 'steering' && message.disposition !== 'followup') || - existing.turnId !== input.previousRootTurnId)) + existing.turnId !== input.turnId ) { throw new Error(`Queued root source ${message.messageId} conflicts with its transcript`); } diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 4c47734384..b66ceb76eb 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -4823,7 +4823,6 @@ export class SessionManager { materializeRootSourceMessages(input: { sessionId: string; turnId: string; - previousRootTurnId: string | null; messages: readonly { messageId: string; content: import('@maka/core/events').MessageContent; From bdebf38f4894cd4fff235bbfec36e2297bf493a2 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 01:46:32 +0800 Subject: [PATCH 17/22] fix(storage): allow delayed follow-up transcript handoff Generated-by: Codex --- packages/storage/src/sqlite-session-metadata-store.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index d66c948d8d..e5a4e5c0b9 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1820,8 +1820,8 @@ export class SqliteSessionMetadataStore { } if (message.turnId === input.turnId) continue; if ( - input.previousRootTurnId === null || - message.turnId !== input.previousRootTurnId + message.turnId !== admission.admission.turnId && + (input.previousRootTurnId === null || message.turnId !== input.previousRootTurnId) ) { throw new SessionMetadataConflictError('Message admission transcript Turn conflict'); } From 5eea09f22d6414eb20e7a3aeb7a177e2c04b1c12 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 01:49:59 +0800 Subject: [PATCH 18/22] chore: format durable lifecycle changes Generated-by: Codex --- .../src/__tests__/execution-host-queue.test.ts | 4 +++- .../src/__tests__/message-coordinator.test.ts | 8 ++------ packages/storage/src/sqlite-session-metadata-store.ts | 4 +--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index fe60562855..1f911bce98 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -314,7 +314,9 @@ test('restart replays an atomically admitted root without duplicating its transc const ledger = await fixture.readTurn(turnId); assert.deepEqual( - ledger.userMessages.filter((message) => message.id === messageId).map((message) => message.id), + ledger.userMessages + .filter((message) => message.id === messageId) + .map((message) => message.id), [messageId], ); assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index b3a39b368e..164e3d8108 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -2461,17 +2461,13 @@ function memoryMessageLifecycleStore( readMessageLifecycleState: async (_sessionId, messageId) => admissions.get(messageId)?.state, listMessageAdmissions: async (sessionId) => [...admissions.values()] - .filter( - ({ admission, state }) => - admission.sessionId === sessionId && state === 'accepted', - ) + .filter(({ admission, state }) => admission.sessionId === sessionId && state === 'accepted') .map(({ admission }) => admission), listUnsettledMessageAdmissions: async (sessionId) => [...admissions.values()] .filter( ({ admission, state }) => - admission.sessionId === sessionId && - (state === 'accepted' || state === 'handed_off'), + admission.sessionId === sessionId && (state === 'accepted' || state === 'handed_off'), ) .map(({ admission }) => admission), rebindMessageAdmissionTranscript: async () => undefined, diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index e5a4e5c0b9..966eb113a7 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -4779,9 +4779,7 @@ export class SqliteSessionMetadataStore { return; } this.db - .prepare( - 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', - ) + .prepare('UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?') .run(SQLITE_SESSION_MESSAGE_CHUNK_MARKER, sessionId, sequence); this.db .prepare( From cb785b5c54a58ff236e04585ca03283470d2ad3a Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 03:10:47 +0800 Subject: [PATCH 19/22] fix(storage): persist follow-up reorder permutations Generated-by: Codex --- .../sqlite-session-metadata-store.test.ts | 45 +++++++++++++++++++ .../src/sqlite-session-metadata-store.ts | 3 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index e15b352ebe..fe0fec1ea7 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -351,6 +351,51 @@ describe('SqliteSessionMetadataStore', () => { } }); + test('persists a follow-up reorder across SQLite restart', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-message-reorder-')); + const path = join(root, 'state.sqlite'); + try { + const store = createSqliteSessionMetadataStore(path); + try { + await store.create(fullHeader({ id: 'session-reorder' })); + for (const [index, messageId] of ['message-first', 'message-second'].entries()) { + await store.commitMessageAdmission({ + sessionId: 'session-reorder', + turnId: 'turn-current', + runId: 'run-current', + messageId, + content: { text: messageId }, + modelContent: { text: messageId }, + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: 20 + index, + }); + } + await store.reorderMessageAdmissions('session-reorder', [ + 'message-second', + 'message-first', + ]); + } finally { + store.close(); + } + + const reopened = createSqliteSessionMetadataStore(path); + try { + assert.deepEqual( + (await reopened.listMessageAdmissions('session-reorder')).map( + (admission) => admission.messageId, + ), + ['message-second', 'message-first'], + ); + } finally { + reopened.close(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + test('rejects an oversized durable Message admission before transcript mutation', async () => { const store = createSqliteSessionMetadataStore(':memory:'); try { diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 966eb113a7..c07243d465 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -1984,9 +1984,10 @@ export class SqliteSessionMetadataStore { ) .all(sessionId) as Array<{ message_id?: unknown }>; const current = rows.map((row) => row.message_id); + const currentIds = new Set(current); if ( current.length !== unique.length || - current.some((messageId, index) => messageId !== unique[index]) + unique.some((messageId) => !currentIds.has(messageId)) ) { throw new SessionMetadataConflictError('Message admission reorder identity conflict'); } From 0fc419ed94a824696d4eeaa713b60a08d9e622d9 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 03:20:07 +0800 Subject: [PATCH 20/22] fix(runtime-host): persist canonical message admission Generated-by: Codex --- .../__tests__/execution-host-recovery.test.ts | 58 ++++++++++++++++ .../fixtures/execution-host-suite.ts | 2 +- .../src/__tests__/goal-root-authority.test.ts | 4 +- .../src/__tests__/message-coordinator.test.ts | 13 ++-- .../__tests__/root-turn-coordinator.test.ts | 68 +++++++++++++++++-- .../src/server/execution-composition.ts | 4 +- .../src/server/message-coordinator.ts | 67 ++++++++++-------- .../src/server/root-turn-coordinator.ts | 2 + .../sqlite-session-metadata-store.test.ts | 36 ++-------- packages/storage/src/message-receipt-store.ts | 11 ++- .../src/sqlite-session-metadata-schema.ts | 2 +- .../src/sqlite-session-metadata-store.ts | 27 ++++---- 12 files changed, 203 insertions(+), 91 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts index 86e95b4cf0..3a7ae6d7c4 100644 --- a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts @@ -299,6 +299,64 @@ test('same idle Message submit is connection-independent and starts one canonica }); }); +test('a rejected idle Message submit leaves no durable transcript entry', async () => { + await withExecutionRoot(async (fixture) => { + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const messageId = randomUUID(); + try { + await assert.rejects( + () => + client.request('turn.message.submit', { + originHostEpoch: host.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: '/skill:missing reject this submit' }, + placement: 'current_turn', + }), + operationError('operation_conflict'), + ); + } finally { + await client.close(); + await fixture.stopHost(host); + } + + assert.deepEqual( + (await fixture.readSessionUserMessages()) + .filter((message) => message.id === messageId) + .map((message) => message.id), + [], + ); + }); +}); + +test('an allowed 32 KiB idle Message crosses the durable admission boundary', async () => { + await withExecutionRoot(async (fixture) => { + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const messageId = randomUUID(); + try { + const started = await client.request('turn.message.submit', { + originHostEpoch: host.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: 'x'.repeat(32 * 1024) }, + placement: 'current_turn', + }); + assert.equal(started.disposition, 'turn_started'); + } finally { + await client.close(); + await fixture.stopHost(host); + } + assert.deepEqual( + (await fixture.readSessionUserMessages()) + .filter((message) => message.id === messageId) + .map((message) => message.text), + ['x'.repeat(32 * 1024)], + ); + }); +}); + test('stale Session operations return not_found across the SQLite-backed UDS Host boundary', async () => { await withExecutionRoot(async (fixture) => { const host = await fixture.startHost(); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index 324c89d478..e9623f1bcc 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -697,7 +697,7 @@ export class ExecutionFixture { runId, messageId: input.messageId, content, - modelContent: content, + submittedContentDigest: contentDigest, submittedPlacement: 'current_turn', placement: 'current_turn', disposition: 'steering', diff --git a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts index 497a5566a5..88621d4238 100644 --- a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts +++ b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts @@ -559,8 +559,8 @@ async function createFixture(options: { recoverAdmissions?: boolean } = {}): Pro readRootState: (sessionId) => requireCoordinator(coordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, lease) => requireCoordinator(coordinator).claimStopFence(input, commitQueueFence, lease), - startFromMessage: (input, lease) => - requireCoordinator(coordinator).startFromMessage(input, lease), + startFromMessage: (input, lease, commitAdmission) => + requireCoordinator(coordinator).startFromMessage(input, lease, commitAdmission), prepareMessage: (input) => requireCoordinator(coordinator).prepareMessage(input), claimStop: (input, commitQueueFence, lease) => requireCoordinator(coordinator).claimStop(input, commitQueueFence, lease), diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index 164e3d8108..a5ddf58051 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -120,7 +120,7 @@ test('submit re-runs admission when the queue revision moves during preflight', owner.release(); }); -test('keeps submitted Skill text durable while handing prepared content to steering and follow-up roots', async () => { +test('persists prepared Skill content while projecting the submitted text', async () => { const fixture = createFixture(); fixture.setMessagePreparation(async (input) => ({ kind: 'ready', @@ -272,7 +272,9 @@ test('recovered followups without a connection owner still form one successor ba runId: ROOT.runId, messageId: 'recovered-followup', content: { text: 'recover without a connection owner' }, - modelContent: { text: 'recover without a connection owner' }, + submittedContentDigest: messageContentDigest({ + text: 'recover without a connection owner', + }), submittedPlacement: 'next_turn', placement: 'next_turn', disposition: 'followup', @@ -315,7 +317,7 @@ test('recovery treats a durable steering event as the handoff proof', async () = runId: ROOT.runId, messageId: 'recovered-steering', content: { text: 'recover this steering event' }, - modelContent: { text: 'recover this steering event' }, + submittedContentDigest: messageContentDigest({ text: 'recover this steering event' }), submittedPlacement: 'current_turn', placement: 'current_turn', disposition: 'steering', @@ -901,7 +903,10 @@ test('editing a promoted entry preserves its original submitted placement', asyn assert.equal(admission.placement, 'current_turn'); assert.equal(admission.disposition, 'steering'); assert.deepEqual(admission.content, { text: 'edited after promotion' }); - assert.deepEqual(admission.modelContent, { text: 'edited after promotion' }); + assert.equal( + admission.submittedContentDigest, + messageContentDigest({ text: 'edited after promotion' }), + ); }); test('entry promote requires an active Turn', async () => { diff --git a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts index 4811893236..0fc991a4b4 100644 --- a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts @@ -812,6 +812,66 @@ test('idle turn.message.submit applies hosted Skill preparation before durable a } }); +test('idle Skill admission persists only canonical content before root handoff', async () => { + const canonicalText = 'Write clearly.\n\nDraft this.'; + const fixture = await createFailureFixture({ + registerBackend: (backends) => + backends.register('ai-sdk', (context) => new FakeBackend(context)), + prepareSkillInvocation: async (): Promise => ({ + disposition: 'ready', + sendText: canonicalText, + skillInvocation: { + loaded: [{ id: 'writer', name: 'Writer' }], + failed: [], + receipts: [], + }, + }), + wrapAdmissionStore: (store) => ({ + admitRootTurn: async () => { + throw new Error('injected root admission failure'); + }, + readRootTurnAdmission: (sessionId, turnId) => store.readRootTurnAdmission(sessionId, turnId), + readRootTurnSourceMessageReceipt: (sessionId, messageId) => + store.readRootTurnSourceMessageReceipt(sessionId, messageId), + listRootTurnAdmissionsForRecovery: (sessionId) => + store.listRootTurnAdmissionsForRecovery(sessionId), + }), + }); + try { + await assert.rejects( + fixture.messages.handlers['turn.message.submit']( + { + originHostEpoch: fixture.hostEpoch, + sessionId: fixture.sessionId, + messageId: 'idle-skill-before-handoff', + content: { text: '/skill:writer Draft this.' }, + placement: 'current_turn', + }, + operationContext(fixture.hostEpoch, fixture.acquireResidency), + ), + /injected root admission failure/, + ); + const admission = await fixture.stores.sessionStore.readMessageAdmission( + fixture.sessionId, + 'idle-skill-before-handoff', + ); + assert.deepEqual(admission?.content, { + text: canonicalText, + displayText: '/skill:writer Draft this.', + inlineReferences: [], + }); + assert.deepEqual( + (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).map((message) => ({ + text: message.type === 'user' ? message.text : undefined, + displayText: message.type === 'user' ? message.displayText : undefined, + })), + [{ text: canonicalText, displayText: '/skill:writer Draft this.' }], + ); + } finally { + await fixture.dispose(); + } +}); + test('turn.start rejects oversized preparation before admission and preserves not-found semantics', async () => { let preparationCount = 0; let preparation: 'blocked' | 'oversized_content' | 'oversized_feedback' = 'blocked'; @@ -2159,8 +2219,8 @@ test('hosted linked child roots share admission, message, terminal, and stop aut readRootState: (sessionId) => requireCoordinator(coordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStopFence(input, commitQueueFence, admission), - startFromMessage: (input, admission) => - requireCoordinator(coordinator).startFromMessage(input, admission), + startFromMessage: (input, admission, commitAdmission) => + requireCoordinator(coordinator).startFromMessage(input, admission, commitAdmission), prepareMessage: (input) => requireCoordinator(coordinator).prepareMessage(input), claimStop: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStop(input, commitQueueFence, admission), @@ -4780,8 +4840,8 @@ async function createFailureFixture(options: { readRootState: (sessionId) => requireCoordinator(coordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStopFence(input, commitQueueFence, admission), - startFromMessage: (input, admission) => - requireCoordinator(coordinator).startFromMessage(input, admission), + startFromMessage: (input, admission, commitAdmission) => + requireCoordinator(coordinator).startFromMessage(input, admission, commitAdmission), prepareMessage: (input) => requireCoordinator(coordinator).prepareMessage(input), claimStop: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStop(input, commitQueueFence, admission), diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index c220755f0f..64b02d3364 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -466,8 +466,8 @@ export async function createExecutionRuntimeHostComposition( requireRootCoordinator(rootCoordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, admission) => requireRootCoordinator(rootCoordinator).claimStopFence(input, commitQueueFence, admission), - startFromMessage: (input, admission) => - requireRootCoordinator(rootCoordinator).startFromMessage(input, admission), + startFromMessage: (input, admission, commitAdmission) => + requireRootCoordinator(rootCoordinator).startFromMessage(input, admission, commitAdmission), startRecoveredMessages: (input, admission) => requireRootCoordinator(rootCoordinator).startRecoveredMessages(input, admission), prepareMessage: (input) => requireRootCoordinator(rootCoordinator).prepareMessage(input), diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index f9db15b100..c72595bf92 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -147,6 +147,7 @@ export interface HostMessageRootPort { startFromMessage( input: HostMessageStartInput, admission: SessionAdmissionLease, + commitAdmission: (canonicalContent: MessageContent) => Promise, ): Promise<{ readonly turnId: string } | { readonly error: string }>; startRecoveredMessages?( input: HostMessageRecoveryBatch, @@ -214,6 +215,7 @@ interface LiveEntry { readonly admittedAt: number; content: MessageContent; modelContent: MessageContent; + submittedContentDigest: `sha256:${string}`; readonly initiatingConnectionId: string; readonly placement: MessagePlacement; readonly disposition: 'steering' | 'followup'; @@ -725,7 +727,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { this.#root.startRecoveredMessages!( { sessionId, - content: aggregateMessageContents(pending.map((entry) => entry.modelContent)), + content: aggregateMessageContents(pending.map((entry) => entry.content)), submittedContent: aggregateMessageContents(pending.map((entry) => entry.content)), sources: pending.map(pendingMessageSource), }, @@ -756,8 +758,9 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { turnId: admission.turnId, runId: admission.runId, admittedAt: admission.admittedAt, - content: admission.content, - modelContent: admission.modelContent, + content: submittedProjectionContent(admission.content), + modelContent: admission.content, + submittedContentDigest: admission.submittedContentDigest, initiatingConnectionId: '', placement: admission.placement, disposition: admission.disposition, @@ -896,26 +899,13 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { ); if ( pendingAdmission && - (!messageContentsEqual(pendingAdmission.content, payload.content) || + (pendingAdmission.submittedContentDigest !== messageContentDigest(payload.content) || pendingAdmission.submittedPlacement !== input.placement) ) { return failure('operation_conflict', 'Message admission has a different payload'); } const turnId = pendingAdmission?.turnId ?? this.#createId(); const runId = pendingAdmission?.runId ?? this.#createId(); - const messageAdmission: PendingMessageAdmission = { - sessionId: input.sessionId, - turnId, - runId, - messageId: input.messageId, - content: payload.content, - modelContent: payload.content, - submittedPlacement: input.placement, - placement: 'current_turn', - disposition: 'steering', - admittedAt: pendingAdmission?.admittedAt ?? Date.now(), - }; - await this.#lifecycle.commitMessageAdmission(messageAdmission); const started = await this.#root.startFromMessage( { sessionId: input.sessionId, @@ -926,9 +916,22 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { runId, }, admission, + async (canonicalContent) => { + await this.#lifecycle.commitMessageAdmission({ + sessionId: input.sessionId, + turnId, + runId, + messageId: input.messageId, + content: canonicalContent, + submittedContentDigest: messageContentDigest(payload.content), + submittedPlacement: input.placement, + placement: 'current_turn', + disposition: 'steering', + admittedAt: pendingAdmission?.admittedAt ?? Date.now(), + }); + }, ); if ('error' in started) { - await this.#lifecycle.cancelMessageAdmissions(input.sessionId, [input.messageId]); return failure('operation_conflict', started.error); } if (!isEntityId(started.turnId)) { @@ -1037,8 +1040,8 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { turnId: rootState.turnId, runId: rootState.runId, messageId: input.messageId, - content: payload.content, - modelContent: prepared.content, + content: prepared.content, + submittedContentDigest: messageContentDigest(payload.content), submittedPlacement: input.placement, placement: input.placement, disposition, @@ -1054,6 +1057,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { admittedAt: messageAdmission.admittedAt, content: payload.content, modelContent: prepared.content, + submittedContentDigest: messageAdmission.submittedContentDigest, initiatingConnectionId, placement: input.placement, disposition, @@ -1347,8 +1351,8 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { turnId: entry.turnId, runId: entry.runId, messageId: entry.messageId, - content: entry.content, - modelContent: entry.modelContent, + content: entry.modelContent, + submittedContentDigest: entry.submittedContentDigest, submittedPlacement: 'next_turn', placement: 'current_turn', disposition: 'steering', @@ -1453,8 +1457,8 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { turnId: queued.entry.turnId, runId: queued.entry.runId, messageId: queued.entry.messageId, - content, - modelContent, + content: modelContent, + submittedContentDigest: messageContentDigest(content), submittedPlacement: admission?.submittedPlacement ?? queued.entry.placement, placement: queued.entry.placement, disposition: queued.entry.disposition, @@ -1462,6 +1466,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { }); queued.entry.content = content; queued.entry.modelContent = modelContent; + queued.entry.submittedContentDigest = messageContentDigest(content); this.#mutated(state); const result = { queueRevision: state.revision }; try { @@ -1829,7 +1834,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { id: leaseId, messageId: entry.messageId, content: normalizeMessageContent(entry.modelContent), - submittedContentDigest: messageContentDigest(entry.content), + submittedContentDigest: entry.submittedContentDigest, }; }); this.#mutated(state); @@ -2184,7 +2189,7 @@ function sourceFromEntry(entry: LiveEntry): RootFollowupSource { return { messageId: entry.messageId, content: normalizeMessageContent(entry.modelContent), - submittedContentDigest: messageContentDigest(entry.content), + submittedContentDigest: entry.submittedContentDigest, placement: entry.placement, disposition: entry.disposition, }; @@ -2193,13 +2198,19 @@ function sourceFromEntry(entry: LiveEntry): RootFollowupSource { function pendingMessageSource(admission: PendingMessageAdmission): RootTurnSourceMessage { return { messageId: admission.messageId, - content: normalizeMessageContent(admission.modelContent), - submittedContentDigest: messageContentDigest(admission.content), + content: normalizeMessageContent(admission.content), + submittedContentDigest: admission.submittedContentDigest, placement: admission.placement, disposition: admission.disposition, }; } +function submittedProjectionContent(content: MessageContent): MessageContent { + const normalized = normalizeMessageContent(content); + const text = normalized.displayText ?? normalized.text; + return normalizeMessageContent({ ...normalized, text, displayText: text }); +} + function queuedSnapshot(entry: LiveEntry): QueuedMessageSnapshot { return { entryId: entry.entryId, diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 0fceafc755..813955b3f3 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -1001,6 +1001,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { startFromMessage( input: HostMessageStartInput, admissionLease: SessionAdmissionLease, + commitAdmission: (canonicalContent: MessageContent) => Promise, ): Promise<{ readonly turnId: string } | { readonly error: string }> { return this.runCommand(async () => { const content = normalizeMessageContent(input.content); @@ -1057,6 +1058,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } await this.prepareFreshAgentGraphEpoch(header); + await commitAdmission(canonicalContent.content); const admitted = await this.rootAdmissionOwner.admitRootTurn({ sessionId: input.sessionId, diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index fe0fec1ea7..3e3d0be12e 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -25,6 +25,7 @@ import { DatabaseSync } from 'node:sqlite'; import { describe, test } from 'node:test'; import { Worker } from 'node:worker_threads'; import { AgentGraphClientTerminalCursorError } from '@maka/core/agent-graph-client-projection'; +import { messageContentDigest } from '@maka/core/events'; import { canReadPath, createReadOnlyPermissionProfile, @@ -248,7 +249,7 @@ describe('SqliteSessionMetadataStore', () => { runId: 'run-1', messageId: 'message-1', content: { text: 'submitted', displayText: 'submitted' }, - modelContent: { text: 'submitted', displayText: 'submitted' }, + submittedContentDigest: messageContentDigest({ text: 'submitted' }), submittedPlacement: 'current_turn', placement: 'current_turn', disposition: 'steering', @@ -258,7 +259,6 @@ describe('SqliteSessionMetadataStore', () => { const normalizedAdmission = { ...admission, content: { text: 'submitted' }, - modelContent: { text: 'submitted' }, }; assert.deepEqual(await store.commitMessageAdmission(admission), normalizedAdmission); assert.deepEqual( @@ -313,7 +313,9 @@ describe('SqliteSessionMetadataStore', () => { runId: 'run-current', messageId: 'message-followup', content: { text: 'queued before the successor root' }, - modelContent: { text: 'queued before the successor root' }, + submittedContentDigest: messageContentDigest({ + text: 'queued before the successor root', + }), submittedPlacement: 'next_turn', placement: 'next_turn', disposition: 'followup', @@ -365,7 +367,7 @@ describe('SqliteSessionMetadataStore', () => { runId: 'run-current', messageId, content: { text: messageId }, - modelContent: { text: messageId }, + submittedContentDigest: messageContentDigest({ text: messageId }), submittedPlacement: 'next_turn', placement: 'next_turn', disposition: 'followup', @@ -396,32 +398,6 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('rejects an oversized durable Message admission before transcript mutation', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - try { - await store.create(fullHeader({ id: 'session-oversized' })); - await assert.rejects( - () => - store.commitMessageAdmission({ - sessionId: 'session-oversized', - turnId: 'turn-oversized', - runId: 'run-oversized', - messageId: 'message-oversized', - content: { text: 'x'.repeat(70_000) }, - modelContent: { text: 'x'.repeat(70_000) }, - submittedPlacement: 'current_turn', - placement: 'current_turn', - disposition: 'steering', - admittedAt: 10, - }), - /exceeds size limit/, - ); - assert.deepEqual(await store.readMessages('session-oversized'), []); - } finally { - store.close(); - } - }); - test('migrates v24 legacy session statuses to active exactly once', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-status-v24-')); const path = join(root, 'state.sqlite'); diff --git a/packages/storage/src/message-receipt-store.ts b/packages/storage/src/message-receipt-store.ts index 42cd1375fe..8fafab521a 100644 --- a/packages/storage/src/message-receipt-store.ts +++ b/packages/storage/src/message-receipt-store.ts @@ -38,7 +38,7 @@ export interface PendingMessageAdmission { readonly runId: string; readonly messageId: string; readonly content: MessageContent; - readonly modelContent: MessageContent; + readonly submittedContentDigest: `sha256:${string}`; readonly submittedPlacement: 'current_turn' | 'next_turn'; readonly placement: 'current_turn' | 'next_turn'; readonly disposition: 'steering' | 'followup'; @@ -96,10 +96,9 @@ export function normalizePendingMessageAdmission( const normalized = Object.freeze({ ...admission, content: normalizeMessageContent(admission.content), - modelContent: normalizeMessageContent(admission.modelContent), }); - if (Buffer.byteLength(JSON.stringify(normalized), 'utf8') > RECEIPT_MAX_BYTES) { - throw new Error('Pending message admission exceeds size limit'); + if (!/^sha256:[a-f0-9]{64}$/u.test(normalized.submittedContentDigest)) { + throw new Error('Invalid pending Message submitted content digest'); } return normalized; } @@ -115,12 +114,12 @@ export function samePendingMessageAdmission( a.turnId === b.turnId && a.runId === b.runId && a.messageId === b.messageId && + a.submittedContentDigest === b.submittedContentDigest && a.submittedPlacement === b.submittedPlacement && a.placement === b.placement && a.disposition === b.disposition && a.admittedAt === b.admittedAt && - isDeepStrictEqual(a.content, b.content) && - isDeepStrictEqual(a.modelContent, b.modelContent) + isDeepStrictEqual(a.content, b.content) ); } diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index ef08df2717..d1ccc46a65 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -830,7 +830,7 @@ const MIGRATIONS: ReadonlyMap = new Map([ run_id TEXT NOT NULL, message_id TEXT NOT NULL, content_json TEXT NOT NULL, - model_content_json TEXT NOT NULL, + submitted_content_digest TEXT NOT NULL, submitted_placement TEXT NOT NULL CHECK (submitted_placement IN ('current_turn', 'next_turn')), placement TEXT NOT NULL CHECK (placement IN ('current_turn', 'next_turn')), diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index c07243d465..6954c2b593 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -231,7 +231,7 @@ interface MessageAdmissionRow { readonly run_id?: unknown; readonly message_id?: unknown; readonly content_json?: unknown; - readonly model_content_json?: unknown; + readonly submitted_content_digest?: unknown; readonly submitted_placement?: unknown; readonly placement?: unknown; readonly disposition?: unknown; @@ -249,7 +249,7 @@ function decodeMessageAdmissionRow( typeof row.run_id !== 'string' || typeof row.message_id !== 'string' || typeof row.content_json !== 'string' || - typeof row.model_content_json !== 'string' || + typeof row.submitted_content_digest !== 'string' || (row.submitted_placement !== 'current_turn' && row.submitted_placement !== 'next_turn') || (row.placement !== 'current_turn' && row.placement !== 'next_turn') || (row.disposition !== 'steering' && row.disposition !== 'followup') || @@ -270,7 +270,8 @@ function decodeMessageAdmissionRow( runId: row.run_id, messageId: row.message_id, content: JSON.parse(row.content_json) as PendingMessageAdmission['content'], - modelContent: JSON.parse(row.model_content_json) as PendingMessageAdmission['modelContent'], + submittedContentDigest: + row.submitted_content_digest as PendingMessageAdmission['submittedContentDigest'], submittedPlacement: row.submitted_placement, placement: row.placement, disposition: row.disposition, @@ -1549,7 +1550,7 @@ export class SqliteSessionMetadataStore { const existingRow = this.db .prepare( ` - SELECT turn_id, run_id, message_id, content_json, model_content_json, + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at FROM message_admissions WHERE session_id = ? AND message_id = ? @@ -1582,7 +1583,7 @@ export class SqliteSessionMetadataStore { .prepare( ` INSERT INTO message_admissions( - session_id, turn_id, run_id, message_id, content_json, model_content_json, + session_id, turn_id, run_id, message_id, content_json, submitted_content_digest, submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'accepted', ?, ?) `, @@ -1593,7 +1594,7 @@ export class SqliteSessionMetadataStore { stored.runId, stored.messageId, JSON.stringify(stored.content), - JSON.stringify(stored.modelContent), + stored.submittedContentDigest, stored.submittedPlacement, stored.placement, stored.disposition, @@ -1662,7 +1663,7 @@ export class SqliteSessionMetadataStore { const row = this.db .prepare( ` - SELECT turn_id, run_id, message_id, content_json, model_content_json, + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at FROM message_admissions WHERE session_id = ? AND message_id = ? @@ -1681,7 +1682,7 @@ export class SqliteSessionMetadataStore { const rows = this.db .prepare( ` - SELECT turn_id, run_id, message_id, content_json, model_content_json, + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at FROM message_admissions WHERE session_id = ? AND lifecycle_state = 'accepted' @@ -1702,7 +1703,7 @@ export class SqliteSessionMetadataStore { const rows = this.db .prepare( ` - SELECT turn_id, run_id, message_id, content_json, model_content_json, + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at FROM message_admissions WHERE session_id = ? AND lifecycle_state IN ('accepted', 'handed_off') @@ -1763,7 +1764,7 @@ export class SqliteSessionMetadataStore { const admissionRow = this.db .prepare( ` - SELECT turn_id, run_id, message_id, content_json, model_content_json, + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at FROM message_admissions WHERE session_id = ? AND message_id = ? @@ -1838,7 +1839,7 @@ export class SqliteSessionMetadataStore { const currentRow = this.db .prepare( ` - SELECT turn_id, run_id, message_id, content_json, model_content_json, + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at FROM message_admissions WHERE session_id = ? AND message_id = ? @@ -1862,13 +1863,13 @@ export class SqliteSessionMetadataStore { .prepare( ` UPDATE message_admissions - SET content_json = ?, model_content_json = ?, placement = ?, disposition = ? + SET content_json = ?, submitted_content_digest = ?, placement = ?, disposition = ? WHERE session_id = ? AND message_id = ? AND lifecycle_state = 'accepted' `, ) .run( JSON.stringify(stored.content), - JSON.stringify(stored.modelContent), + stored.submittedContentDigest, stored.placement, stored.disposition, stored.sessionId, From d691232f475ad902f0bb25ed9745a6f27f83c8af Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 03:23:26 +0800 Subject: [PATCH 21/22] refactor(runtime): remove root message rematerialization Generated-by: Codex --- .../__tests__/execution-host-message.test.ts | 1 + .../src/server/hosted-execution-recovery.ts | 27 ++++++++- .../src/server/root-turn-coordinator.ts | 24 +------- .../runtime-kernel-interaction.test.ts | 42 +------------- packages/runtime/src/agent-run.ts | 47 +++++++-------- packages/runtime/src/runtime-kernel.ts | 58 +------------------ packages/runtime/src/session-manager.ts | 15 ----- packages/storage/src/agent-run-store.ts | 3 +- 8 files changed, 58 insertions(+), 159 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-host-message.test.ts b/packages/runtime-host/src/__tests__/execution-host-message.test.ts index 61c1ee357e..15bbebde4e 100644 --- a/packages/runtime-host/src/__tests__/execution-host-message.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-message.test.ts @@ -214,6 +214,7 @@ test('steering becomes durable and ordered followups automatically start the nex const chain = await fixture.readAdmissionChain(); assert.equal(chain.length, 2); assert.equal(chain[1]?.previousRootTurnId, firstTurnId); + assert.equal(chain[1]?.userMessageId, null); assert.deepEqual( chain[1]?.sourceMessages.map(({ messageId, content, placement, disposition }) => ({ messageId, diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 032f745ff2..6de7a9a061 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -87,12 +87,18 @@ export async function prepareHostedExecutionRecovery( `Admitted Turn ${admission.turnId} has queue-independent execution with Message queue sources`, ); } - if (executionContract.requiresUserMessage !== (admission.userMessageId !== null)) { + const requiresUserMessage = + executionContract.requiresUserMessage && + !(admission.execution.kind === 'external_message' && admission.sourceMessages.length > 1); + if (requiresUserMessage !== (admission.userMessageId !== null)) { throw new Error( `Admitted Turn ${admission.turnId} has an invalid UserMessage execution contract`, ); } if (admission.userMessageId === null) { + if (admission.sourceMessages.length > 0) { + verifyQueueSourceMessages(admission, messageIndex); + } if (rootUserMessages.length > 0) { throw new Error(`Admitted Turn ${admission.turnId} must not record a UserMessage`); } @@ -301,6 +307,25 @@ function verifyOrRecoverUserMessage( indexRecoveryMessage(index, recoveredMessage); } +function verifyQueueSourceMessages( + admission: RootTurnAdmission, + index: RecoveryMessageIndex, +): void { + for (const source of admission.sourceMessages) { + const owners = index.messagesById.get(source.messageId) ?? []; + if ( + owners.length !== 1 || + owners[0]?.type !== 'user' || + owners[0].turnId !== admission.turnId || + !messageContentsEqual(normalizeMessageContent(owners[0]), source.content) + ) { + throw new Error( + `Admitted Turn ${admission.turnId} does not match queue source ${source.messageId}`, + ); + } + } +} + function recoveryUserMessage(admission: RootTurnAdmission): RecoveryUserMessage { if (!admission.userMessageId || !admission.normalizedInput) { throw new Error(`Admitted Turn ${admission.turnId} does not own a UserMessage`); diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 813955b3f3..709e4facf1 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -157,7 +157,6 @@ interface ActiveRootTurn { residency: RuntimeHostResidency; stopRequested: boolean; messageTransitionCommitted: boolean; - initialUserMessagesMaterialized: boolean; } export type TurnStartOutcome = OperationOutcome<'turn.start'>; @@ -1134,7 +1133,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId: input.sessionId, turnId, proposedRunId: randomUUID(), - proposedUserMessageId: randomUUID(), + proposedUserMessageId: input.sources.length === 1 ? input.sources[0]!.messageId : null, execution: { kind: 'external_message', inputDigest: messageContentDigest(input.submittedContent), @@ -2014,21 +2013,6 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { userMessageId: admission.userMessageId, execution: admission.execution, }); - const initialUserMessagesMaterialized = admission.sourceMessages.length > 0; - if (initialUserMessagesMaterialized) { - await this.manager.materializeRootSourceMessages({ - sessionId: input.sessionId, - turnId: input.turnId, - messages: admission.sourceMessages.map((source) => ({ - messageId: source.messageId, - content: source.content, - ...(source.submittedContentDigest - ? { submittedContentDigest: source.submittedContentDigest } - : {}), - disposition: source.disposition, - })), - }); - } const { runId } = admission; const existingRun = await this.readRunIfPresent(input.sessionId, runId); if (replacing && existingRun) { @@ -2121,7 +2105,6 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { residency, stopRequested: false, messageTransitionCommitted: false, - initialUserMessagesMaterialized, }; if (replacing && this.#executions.get(input.sessionId) !== replacing) { residency.release(); @@ -2218,8 +2201,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { }, { runId: active.runId, - userMessageId: active.userMessageId ?? undefined, - recordInitialUserMessage: !active.initialUserMessagesMaterialized, + userMessageId: active.userMessageId, durability: 'required', onRunStarted: async (startedRunId) => { if (startedRunId !== active.runId) { @@ -2433,7 +2415,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId: batch.sessionId, turnId, proposedRunId: randomUUID(), - proposedUserMessageId: randomUUID(), + proposedUserMessageId: batch.sources.length === 1 ? batch.sources[0]!.messageId : null, execution: { kind: 'external_message', inputDigest: messageContentDigest(batch.submittedContent), diff --git a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts index 8f65b55009..4eab4a25e4 100644 --- a/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts +++ b/packages/runtime/src/__tests__/runtime-kernel-interaction.test.ts @@ -20,7 +20,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import { messageContentDigest, type SessionEvent } from '@maka/core/events'; +import type { SessionEvent } from '@maka/core/events'; import type { SessionHeader, StoredMessage } from '@maka/core/session'; import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary'; @@ -39,46 +39,6 @@ import { import { BackendRegistry, type SessionStore } from '../session-manager.js'; describe('RuntimeKernel Interaction close cleanup', () => { - test('accepts submitted transcript content when the root source is model-prepared', async () => { - const store = memoryStore(); - const submitted = { text: '/skill:writer inspect' }; - await store.appendMessage(SESSION_ID, { - type: 'user', - id: 'submitted-message', - turnId: 'prepared-turn', - ts: 1, - ...submitted, - }); - const kernel = new RuntimeKernel({ - store, - backends: new BackendRegistry(), - newId: () => 'materialize-id', - now: () => 1, - }); - - await kernel.materializeRootSourceMessages({ - sessionId: SESSION_ID, - turnId: 'prepared-turn', - messages: [ - { - messageId: 'submitted-message', - content: { text: 'inspect' }, - submittedContentDigest: messageContentDigest(submitted), - disposition: 'turn_started', - }, - ], - }); - assert.deepEqual(await store.readMessages(SESSION_ID), [ - { - type: 'user', - id: 'submitted-message', - turnId: 'prepared-turn', - ts: 1, - ...submitted, - }, - ]); - }); - test('reserve followed by begin failure settles a concurrent stop claim', async () => { const store = memoryStore(); const updateHeader = store.updateHeader; diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index 0c63293c45..ee3cf91e6c 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -145,7 +145,7 @@ export interface AgentRunInput { userInput: UserMessageInput; rootExecutionKind?: AgentRunHeader['rootExecutionKind']; runId?: string; - userMessageId?: string; + userMessageId?: string | null; durability?: AgentRunDurability; store: AgentRunSessionStore; runStore?: AgentRunStore; @@ -161,7 +161,6 @@ export interface AgentRunInput { commitContinuationStart?: (startedAt: number) => Promise<{ startEventId: string; created: true }>; hooks: AgentRunHooks; recordSessionMessages?: boolean; - recordInitialUserMessage?: boolean; invocationId?: string; /** Pre-resolved snapshot used by continuations; normal turns derive it from header + input. */ effectiveOrchestration?: EffectiveOrchestration; @@ -647,28 +646,30 @@ export class AgentRun { let initialRuntimeEventId: string; if (this.recordsSessionMessages()) { - const userMessageId = this.input.userMessageId ?? this.input.newId(); const userMessageTs = this.input.now(); - initialRuntimeEventId = userMessageId; - const userMsg = cloneAndFreezeRuntimeSnapshot({ - type: 'user', - id: userMessageId, - turnId: this.turnId, - ts: userMessageTs, - text: this.input.userInput.text, - ...(this.input.userInput.displayText !== undefined - ? { displayText: this.input.userInput.displayText } - : {}), - ...(this.input.userInput.attachments - ? { attachments: this.input.userInput.attachments } - : {}), - ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), - ...(this.input.userInput.inlineReferences - ? { inlineReferences: this.input.userInput.inlineReferences } - : {}), - ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), - }); - if (this.input.recordInitialUserMessage !== false) { + if (this.input.userMessageId === null) { + initialRuntimeEventId = this.input.newId(); + } else { + const userMessageId = this.input.userMessageId ?? this.input.newId(); + initialRuntimeEventId = userMessageId; + const userMsg = cloneAndFreezeRuntimeSnapshot({ + type: 'user', + id: userMessageId, + turnId: this.turnId, + ts: userMessageTs, + text: this.input.userInput.text, + ...(this.input.userInput.displayText !== undefined + ? { displayText: this.input.userInput.displayText } + : {}), + ...(this.input.userInput.attachments + ? { attachments: this.input.userInput.attachments } + : {}), + ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), + ...(this.input.userInput.inlineReferences + ? { inlineReferences: this.input.userInput.inlineReferences } + : {}), + ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), + }); await appendUserMessageOnce(this.input.store, this.sessionId, userMsg); } await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage); diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index c388139360..f93c8b7ee3 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -34,12 +34,8 @@ import type { } from '@maka/core/runtime-event-store'; import { isSessionInlineRun } from '@maka/core/agent-run'; import { - messageContentDigest, - messageContentsEqual, - normalizeMessageContent, type ActiveInteractionRequestEvent, type CompleteEvent, - type MessageContent, type QueueEnqueueOutcome, type SessionEvent, type TokenUsageEvent, @@ -188,16 +184,6 @@ export interface RuntimeKernelLike { respondToSandboxBoundary(sessionId: string, response: SandboxBoundaryResponse): Promise; listActiveInteractions?(sessionId: string): ActiveInteractionRequestEvent[]; respondToUserQuestion?(sessionId: string, response: UserQuestionResponse): Promise; - materializeRootSourceMessages?(input: { - sessionId: string; - turnId: string; - messages: readonly { - messageId: string; - content: MessageContent; - submittedContentDigest?: `sha256:${string}`; - disposition: 'steering' | 'followup' | 'turn_started'; - }[]; - }): Promise; /** Compatibility surface; durable message admission belongs to Runtime Host. */ steer(sessionId: string, text: string): QueueEnqueueOutcome; queueMessage(sessionId: string, text: string): QueueEnqueueOutcome; @@ -242,8 +228,7 @@ export class RuntimeContextCompactError extends Error { export interface TurnStartOptions { runId?: string; - userMessageId?: string; - recordInitialUserMessage?: boolean; + userMessageId?: string | null; durability?: AgentRunDurability; /** * Resolve turn admission after this Session has registered a pending start @@ -694,7 +679,6 @@ export class RuntimeKernel implements RuntimeKernelLike { userInput: input, runId: options.runId, userMessageId: options.userMessageId, - recordInitialUserMessage: options.recordInitialUserMessage, durability: options.durability, store: this.deps.store, runStore: this.deps.runStore, @@ -2410,46 +2394,6 @@ export class RuntimeKernel implements RuntimeKernelLike { return ''; } - async materializeRootSourceMessages(input: { - sessionId: string; - turnId: string; - messages: readonly { - messageId: string; - content: MessageContent; - submittedContentDigest?: `sha256:${string}`; - disposition: 'steering' | 'followup' | 'turn_started'; - }[]; - }): Promise { - const existingById = new Map( - (await this.deps.store.readMessages(input.sessionId)).map((message) => [message.id, message]), - ); - for (const message of input.messages) { - const existing = existingById.get(message.messageId); - if (existing) { - if ( - existing.type !== 'user' || - (!messageContentsEqual(normalizeMessageContent(existing), message.content) && - (message.submittedContentDigest === undefined || - messageContentDigest(normalizeMessageContent(existing)) !== - message.submittedContentDigest)) || - existing.turnId !== input.turnId - ) { - throw new Error(`Queued root source ${message.messageId} conflicts with its transcript`); - } - continue; - } - const materialized = { - type: 'user' as const, - id: message.messageId, - turnId: input.turnId, - ts: this.deps.now(), - ...structuredClone(message.content), - }; - await this.deps.store.appendMessage(input.sessionId, materialized); - existingById.set(message.messageId, materialized); - } - } - hasActiveRuns(sessionId: string): boolean { return this.backendGenerationsFor(sessionId).some((active) => active.activeRuns.size > 0); } diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index b66ceb76eb..c19b6eaf11 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -4820,21 +4820,6 @@ export class SessionManager { : this.runtimeKernel.stopSession(identity.sessionId, input); } - materializeRootSourceMessages(input: { - sessionId: string; - turnId: string; - messages: readonly { - messageId: string; - content: import('@maka/core/events').MessageContent; - submittedContentDigest?: `sha256:${string}`; - disposition: 'steering' | 'followup' | 'turn_started'; - }[]; - }): Promise { - const materialize = this.runtimeKernel.materializeRootSourceMessages; - if (!materialize) throw new Error('Runtime root message materialization is unavailable'); - return materialize.call(this.runtimeKernel, input); - } - /** Queue a user message for mid-turn injection at the next step boundary. */ steer(sessionId: string, text: string): QueueEnqueueOutcome { return this.runtimeKernel.steer(sessionId, text); diff --git a/packages/storage/src/agent-run-store.ts b/packages/storage/src/agent-run-store.ts index 069635e85d..f11839265a 100644 --- a/packages/storage/src/agent-run-store.ts +++ b/packages/storage/src/agent-run-store.ts @@ -1731,7 +1731,8 @@ function assertRootTurnAdmissionContract(admission: RootTurnAdmission): void { const providerRetry = execution.kind === 'linked_child_provider_retry'; const inputlessExecution = execution.kind === 'safe_boundary_continuation' || execution.kind === 'context_compact'; - const messageLessExecution = inputlessExecution || providerRetry; + const sourceBatch = execution.kind === 'external_message' && admission.sourceMessages.length > 1; + const messageLessExecution = inputlessExecution || providerRetry || sourceBatch; if (execution.kind === 'agent_graph_supervisor_wake') { if ( admission.turnOrchestration?.mode !== 'graph' || From 490ffb31a40346056449bdb6760873994979401f Mon Sep 17 00:00:00 2001 From: AstroHan Date: Tue, 25 Aug 2026 03:35:03 +0800 Subject: [PATCH 22/22] test(runtime-host): align multi-source root fixture Generated-by: Codex --- .../runtime-host/src/__tests__/root-admission-owner.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime-host/src/__tests__/root-admission-owner.test.ts b/packages/runtime-host/src/__tests__/root-admission-owner.test.ts index fdc0552dba..2502955ae0 100644 --- a/packages/runtime-host/src/__tests__/root-admission-owner.test.ts +++ b/packages/runtime-host/src/__tests__/root-admission-owner.test.ts @@ -325,7 +325,7 @@ function multiSourceAdmitInput(sessionId: string, turnId: string, admittedAt: nu sessionId, turnId, proposedRunId: `run-${turnId}`, - proposedUserMessageId: `message-${turnId}`, + proposedUserMessageId: null, execution: { kind: 'external_message' as const }, normalizedInput: { text: 'model text\n\nfollowup text',