diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index 4a58287e49..15c6170fb5 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -26,6 +26,7 @@ * Connection-setup events live in ./connections.ts (separate channel). */ +import * as nodeCrypto from 'node:crypto'; import type { AdditionalPermissionRequest, PermissionMode, @@ -398,6 +399,24 @@ export function messageContentsEqual(left: MessageContent, right: MessageContent ); } +export function messageContentDigest(content: MessageContent): `sha256:${string}` { + return `sha256:${nodeCrypto + .createHash('sha256') + .update(JSON.stringify(canonicalizeMessageContent(normalizeMessageContent(content)))) + .digest('hex')}`; +} + +function canonicalizeMessageContent(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalizeMessageContent); + if (value === null || typeof value !== 'object') return value; + return Object.fromEntries( + Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => [key, canonicalizeMessageContent(entry)]), + ); +} + function inlineReferencesEqual(left: InlineReference, right: InlineReference): boolean { return ( left.kind === right.kind && diff --git a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts index bb7a568554..31278d18cd 100644 --- a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts +++ b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts @@ -543,8 +543,10 @@ function createMessages( stores.agentRunStore.readRootTurnSourceMessageReceipt(requestedSessionId, messageId), readImmutableSteeringMessageProof: (requestedSessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(requestedSessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission: new SessionAdmissionGate(), acquireResidency: () => ({ release: () => undefined }), preflightSessionSnapshot: () => true, diff --git a/packages/runtime-host/src/__tests__/execution-host-message.test.ts b/packages/runtime-host/src/__tests__/execution-host-message.test.ts index 8c75c21e5a..15bbebde4e 100644 --- a/packages/runtime-host/src/__tests__/execution-host-message.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-message.test.ts @@ -195,6 +195,7 @@ test('steering becomes durable and ordered followups automatically start the nex await first.close(); await second.close(); await fixture.stopHost(host); + assert.equal(await fixture.readMessageLifecycleState(steeringId), 'handed_off'); const firstLedger = await fixture.readTurn(firstTurnId); const steeringEvents = firstLedger.runtimeEvents.filter( @@ -213,6 +214,7 @@ test('steering becomes durable and ordered followups automatically start the nex const chain = await fixture.readAdmissionChain(); assert.equal(chain.length, 2); assert.equal(chain[1]?.previousRootTurnId, firstTurnId); + assert.equal(chain[1]?.userMessageId, null); assert.deepEqual( chain[1]?.sourceMessages.map(({ messageId, content, placement, disposition }) => ({ messageId, @@ -236,9 +238,23 @@ test('steering becomes durable and ordered followups automatically start the nex assert.ok(followupTurnId); const followupLedger = await fixture.readTurn(followupTurnId); const expectedQuotes = followupSources.flatMap((source) => source.content.quotes ?? []); - assert.equal(followupLedger.userMessages.length, 1); - assert.deepEqual(followupLedger.userMessages[0]?.quotes, expectedQuotes); + assert.equal(followupLedger.userMessages.length, followupSources.length); + assert.deepEqual( + followupLedger.userMessages.flatMap((message) => message.quotes ?? []), + expectedQuotes, + ); assert.deepEqual(userRuntimeContent(followupLedger.runtimeEvents)?.quotes, expectedQuotes); + const sessionUserMessages = await fixture.readSessionUserMessages(); + for (const source of followupSources) { + assert.equal( + sessionUserMessages.filter((message) => message.id === source.messageId).length, + 1, + ); + } + assert.equal( + sessionUserMessages.filter((message) => message.turnId === followupTurnId).length, + followupSources.length, + ); }); }); diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index 6cc97ee966..1f911bce98 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -205,6 +205,7 @@ test('subscribed Clients share one canonical queue and ordered root handoff', as await waitForTerminalTurn(tui, fixture.sessionId, successor.snapshot.rootTurn.turnId); await tui.close(); await fixture.stopHost(host); + assert.equal(await fixture.readMessageLifecycleState(followupId), 'handed_off'); const chain = await fixture.readAdmissionChain(); assert.deepEqual( @@ -215,6 +216,113 @@ test('subscribed Clients share one canonical queue and ordered root handoff', as }); }); +test('production UDS admission commits one transcript before the root handoff', async () => { + await withExecutionRoot(async (fixture) => { + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const messageId = randomUUID(); + const started = await client.request('turn.message.submit', { + originHostEpoch: host.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: FAKE_ASK_USER_QUESTION_PROMPT }, + placement: 'current_turn', + }); + assert.equal(started.disposition, 'turn_started'); + if (started.disposition !== 'turn_started') return; + const active = await client.queryTurn({ sessionId: fixture.sessionId, turnId: started.turnId }); + await client.stopTurn({ + sessionId: fixture.sessionId, + turnId: started.turnId, + runId: active.runId, + }); + await client.close(); + await fixture.stopHost(host); + const ledger = await fixture.readTurn(started.turnId); + assert.deepEqual( + ledger.userMessages + .filter((message) => message.id === messageId) + .map((message) => message.id), + [messageId], + ); + assert.equal(await fixture.readMessageLifecycleState(messageId), 'cancelled'); + }); +}); + +test('a Host crash after queue admission recovers the durable successor once', async () => { + await withExecutionRoot(async (fixture) => { + const firstHost = await fixture.startHost(); + const first = await connectClient(fixture.root); + const started = requireStartedTurn( + await first.startTurn({ + sessionId: fixture.sessionId, + turnId: randomUUID(), + content: { text: FAKE_ASK_USER_QUESTION_PROMPT }, + }), + ); + const messageId = randomUUID(); + const queued = await first.request('turn.message.submit', { + originHostEpoch: firstHost.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: 'recover this accepted successor' }, + placement: 'next_turn', + }); + assert.equal(queued.disposition, 'followup'); + await fixture.killHost(firstHost); + await first.closed; + + const secondHost = await fixture.startHost(); + const second = await connectClient(fixture.root); + const subscription = await second.openSessionSubscription({ + sessionId: fixture.sessionId, + transcript: { kind: 'none' }, + }); + const probe = new SubscriptionProbe(subscription); + const successor = await probe.waitFor( + (frame) => + frame.kind === 'subscription.session_projection' && + frame.snapshot.rootTurn !== null && + frame.snapshot.rootTurn.turnId !== started.turnId, + 'durable successor was not recovered after the Host crash', + ); + assert.equal(successor.kind, 'subscription.session_projection'); + if (successor.kind !== 'subscription.session_projection' || !successor.snapshot.rootTurn) + return; + await waitForTerminalTurn(second, fixture.sessionId, successor.snapshot.rootTurn.turnId); + await subscription.close(); + await probe.done; + await second.close(); + await fixture.stopHost(secondHost); + assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); + }); +}); + +test('restart replays an atomically admitted root without duplicating its transcript', async () => { + await withExecutionRoot(async (fixture) => { + const turnId = randomUUID(); + const messageId = randomUUID(); + const content = { text: 'recover the root after admission before Run creation' }; + await fixture.seedAtomicRootAdmissionWithoutRun({ turnId, messageId, content }); + + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const terminal = await waitForTerminalTurn(client, fixture.sessionId, turnId); + assert.equal(terminal.status, 'completed'); + await client.close(); + await fixture.stopHost(host); + + const ledger = await fixture.readTurn(turnId); + assert.deepEqual( + ledger.userMessages + .filter((message) => message.id === messageId) + .map((message) => message.id), + [messageId], + ); + assert.equal(await fixture.readMessageLifecycleState(messageId), 'handed_off'); + }); +}); + test('concurrent root admission for one Session has a single winner', async () => { await withExecutionRoot(async (fixture) => { const host = await fixture.startHost(); diff --git a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts index 86e95b4cf0..3a7ae6d7c4 100644 --- a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts @@ -299,6 +299,64 @@ test('same idle Message submit is connection-independent and starts one canonica }); }); +test('a rejected idle Message submit leaves no durable transcript entry', async () => { + await withExecutionRoot(async (fixture) => { + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const messageId = randomUUID(); + try { + await assert.rejects( + () => + client.request('turn.message.submit', { + originHostEpoch: host.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: '/skill:missing reject this submit' }, + placement: 'current_turn', + }), + operationError('operation_conflict'), + ); + } finally { + await client.close(); + await fixture.stopHost(host); + } + + assert.deepEqual( + (await fixture.readSessionUserMessages()) + .filter((message) => message.id === messageId) + .map((message) => message.id), + [], + ); + }); +}); + +test('an allowed 32 KiB idle Message crosses the durable admission boundary', async () => { + await withExecutionRoot(async (fixture) => { + const host = await fixture.startHost(); + const client = await connectClient(fixture.root); + const messageId = randomUUID(); + try { + const started = await client.request('turn.message.submit', { + originHostEpoch: host.hostEpoch, + sessionId: fixture.sessionId, + messageId, + content: { text: 'x'.repeat(32 * 1024) }, + placement: 'current_turn', + }); + assert.equal(started.disposition, 'turn_started'); + } finally { + await client.close(); + await fixture.stopHost(host); + } + assert.deepEqual( + (await fixture.readSessionUserMessages()) + .filter((message) => message.id === messageId) + .map((message) => message.text), + ['x'.repeat(32 * 1024)], + ); + }); +}); + test('stale Session operations return not_found across the SQLite-backed UDS Host boundary', async () => { await withExecutionRoot(async (fixture) => { const host = await fixture.startHost(); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index f44e88b8fa..e9623f1bcc 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -38,7 +38,11 @@ import { test } from 'node:test'; import { TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import type { AgentRunHeader } from '@maka/core/agent-run'; -import { normalizeMessageContent, type MessageContent } from '@maka/core/events'; +import { + messageContentDigest, + normalizeMessageContent, + type MessageContent, +} from '@maka/core/events'; import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; import type { StoredMessage } from '@maka/core/session'; import type { Task } from '@maka/core/task-ledger'; @@ -672,6 +676,59 @@ export class ExecutionFixture { return this.seedTurnState(turnId, content, false, false); } + async seedAtomicRootAdmissionWithoutRun(input: { + turnId: string; + messageId: string; + content: MessageContent; + }): Promise { + const owner = await tryAcquireInteractiveRootOwner(this.capability); + assert.ok(owner); + if (!owner) throw new Error('Unable to acquire execution root for atomic root setup'); + let stores: Awaited> | undefined; + try { + stores = await openInteractiveExecutionStoresForWrite(owner.lease); + const admittedAt = Date.now(); + const content = normalizeMessageContent(input.content); + const contentDigest = messageContentDigest(content); + const runId = randomUUID(); + await stores.sessionStore.commitMessageAdmission({ + sessionId: this.sessionId, + turnId: input.turnId, + runId, + messageId: input.messageId, + content, + submittedContentDigest: contentDigest, + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt, + }); + const result = await stores.agentRunStore.admitRootTurn({ + sessionId: this.sessionId, + turnId: input.turnId, + proposedRunId: runId, + proposedUserMessageId: input.messageId, + execution: { kind: 'external_message', inputDigest: contentDigest }, + previousRootTurnId: null, + normalizedInput: content, + sourceMessages: [ + { + messageId: input.messageId, + content, + submittedContentDigest: contentDigest, + placement: 'current_turn', + disposition: 'turn_started', + }, + ], + admittedAt, + }); + assert.equal(result.kind, 'admitted'); + } finally { + await stores?.sessionStore.close?.(); + await owner.close(); + } + } + async archiveSession(): Promise { const owner = await tryAcquireInteractiveRootOwner(this.capability); assert.ok(owner); @@ -943,6 +1000,20 @@ export class ExecutionFixture { } } + async readSessionUserMessages(): Promise>> { + const reader = await acquireReader(this.capability); + let stores: Awaited> | undefined; + try { + stores = await openInteractiveExecutionStoresForRead(reader.lease); + return (await stores.sessionStore.readMessages(this.sessionId)).filter( + (message): message is Extract => message.type === 'user', + ); + } finally { + await stores?.sessionStore.close?.(); + await reader.close(); + } + } + async readAdmissionChain() { const owner = await tryAcquireInteractiveRootOwner(this.capability); assert.ok(owner); @@ -957,6 +1028,18 @@ export class ExecutionFixture { } } + async readMessageLifecycleState(messageId: string) { + const reader = await acquireReader(this.capability); + let stores: Awaited> | undefined; + try { + stores = await openInteractiveExecutionStoresForRead(reader.lease); + return await stores.sessionStore.readMessageLifecycleState(this.sessionId, messageId); + } finally { + await stores?.sessionStore.close?.(); + await reader.close(); + } + } + async readTurnFootprint(turnId: string): Promise<{ admitted: boolean; runCount: number; diff --git a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts index f5f74ebaa0..88621d4238 100644 --- a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts +++ b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts @@ -559,8 +559,8 @@ async function createFixture(options: { recoverAdmissions?: boolean } = {}): Pro readRootState: (sessionId) => requireCoordinator(coordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, lease) => requireCoordinator(coordinator).claimStopFence(input, commitQueueFence, lease), - startFromMessage: (input, lease) => - requireCoordinator(coordinator).startFromMessage(input, lease), + startFromMessage: (input, lease, commitAdmission) => + requireCoordinator(coordinator).startFromMessage(input, lease, commitAdmission), prepareMessage: (input) => requireCoordinator(coordinator).prepareMessage(input), claimStop: (input, commitQueueFence, lease) => requireCoordinator(coordinator).claimStop(input, commitQueueFence, lease), @@ -575,8 +575,10 @@ async function createFixture(options: { recoverAdmissions?: boolean } = {}): Pro stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission: admission, acquireResidency, requestDrain: () => { diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index 934227bd5a..a5ddf58051 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -19,11 +19,13 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import type { MessageContent } from '@maka/core/events'; +import { messageContentDigest, type MessageContent } from '@maka/core/events'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { + MessageLifecycleStore, MessageOperationReceipt, MessageReceiptStore, + PendingMessageAdmission, RootTurnSourceMessageReceipt, } from '@maka/storage/execution-stores'; import { @@ -39,7 +41,6 @@ import { type HostMessageRootPort, type HostMessageRootState, } from '../server/message-coordinator.js'; -import { messageContentDigest } from '../server/message-content-digest.js'; import { SessionAdmissionGate } from '../server/session-admission-gate.js'; const ROOT = { sessionId: 'session-1', turnId: 'turn-1', runId: 'run-1' } as const; @@ -119,7 +120,7 @@ test('submit re-runs admission when the queue revision moves during preflight', owner.release(); }); -test('keeps submitted Skill text durable while handing prepared content to steering and follow-up roots', async () => { +test('persists prepared Skill content while projecting the submitted text', async () => { const fixture = createFixture(); fixture.setMessagePreparation(async (input) => ({ kind: 'ready', @@ -263,6 +264,79 @@ test('partitions a mixed-Client follow-up queue across root handoffs', async () await fixture.coordinator.close(); }); +test('recovered followups without a connection owner still form one successor batch', async () => { + const fixture = createFixture(); + await fixture.lifecycle.commitMessageAdmission({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + messageId: 'recovered-followup', + content: { text: 'recover without a connection owner' }, + submittedContentDigest: messageContentDigest({ + text: 'recover without a connection owner', + }), + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: 1, + }); + + await fixture.coordinator.recoverPendingAfterHostRestart([ROOT.sessionId]); + const owner = fixture.coordinator.bindRun(ROOT); + owner.release(); + const batch = fixture.coordinator.beginTerminalTransition(ROOT); + assert.deepEqual( + batch.sources.map((source) => source.messageId), + ['recovered-followup'], + ); + fixture.coordinator.commitNextRoot(batch, { + sessionId: ROOT.sessionId, + turnId: 'turn-recovered-successor', + runId: 'run-recovered-successor', + }); + const successor = fixture.coordinator.bindRun({ + sessionId: ROOT.sessionId, + turnId: 'turn-recovered-successor', + runId: 'run-recovered-successor', + }); + successor.release(); + fixture.coordinator.completeIdle( + fixture.coordinator.beginTerminalTransition({ + sessionId: ROOT.sessionId, + turnId: 'turn-recovered-successor', + runId: 'run-recovered-successor', + }), + ); +}); + +test('recovery treats a durable steering event as the handoff proof', async () => { + const fixture = createFixture(); + await fixture.lifecycle.commitMessageAdmission({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + messageId: 'recovered-steering', + content: { text: 'recover this steering event' }, + submittedContentDigest: messageContentDigest({ text: 'recover this steering event' }), + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt: 1, + }); + fixture.events.push(steeringEvent('recovered-steering', 'recover this steering event')); + + await fixture.coordinator.recoverPendingAfterHostRestart([ROOT.sessionId]); + + assert.equal(fixture.readMessageLifecycleState('recovered-steering'), 'handed_off'); + assert.deepEqual(fixture.coordinator.projection(ROOT.sessionId), { + hostEpoch: 'epoch-1', + queueRevision: 0, + steering: [], + followup: [], + }); + await fixture.coordinator.close(); +}); + test('binds the exact reserved Run after a pre-bind stop fence', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); @@ -795,6 +869,46 @@ test('entry promote moves a follow-up into the steering queue', async () => { assert.equal(fixture.liveResidencies(), 0); }); +test('editing a promoted entry preserves its original submitted placement', async () => { + const fixture = createFixture(); + fixture.coordinator.reserveRootTurn(ROOT); + + await submit(fixture, 'follow-1', 'first', 'next_turn'); + const promoted = await fixture.coordinator.handlers['queue.entry.promote']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + entryId: 'id-1', + promoteId: 'promote-edit', + }, + operationContext(), + ); + assert.equal(promoted.ok, true); + + const updated = await fixture.coordinator.handlers['queue.entry.update']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + entryId: 'id-1', + updateId: 'update-promoted', + expectedQueueRevision: 2, + text: 'edited after promotion', + }, + operationContext(), + ); + assert.equal(updated.ok, true); + const admission = fixture.readMessageAdmission('follow-1'); + assert.ok(admission); + assert.equal(admission.submittedPlacement, 'next_turn'); + assert.equal(admission.placement, 'current_turn'); + assert.equal(admission.disposition, 'steering'); + assert.deepEqual(admission.content, { text: 'edited after promotion' }); + assert.equal( + admission.submittedContentDigest, + messageContentDigest({ text: 'edited after promotion' }), + ); +}); + test('entry promote requires an active Turn', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); @@ -1562,6 +1676,37 @@ test('terminal transition atomically folds messages submitted after run release' fixture.coordinator.completeIdle(empty); }); +test('terminal settlement executes only steering admissions with a provider proof', async () => { + const fixture = createFixture(); + fixture.coordinator.reserveRootTurn(ROOT); + const owner = fixture.coordinator.bindRun(ROOT); + await submit(fixture, 'steer-proved', 'provider must see this', 'current_turn'); + const [lease] = owner.pull(); + assert.ok(lease); + owner.ack([lease.id]); + owner.release(); + fixture.events.push(steeringEvent('steer-proved', 'provider must see this')); + let providerProofAfter = -1; + fixture.setProviderRequestProof((admittedAt) => { + providerProofAfter = admittedAt; + return true; + }); + + await fixture.coordinator.settleMessagesAfterRoot({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + admittedAt: 0, + messageIds: [], + terminalStatus: 'completed', + }); + + assert.equal(fixture.readMessageLifecycleState('steer-proved'), 'executed'); + assert.equal(providerProofAfter, 1); + const batch = fixture.coordinator.beginTerminalTransition(ROOT); + fixture.coordinator.completeIdle(batch); +}); + test('administrative drain preserves accepted entries until the terminal stop fence', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); @@ -2092,6 +2237,7 @@ function createFixture( let drainRequests = 0; let receiptReads = 0; let rootReads = 0; + let providerRequestProof: boolean | ((admittedAt: number) => boolean) = false; let stopDeliveryError: Error | undefined; let prepareMessage: NonNullable = async (input) => ({ kind: 'ready', @@ -2107,6 +2253,13 @@ function createFixture( const receipts = new Map(); const events: RuntimeEvent[] = []; const operationReceipts = new Map(); + const messageAdmissions = new Map< + string, + { + admission: PendingMessageAdmission; + state: 'accepted' | 'handed_off' | 'executed' | 'cancelled'; + } + >(); const receiptDelays = new Map< string, { @@ -2115,6 +2268,7 @@ function createFixture( readonly error?: Error; } >(); + const lifecycle = memoryMessageLifecycleStore(messageAdmissions); const stopClaimed = deferred(); const terminal = deferred(); let coordinator: HostMessageCoordinator; @@ -2184,6 +2338,10 @@ function createFixture( ); return event ? { event } : undefined; }, + readProviderRequestProof: async ({ admittedAt }) => + typeof providerRequestProof === 'function' + ? providerRequestProof(admittedAt) + : providerRequestProof, }, receipts: memoryReceiptStore( operationReceipts, @@ -2199,6 +2357,7 @@ function createFixture( receiptReads += 1; }, ), + lifecycle, sessionAdmission: new SessionAdmissionGate(), acquireResidency: () => { liveResidencies += 1; @@ -2221,6 +2380,7 @@ function createFixture( coordinator = new HostMessageCoordinator(options); return { coordinator, + lifecycle, setRootState: (state: HostMessageRootState) => { rootState = state; }, @@ -2230,12 +2390,17 @@ function createFixture( startCalls: () => startCalls, events, receipts, + readMessageAdmission: (messageId: string) => messageAdmissions.get(messageId)?.admission, + readMessageLifecycleState: (messageId: string) => messageAdmissions.get(messageId)?.state, stopClaimed, resolveTerminal: terminal.resolve, liveResidencies: () => liveResidencies, drainRequests: () => drainRequests, receiptReads: () => receiptReads, rootReads: () => rootReads, + setProviderRequestProof: (proved: boolean | ((admittedAt: number) => boolean)) => { + providerRequestProof = proved; + }, failStopDelivery: (error: Error) => { stopDeliveryError = error; }, @@ -2281,6 +2446,65 @@ function memoryReceiptStore( }; } +function memoryMessageLifecycleStore( + admissions: Map< + string, + { + admission: PendingMessageAdmission; + state: 'accepted' | 'handed_off' | 'executed' | 'cancelled'; + } + >, +): MessageLifecycleStore { + return { + commitMessageAdmission: async (admission) => { + const existing = admissions.get(admission.messageId); + if (existing) return existing.admission; + admissions.set(admission.messageId, { admission, state: 'accepted' }); + return admission; + }, + readMessageAdmission: async (_sessionId, messageId) => admissions.get(messageId)?.admission, + readMessageLifecycleState: async (_sessionId, messageId) => admissions.get(messageId)?.state, + listMessageAdmissions: async (sessionId) => + [...admissions.values()] + .filter(({ admission, state }) => admission.sessionId === sessionId && state === 'accepted') + .map(({ admission }) => admission), + listUnsettledMessageAdmissions: async (sessionId) => + [...admissions.values()] + .filter( + ({ admission, state }) => + admission.sessionId === sessionId && (state === 'accepted' || state === 'handed_off'), + ) + .map(({ admission }) => admission), + rebindMessageAdmissionTranscript: async () => undefined, + updateMessageAdmission: async (admission) => { + const existing = admissions.get(admission.messageId); + if (!existing) throw new Error(`Missing admission ${admission.messageId}`); + existing.admission = admission; + }, + reorderMessageAdmissions: async () => undefined, + cancelMessageAdmissions: async (_sessionId, messageIds) => { + for (const messageId of messageIds) { + const existing = admissions.get(messageId); + if (existing && (existing.state === 'accepted' || existing.state === 'handed_off')) { + existing.state = 'cancelled'; + } + } + }, + markMessagesHandedOff: async (_sessionId, messageIds) => { + for (const messageId of messageIds) { + const existing = admissions.get(messageId); + if (existing?.state === 'accepted') existing.state = 'handed_off'; + } + }, + markMessagesExecuted: async (_sessionId, messageIds) => { + for (const messageId of messageIds) { + const existing = admissions.get(messageId); + if (existing?.state === 'handed_off') existing.state = 'executed'; + } + }, + }; +} + function submit( fixture: ReturnType, messageId: string, diff --git a/packages/runtime-host/src/__tests__/root-admission-owner.test.ts b/packages/runtime-host/src/__tests__/root-admission-owner.test.ts index fdc0552dba..2502955ae0 100644 --- a/packages/runtime-host/src/__tests__/root-admission-owner.test.ts +++ b/packages/runtime-host/src/__tests__/root-admission-owner.test.ts @@ -325,7 +325,7 @@ function multiSourceAdmitInput(sessionId: string, turnId: string, admittedAt: nu sessionId, turnId, proposedRunId: `run-${turnId}`, - proposedUserMessageId: `message-${turnId}`, + proposedUserMessageId: null, execution: { kind: 'external_message' as const }, normalizedInput: { text: 'model text\n\nfollowup text', diff --git a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts index aad3fdb039..0fc991a4b4 100644 --- a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts @@ -812,6 +812,66 @@ test('idle turn.message.submit applies hosted Skill preparation before durable a } }); +test('idle Skill admission persists only canonical content before root handoff', async () => { + const canonicalText = 'Write clearly.\n\nDraft this.'; + const fixture = await createFailureFixture({ + registerBackend: (backends) => + backends.register('ai-sdk', (context) => new FakeBackend(context)), + prepareSkillInvocation: async (): Promise => ({ + disposition: 'ready', + sendText: canonicalText, + skillInvocation: { + loaded: [{ id: 'writer', name: 'Writer' }], + failed: [], + receipts: [], + }, + }), + wrapAdmissionStore: (store) => ({ + admitRootTurn: async () => { + throw new Error('injected root admission failure'); + }, + readRootTurnAdmission: (sessionId, turnId) => store.readRootTurnAdmission(sessionId, turnId), + readRootTurnSourceMessageReceipt: (sessionId, messageId) => + store.readRootTurnSourceMessageReceipt(sessionId, messageId), + listRootTurnAdmissionsForRecovery: (sessionId) => + store.listRootTurnAdmissionsForRecovery(sessionId), + }), + }); + try { + await assert.rejects( + fixture.messages.handlers['turn.message.submit']( + { + originHostEpoch: fixture.hostEpoch, + sessionId: fixture.sessionId, + messageId: 'idle-skill-before-handoff', + content: { text: '/skill:writer Draft this.' }, + placement: 'current_turn', + }, + operationContext(fixture.hostEpoch, fixture.acquireResidency), + ), + /injected root admission failure/, + ); + const admission = await fixture.stores.sessionStore.readMessageAdmission( + fixture.sessionId, + 'idle-skill-before-handoff', + ); + assert.deepEqual(admission?.content, { + text: canonicalText, + displayText: '/skill:writer Draft this.', + inlineReferences: [], + }); + assert.deepEqual( + (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).map((message) => ({ + text: message.type === 'user' ? message.text : undefined, + displayText: message.type === 'user' ? message.displayText : undefined, + })), + [{ text: canonicalText, displayText: '/skill:writer Draft this.' }], + ); + } finally { + await fixture.dispose(); + } +}); + test('turn.start rejects oversized preparation before admission and preserves not-found semantics', async () => { let preparationCount = 0; let preparation: 'blocked' | 'oversized_content' | 'oversized_feedback' = 'blocked'; @@ -2159,8 +2219,8 @@ test('hosted linked child roots share admission, message, terminal, and stop aut readRootState: (sessionId) => requireCoordinator(coordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStopFence(input, commitQueueFence, admission), - startFromMessage: (input, admission) => - requireCoordinator(coordinator).startFromMessage(input, admission), + startFromMessage: (input, admission, commitAdmission) => + requireCoordinator(coordinator).startFromMessage(input, admission, commitAdmission), prepareMessage: (input) => requireCoordinator(coordinator).prepareMessage(input), claimStop: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStop(input, commitQueueFence, admission), @@ -2175,8 +2235,10 @@ test('hosted linked child roots share admission, message, terminal, and stop aut stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission, acquireResidency, requestDrain: () => { @@ -4778,8 +4840,8 @@ async function createFailureFixture(options: { readRootState: (sessionId) => requireCoordinator(coordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStopFence(input, commitQueueFence, admission), - startFromMessage: (input, admission) => - requireCoordinator(coordinator).startFromMessage(input, admission), + startFromMessage: (input, admission, commitAdmission) => + requireCoordinator(coordinator).startFromMessage(input, admission, commitAdmission), prepareMessage: (input) => requireCoordinator(coordinator).prepareMessage(input), claimStop: (input, commitQueueFence, admission) => requireCoordinator(coordinator).claimStop(input, commitQueueFence, admission), @@ -4799,8 +4861,10 @@ async function createFailureFixture(options: { stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async () => false, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission, acquireResidency, requestDrain, diff --git a/packages/runtime-host/src/server/client-capability-coordinator.ts b/packages/runtime-host/src/server/client-capability-coordinator.ts index 40813af1b8..09e18cd565 100644 --- a/packages/runtime-host/src/server/client-capability-coordinator.ts +++ b/packages/runtime-host/src/server/client-capability-coordinator.ts @@ -21,6 +21,7 @@ import { createHash } from 'node:crypto'; import { AsyncLocalStorage } from 'node:async_hooks'; import { buildMcpTools, mcpProxyToolName, type McpToolProvider } from '@maka/runtime/mcp-tools'; import { type MakaTool } from '@maka/runtime/tool-runtime'; +import type { RootExecutionDescriptor } from '@maka/core/agent-run'; import { type ToolGroup } from '@maka/runtime/tool-availability'; import { type ClientCapabilityOffer, @@ -241,6 +242,26 @@ export class HostClientCapabilityCoordinator implements ClientCapabilityService } } + /** Rebuild Session-scoped capability bindings from a durable root contract. */ + async bindDurableRoot(input: { + sessionId: string; + userMessageId: string | null; + execution: RootExecutionDescriptor; + }): Promise { + if (input.execution.kind !== 'external_message' || input.userMessageId === null) return; + // A live root already selected its Client capabilities at admission. Only + // cold recovery needs to rebuild a missing in-memory binding from the + // durable root contract; reselecting here would discard the active + // connection/turn-affine binding and can make providers ambiguous. + if (this.#sessions.has(input.sessionId)) return; + await this.#activation.runMutation(async () => { + const selection = this.#selectSessionState(input.sessionId, '', 'degrade'); + if (!selection.ok) throw new Error(selection.message); + this.#storeSessionState(input.sessionId, selection.state); + if (selection.modelToolsChanged) this.#onModelToolsChanged(); + }); + } + async #bindSession( sessionId: string, initiatingConnectionId: string, diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 639a8beb7e..64b02d3364 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -466,8 +466,10 @@ export async function createExecutionRuntimeHostComposition( requireRootCoordinator(rootCoordinator).readRootState(sessionId), claimStopFence: (input, commitQueueFence, admission) => requireRootCoordinator(rootCoordinator).claimStopFence(input, commitQueueFence, admission), - startFromMessage: (input, admission) => - requireRootCoordinator(rootCoordinator).startFromMessage(input, admission), + startFromMessage: (input, admission, commitAdmission) => + requireRootCoordinator(rootCoordinator).startFromMessage(input, admission, commitAdmission), + startRecoveredMessages: (input, admission) => + requireRootCoordinator(rootCoordinator).startRecoveredMessages(input, admission), prepareMessage: (input) => requireRootCoordinator(rootCoordinator).prepareMessage(input), claimStop: (input, commitQueueFence, admission) => requireRootCoordinator(rootCoordinator).claimStop(input, commitQueueFence, admission), @@ -480,8 +482,18 @@ export async function createExecutionRuntimeHostComposition( stores.agentRunStore.readRootTurnSourceMessageReceipt(sessionId, messageId), readImmutableSteeringMessageProof: (sessionId, messageId) => stores.runtimeEventStore.readImmutableSteeringMessageProof(sessionId, messageId), + readProviderRequestProof: async ({ sessionId, turnId, runId, admittedAt }) => { + const events = await stores.agentRunStore.readEvents(sessionId, runId); + return events.some( + (event) => + event.turnId === turnId && + event.ts >= admittedAt && + event.type === 'model_call_attempt_recorded', + ); + }, }, receipts: stores.messageReceiptStore, + lifecycle: stores.sessionStore, sessionAdmission, acquireResidency: () => context.acquireResidency('message-queue'), requestDrain: context.requestDrain, @@ -1485,6 +1497,9 @@ export async function createExecutionRuntimeHostComposition( ), ); await coordinator.recover(); + await messages.recoverPendingAfterHostRestart( + recoverySessions.map((session) => session.id), + ); rootRecoveryCompleted = true; }, }, diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 5187583b16..6de7a9a061 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -69,7 +69,7 @@ export async function prepareHostedExecutionRecovery( const run = runsById.get(admission.runId); const rootUserMessages = ( messageIndex.userMessagesByTurnId.get(admission.turnId) ?? [] - ).filter((message) => message.steeringEventId === undefined); + ).filter((message) => message.id === admission.userMessageId); const messageIdOwners = admission.userMessageId ? (messageIndex.messagesById.get(admission.userMessageId) ?? []) : []; @@ -87,12 +87,18 @@ export async function prepareHostedExecutionRecovery( `Admitted Turn ${admission.turnId} has queue-independent execution with Message queue sources`, ); } - if (executionContract.requiresUserMessage !== (admission.userMessageId !== null)) { + const requiresUserMessage = + executionContract.requiresUserMessage && + !(admission.execution.kind === 'external_message' && admission.sourceMessages.length > 1); + if (requiresUserMessage !== (admission.userMessageId !== null)) { throw new Error( `Admitted Turn ${admission.turnId} has an invalid UserMessage execution contract`, ); } if (admission.userMessageId === null) { + if (admission.sourceMessages.length > 0) { + verifyQueueSourceMessages(admission, messageIndex); + } if (rootUserMessages.length > 0) { throw new Error(`Admitted Turn ${admission.turnId} must not record a UserMessage`); } @@ -145,7 +151,16 @@ export async function prepareHostedExecutionRecovery( continue; } if (!run) { - if (rootUserMessages.length > 0 || messageIdOwner) { + if (executionContract.pendingWithoutRun === 'root_replay') { + verifyOrRecoverUserMessage( + admission, + rootUserMessages, + messageIdOwner, + missingMessages, + messageIndex, + false, + ); + } else if (rootUserMessages.length > 0 || messageIdOwner) { throw new Error(`Admitted Turn ${admission.turnId} has a UserMessage but no Run`); } replayAdmissions.push(admission); @@ -263,6 +278,7 @@ function verifyOrRecoverUserMessage( messageIdOwner: StoredMessage | undefined, missingMessages: RecoveryUserMessage[], index: RecoveryMessageIndex, + materializeMissing = true, ): void { if (rootUserMessages.length > 1) { throw new Error(`Admitted Turn ${admission.turnId} has multiple UserMessages`); @@ -285,11 +301,31 @@ function verifyOrRecoverUserMessage( if (messageIdOwner) { throw new Error(`Admitted Turn ${admission.turnId} reuses another message identity`); } + if (!materializeMissing) return; const recoveredMessage = recoveryUserMessage(admission); missingMessages.push(recoveredMessage); indexRecoveryMessage(index, recoveredMessage); } +function verifyQueueSourceMessages( + admission: RootTurnAdmission, + index: RecoveryMessageIndex, +): void { + for (const source of admission.sourceMessages) { + const owners = index.messagesById.get(source.messageId) ?? []; + if ( + owners.length !== 1 || + owners[0]?.type !== 'user' || + owners[0].turnId !== admission.turnId || + !messageContentsEqual(normalizeMessageContent(owners[0]), source.content) + ) { + throw new Error( + `Admitted Turn ${admission.turnId} does not match queue source ${source.messageId}`, + ); + } + } +} + function recoveryUserMessage(admission: RootTurnAdmission): RecoveryUserMessage { if (!admission.userMessageId || !admission.normalizedInput) { throw new Error(`Admitted Turn ${admission.turnId} does not own a UserMessage`); diff --git a/packages/runtime-host/src/server/message-content-digest.ts b/packages/runtime-host/src/server/message-content-digest.ts deleted file mode 100644 index 03f88d598a..0000000000 --- a/packages/runtime-host/src/server/message-content-digest.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { createHash } from 'node:crypto'; -import { normalizeMessageContent, type MessageContent } from '@maka/core/events'; - -export function messageContentDigest(content: MessageContent): `sha256:${string}` { - return `sha256:${createHash('sha256') - .update(JSON.stringify(canonicalize(normalizeMessageContent(content)))) - .digest('hex')}`; -} - -function canonicalize(value: unknown): unknown { - if (Array.isArray(value)) return value.map(canonicalize); - if (value === null || typeof value !== 'object') return value; - return Object.fromEntries( - Object.entries(value) - .filter(([, entry]) => entry !== undefined) - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([key, entry]) => [key, canonicalize(entry)]), - ); -} diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index 8d8ac47e9b..c72595bf92 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -22,6 +22,7 @@ import { isDeepStrictEqual } from 'node:util'; import type { SteeringLease } from '@maka/core/backend-types'; import { aggregateMessageContents, + messageContentDigest, messageContentsEqual, normalizeMessageContent, type MessageContent, @@ -36,8 +37,10 @@ import { import { normalizeRootTurnAdmissionPayload, type ImmutableSteeringMessageProof, + type MessageLifecycleStore, type MessageReceiptOperation, type MessageReceiptStore, + type PendingMessageAdmission, type RootTurnSourceMessage, type RootTurnSourceMessageReceipt, } from '@maka/storage/execution-stores'; @@ -70,7 +73,6 @@ import type { RuntimeHostResidency } from './host-kernel.js'; import { worstCaseFailedTurnSnapshot } from './canonical-turn-snapshot.js'; import { worstCaseMessageQueueProjection } from './message-queue-capacity.js'; import type { ConnectionContext, MessageOperationHandlerMap } from './operation-dispatcher.js'; -import { messageContentDigest } from './message-content-digest.js'; import { type SessionAdmissionLease, SessionAdmissionGate } from './session-admission-gate.js'; type MessageOperationErrorCode = @@ -104,6 +106,15 @@ export interface HostMessageStartInput { readonly content: MessageContent; readonly sourceMessage: RootTurnSourceMessage; readonly initiatingConnectionId: string; + readonly turnId?: string; + readonly runId?: string; +} + +export interface HostMessageRecoveryBatch { + readonly sessionId: string; + readonly content: MessageContent; + readonly submittedContent: MessageContent; + readonly sources: readonly RootTurnSourceMessage[]; } export interface HostMessagePreparationInput { @@ -136,6 +147,11 @@ export interface HostMessageRootPort { startFromMessage( input: HostMessageStartInput, admission: SessionAdmissionLease, + commitAdmission: (canonicalContent: MessageContent) => Promise, + ): Promise<{ readonly turnId: string } | { readonly error: string }>; + startRecoveredMessages?( + input: HostMessageRecoveryBatch, + admission: SessionAdmissionLease, ): Promise<{ readonly turnId: string } | { readonly error: string }>; prepareMessage( input: HostMessagePreparationInput, @@ -160,6 +176,13 @@ export interface HostMessageDurableProofReader { sessionId: string, messageId: string, ): Promise; + /** True only when the admitted root has a durable downstream provider proof. */ + readProviderRequestProof(input: { + sessionId: string; + turnId: string; + runId: string; + admittedAt: number; + }): Promise; } export interface HostMessageCoordinatorOptions { @@ -167,6 +190,7 @@ export interface HostMessageCoordinatorOptions { readonly root: HostMessageRootPort; readonly durableProof: HostMessageDurableProofReader; readonly receipts: MessageReceiptStore; + readonly lifecycle: MessageLifecycleStore; readonly sessionAdmission: SessionAdmissionGate; readonly acquireResidency: () => RuntimeHostResidency; readonly requestDrain?: () => void; @@ -186,8 +210,12 @@ export type CandidateSnapshotPreflight = ( interface LiveEntry { readonly entryId: string; readonly messageId: string; + readonly turnId: string; + readonly runId: string; + readonly admittedAt: number; content: MessageContent; modelContent: MessageContent; + submittedContentDigest: `sha256:${string}`; readonly initiatingConnectionId: string; readonly placement: MessagePlacement; readonly disposition: 'steering' | 'followup'; @@ -310,6 +338,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { readonly #root: HostMessageRootPort; readonly #durableProof: HostMessageDurableProofReader; readonly #receipts: MessageReceiptStore; + readonly #lifecycle: MessageLifecycleStore; readonly #sessionAdmission: SessionAdmissionGate; readonly #acquireResidency: () => RuntimeHostResidency; readonly #requestDrain: () => void; @@ -330,6 +359,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { this.#root = options.root; this.#durableProof = options.durableProof; this.#receipts = options.receipts; + this.#lifecycle = options.lifecycle; this.#sessionAdmission = options.sessionAdmission; this.#acquireResidency = options.acquireResidency; this.#requestDrain = options.requestDrain ?? (() => undefined); @@ -503,6 +533,248 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { this.#draining = true; } + /** + * Commit the root-admission proof before Runtime activation. The in-memory + * queue never owns this transition: it only projects the durable result. + */ + async handoffRootSources(input: { + sessionId: string; + turnId: string; + runId: string; + previousRootTurnId: string | null; + messageIds: readonly string[]; + }): Promise { + const handoff: string[] = []; + for (const messageId of new Set(input.messageIds)) { + const proof = await this.#durableProof.readRootTurnSourceMessageReceipt( + input.sessionId, + messageId, + ); + if ( + !proof || + proof.admission.turnId !== input.turnId || + proof.admission.runId !== input.runId || + proof.sourceMessage.messageId !== messageId + ) { + throw new RuntimeMessageAuthorityInvariantError( + `Root admission does not prove Message handoff ${messageId}`, + ); + } + const state = await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId); + if (state === 'accepted') handoff.push(messageId); + else if (state === 'handed_off' || state === 'executed') continue; + else { + throw new RuntimeMessageAuthorityInvariantError( + `Message ${messageId} cannot be handed off from lifecycle state ${state ?? 'missing'}`, + ); + } + } + await this.#lifecycle.rebindMessageAdmissionTranscript({ + sessionId: input.sessionId, + messageIds: [...new Set(input.messageIds)], + turnId: input.turnId, + previousRootTurnId: input.previousRootTurnId, + }); + await this.#lifecycle.markMessagesHandedOff(input.sessionId, handoff); + } + + /** + * One settlement owner for both the normal terminal path and Host recovery. + * A queued message becomes Executed only after the durable root has recorded + * a provider request downstream of its admitted root contract. + */ + async settleMessagesAfterRoot(input: { + sessionId: string; + turnId: string; + runId: string; + admittedAt: number; + messageIds: readonly string[]; + terminalStatus?: 'completed' | 'failed' | 'cancelled'; + }): Promise { + const messageIds = new Set(); + const providerProofAfter = new Map(); + const admissions = await this.#lifecycle.listUnsettledMessageAdmissions(input.sessionId); + for (const messageId of new Set(input.messageIds)) { + const proof = await this.#durableProof.readRootTurnSourceMessageReceipt( + input.sessionId, + messageId, + ); + if ( + proof?.admission.turnId === input.turnId && + proof.admission.runId === input.runId && + proof.sourceMessage.messageId === messageId + ) { + messageIds.add(messageId); + } + } + for (const admission of admissions) { + if ( + admission.turnId !== input.turnId || + admission.runId !== input.runId || + admission.disposition !== 'steering' + ) { + continue; + } + const proof = await this.#durableProof.readImmutableSteeringMessageProof( + input.sessionId, + admission.messageId, + ); + if (proof?.event.turnId === input.turnId && proof.event.runId === input.runId) { + messageIds.add(admission.messageId); + providerProofAfter.set(admission.messageId, proof.event.ts); + } + } + const handedOff: string[] = []; + for (const messageId of messageIds) { + if ( + (await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId)) === 'accepted' + ) { + handedOff.push(messageId); + } + } + await this.#lifecycle.markMessagesHandedOff(input.sessionId, handedOff); + const proved: string[] = []; + for (const messageId of messageIds) { + if ( + await this.#durableProof.readProviderRequestProof({ + ...input, + admittedAt: providerProofAfter.get(messageId) ?? input.admittedAt, + }) + ) { + proved.push(messageId); + } + } + if (proved.length > 0) { + const executed: string[] = []; + for (const messageId of proved) { + if ( + (await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId)) === + 'handed_off' + ) { + executed.push(messageId); + } + } + await this.#lifecycle.markMessagesExecuted(input.sessionId, executed); + } + if (input.terminalStatus !== 'cancelled') return; + const cancelled: string[] = []; + for (const messageId of messageIds) { + const state = await this.#lifecycle.readMessageLifecycleState(input.sessionId, messageId); + if (state === 'accepted' || state === 'handed_off') cancelled.push(messageId); + } + await this.#lifecycle.cancelMessageAdmissions(input.sessionId, cancelled); + } + + async cancelMessages(sessionId: string, messageIds: readonly string[]): Promise { + await this.#lifecycle.cancelMessageAdmissions(sessionId, messageIds); + } + + async recoverPendingAfterHostRestart(sessionIds: readonly string[]): Promise { + for (const sessionId of sessionIds) { + const admissions = await this.#lifecycle.listMessageAdmissions(sessionId); + const unsettled = await this.#lifecycle.listUnsettledMessageAdmissions(sessionId); + if (unsettled.length === 0) continue; + const acceptedIds = new Set(admissions.map((admission) => admission.messageId)); + const pending = [] as PendingMessageAdmission[]; + for (const admission of unsettled) { + const source = await this.#durableProof.readRootTurnSourceMessageReceipt( + sessionId, + admission.messageId, + ); + if ( + source?.admission.turnId === admission.turnId && + source.admission.runId === admission.runId && + source.sourceMessage.messageId === admission.messageId + ) { + await this.settleMessagesAfterRoot({ + sessionId, + turnId: source.admission.turnId, + runId: source.admission.runId, + admittedAt: source.admission.admittedAt, + messageIds: [admission.messageId], + }); + } else { + const steering = await this.#durableProof.readImmutableSteeringMessageProof( + sessionId, + admission.messageId, + ); + if ( + steering?.event.turnId === admission.turnId && + steering.event.runId === admission.runId + ) { + await this.settleMessagesAfterRoot({ + sessionId, + turnId: admission.turnId, + runId: admission.runId, + admittedAt: admission.admittedAt, + messageIds: [admission.messageId], + }); + } else if (acceptedIds.has(admission.messageId)) { + pending.push(admission); + } + } + } + if (pending.length === 0) continue; + const rootState = await this.#root.readRootState(sessionId); + if (rootState.kind !== 'active') { + if (rootState.kind !== 'idle') continue; + if (!this.#root.startRecoveredMessages) { + throw new RuntimeMessageAuthorityInvariantError( + 'Message recovery authority is unavailable', + ); + } + const started = await this.#sessionAdmission.run(sessionId, (admission) => + this.#root.startRecoveredMessages!( + { + sessionId, + content: aggregateMessageContents(pending.map((entry) => entry.content)), + submittedContent: aggregateMessageContents(pending.map((entry) => entry.content)), + sources: pending.map(pendingMessageSource), + }, + admission, + ), + ); + if ('error' in started) { + throw new RuntimeMessageAuthorityInvariantError( + `Durable Message recovery failed: ${started.error}`, + ); + } + continue; + } + if (!this.#sessions.has(sessionId)) this.#state(sessionId); + const state = this.#requireState(sessionId); + if (!state.reservedRoot) this.reserveRootTurn(rootState); + if (!sameRun(state.reservedRoot!, rootState)) continue; + for (const admission of admissions) { + if (admission.turnId !== rootState.turnId || admission.runId !== rootState.runId) continue; + const existing = allLiveEntries(state).find( + (entry) => entry.messageId === admission.messageId, + ); + if (existing) continue; + const residency = this.#acquireResidency(); + const entry: LiveEntry = { + entryId: this.#createId(), + messageId: admission.messageId, + turnId: admission.turnId, + runId: admission.runId, + admittedAt: admission.admittedAt, + content: submittedProjectionContent(admission.content), + modelContent: admission.content, + submittedContentDigest: admission.submittedContentDigest, + initiatingConnectionId: '', + placement: admission.placement, + disposition: admission.disposition, + generation: state.generation, + residency, + state: 'queued', + }; + if (entry.disposition === 'steering') state.steering.push(entry); + else state.followup.push(entry); + this.#mutated(state); + } + } + } + commitStopFence(identity: RuntimeMessageRunIdentity): QueueFenceResult { return this.#commitQueueFence(identity); } @@ -621,14 +893,43 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { placement: input.placement, disposition: 'turn_started', }; + const pendingAdmission = await this.#lifecycle.readMessageAdmission( + input.sessionId, + input.messageId, + ); + if ( + pendingAdmission && + (pendingAdmission.submittedContentDigest !== messageContentDigest(payload.content) || + pendingAdmission.submittedPlacement !== input.placement) + ) { + return failure('operation_conflict', 'Message admission has a different payload'); + } + const turnId = pendingAdmission?.turnId ?? this.#createId(); + const runId = pendingAdmission?.runId ?? this.#createId(); const started = await this.#root.startFromMessage( { sessionId: input.sessionId, content: payload.content, sourceMessage, initiatingConnectionId, + turnId, + runId, }, admission, + async (canonicalContent) => { + await this.#lifecycle.commitMessageAdmission({ + sessionId: input.sessionId, + turnId, + runId, + messageId: input.messageId, + content: canonicalContent, + submittedContentDigest: messageContentDigest(payload.content), + submittedPlacement: input.placement, + placement: 'current_turn', + disposition: 'steering', + admittedAt: pendingAdmission?.admittedAt ?? Date.now(), + }); + }, ); if ('error' in started) { return failure('operation_conflict', started.error); @@ -734,12 +1035,29 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { continue; } const result = { disposition, queueRevision: candidateRevision + 1 } as const; + const messageAdmission: PendingMessageAdmission = { + sessionId: input.sessionId, + turnId: rootState.turnId, + runId: rootState.runId, + messageId: input.messageId, + content: prepared.content, + submittedContentDigest: messageContentDigest(payload.content), + submittedPlacement: input.placement, + placement: input.placement, + disposition, + admittedAt: Date.now(), + }; + await this.#lifecycle.commitMessageAdmission(messageAdmission); const residency = this.#acquireResidency(); const entry: LiveEntry = { entryId, messageId: input.messageId, + turnId: rootState.turnId, + runId: rootState.runId, + admittedAt: messageAdmission.admittedAt, content: payload.content, modelContent: prepared.content, + submittedContentDigest: messageAdmission.submittedContentDigest, initiatingConnectionId, placement: input.placement, disposition, @@ -794,6 +1112,10 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { queueRevision: state.revision + (queued.length > 0 ? 1 : 0), retracted: queued.map(retractedSnapshot), }; + await this.#lifecycle.cancelMessageAdmissions( + input.sessionId, + queued.map((entry) => entry.messageId), + ); const retracted = this.#retractQueued(state); if (retracted.length > 0) this.#mutated(state); if (!isDeepStrictEqual(result, { queueRevision: state.revision, retracted })) { @@ -970,6 +1292,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } return failure('not_found', 'Message queue entry does not exist'); } + await this.#lifecycle.cancelMessageAdmissions(input.sessionId, [queued.entry.messageId]); queued.remove(); this.#releaseEntry(queued.entry); this.#mutated(state); @@ -1023,6 +1346,18 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } return failure('not_found', 'Message queue entry does not exist'); } + await this.#lifecycle.updateMessageAdmission({ + sessionId: input.sessionId, + turnId: entry.turnId, + runId: entry.runId, + messageId: entry.messageId, + content: entry.modelContent, + submittedContentDigest: entry.submittedContentDigest, + submittedPlacement: 'next_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt: entry.admittedAt, + }); state.followup.splice(index, 1); state.steering.push({ ...entry, placement: 'current_turn', disposition: 'steering' }); this.#mutated(state); @@ -1113,8 +1448,25 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { ) { return failure('session_busy', 'Message queue changed during update'); } + const admission = await this.#lifecycle.readMessageAdmission( + input.sessionId, + queued.entry.messageId, + ); + await this.#lifecycle.updateMessageAdmission({ + sessionId: input.sessionId, + turnId: queued.entry.turnId, + runId: queued.entry.runId, + messageId: queued.entry.messageId, + content: modelContent, + submittedContentDigest: messageContentDigest(content), + submittedPlacement: admission?.submittedPlacement ?? queued.entry.placement, + placement: queued.entry.placement, + disposition: queued.entry.disposition, + admittedAt: queued.entry.admittedAt, + }); queued.entry.content = content; queued.entry.modelContent = modelContent; + queued.entry.submittedContentDigest = messageContentDigest(content); this.#mutated(state); const result = { queueRevision: state.revision }; try { @@ -1153,6 +1505,10 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { reordered.push(entry); } if (reordered.some((entry, index) => current[index] !== entry)) { + await this.#lifecycle.reorderMessageAdmissions( + input.sessionId, + reordered.map((entry) => entry.messageId), + ); state.followup = reordered; this.#mutated(state); } @@ -1478,7 +1834,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { id: leaseId, messageId: entry.messageId, content: normalizeMessageContent(entry.modelContent), - submittedContentDigest: messageContentDigest(entry.content), + submittedContentDigest: entry.submittedContentDigest, }; }); this.#mutated(state); @@ -1833,12 +2189,28 @@ function sourceFromEntry(entry: LiveEntry): RootFollowupSource { return { messageId: entry.messageId, content: normalizeMessageContent(entry.modelContent), - submittedContentDigest: messageContentDigest(entry.content), + submittedContentDigest: entry.submittedContentDigest, placement: entry.placement, disposition: entry.disposition, }; } +function pendingMessageSource(admission: PendingMessageAdmission): RootTurnSourceMessage { + return { + messageId: admission.messageId, + content: normalizeMessageContent(admission.content), + submittedContentDigest: admission.submittedContentDigest, + placement: admission.placement, + disposition: admission.disposition, + }; +} + +function submittedProjectionContent(content: MessageContent): MessageContent { + const normalized = normalizeMessageContent(content); + const text = normalized.displayText ?? normalized.text; + return normalizeMessageContent({ ...normalized, text, displayText: text }); +} + function queuedSnapshot(entry: LiveEntry): QueuedMessageSnapshot { return { entryId: entry.entryId, @@ -1979,7 +2351,10 @@ function canonicalFollowupBatch(entries: readonly LiveEntry[]): { function sameInitiatingClientPrefix(entries: readonly LiveEntry[]): LiveEntry[] { const initiatingConnectionId = entries[0]?.initiatingConnectionId; - if (!initiatingConnectionId) return []; + if (!initiatingConnectionId) { + const boundary = entries.findIndex((entry) => entry.initiatingConnectionId !== ''); + return entries.slice(0, boundary === -1 ? entries.length : boundary); + } const boundary = entries.findIndex( (entry) => entry.initiatingConnectionId !== initiatingConnectionId, ); diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index c05a8a019d..709e4facf1 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -23,6 +23,7 @@ import type { BackendStopMode } from '@maka/core/backend-types'; import type { AgentRunHeader, RootExecutionDescriptor } from '@maka/core/agent-run'; import { INLINE_REFERENCE_MAX_COUNT, + messageContentDigest, messageContentsEqual, normalizeMessageContent, type AttachmentRef, @@ -79,6 +80,7 @@ import type { HostInteractionCoordinator } from './interaction-coordinator.js'; import { type HostMessageRootState, type HostMessagePreparationInput, + type HostMessageRecoveryBatch, type HostMessageSessionHeader, type HostMessageStartInput, type HostMessageStopClaim, @@ -87,7 +89,6 @@ import { type QueueFenceResult, type RootFollowupBatch, } from './message-coordinator.js'; -import { messageContentDigest } from './message-content-digest.js'; import type { ConnectionContext, TurnOperationHandlerMap } from './operation-dispatcher.js'; import { RootAdmissionOwner } from './root-admission-owner.js'; import { type SessionAdmissionLease, SessionAdmissionGate } from './session-admission-gate.js'; @@ -360,7 +361,18 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { admission.runId, run, ); - if (!isTerminalSnapshot(snapshot)) { + if (isTerminalSnapshot(snapshot)) { + if (admission.sourceMessages.length > 0) { + await this.messages.settleMessagesAfterRoot({ + sessionId, + turnId: admission.turnId, + runId: admission.runId, + admittedAt: admission.admittedAt, + messageIds: admission.sourceMessages.map((source) => source.messageId), + terminalStatus: snapshot.status, + }); + } + } else { if (admission.execution.kind !== 'safe_boundary_continuation') { throw new Error(`Startup recovery left Turn ${admission.turnId} non-terminal`); } @@ -396,6 +408,13 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { `Unable to recover admitted Turn ${admission.turnId}: ${continuation.plan.reason}`, ); } + await this.messages.handoffRootSources({ + sessionId, + turnId: admission.turnId, + runId: admission.runId, + previousRootTurnId: admission.previousRootTurnId, + messageIds: admission.sourceMessages.map((source) => source.messageId), + }); return this.prepareAdmittedTurn( input, admission, @@ -981,6 +1000,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { startFromMessage( input: HostMessageStartInput, admissionLease: SessionAdmissionLease, + commitAdmission: (canonicalContent: MessageContent) => Promise, ): Promise<{ readonly turnId: string } | { readonly error: string }> { return this.runCommand(async () => { const content = normalizeMessageContent(input.content); @@ -1003,7 +1023,8 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { const header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId); const unavailableReason = runtimeHostExternalTurnUnavailableReason(header); if (unavailableReason) return { error: unavailableReason }; - const turnId = randomUUID(); + const turnId = input.turnId ?? randomUUID(); + const runId = input.runId ?? randomUUID(); const hasSkillInvocation = parseSkillInvocationTokens(content.text).length > 0; const prepared = hasSkillInvocation ? await this.prepareHostedSkillInvocationContent( @@ -1036,11 +1057,12 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } await this.prepareFreshAgentGraphEpoch(header); + await commitAdmission(canonicalContent.content); const admitted = await this.rootAdmissionOwner.admitRootTurn({ sessionId: input.sessionId, turnId, - proposedRunId: randomUUID(), + proposedRunId: runId, proposedUserMessageId: input.sourceMessage.messageId, execution: { kind: 'external_message', @@ -1060,6 +1082,13 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { 'Fresh Message root Turn identity already existed', ); } + await this.messages.handoffRootSources({ + sessionId: input.sessionId, + turnId, + runId, + previousRootTurnId: admitted.admission.previousRootTurnId, + messageIds: [input.sourceMessage.messageId], + }); const disposition = await this.prepareAdmittedTurn( { sessionId: input.sessionId, @@ -1085,6 +1114,64 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { }); } + startRecoveredMessages( + input: HostMessageRecoveryBatch, + admissionLease: SessionAdmissionLease, + ): Promise<{ readonly turnId: string } | { readonly error: string }> { + return this.runCommand(async () => { + if (this.#executions.has(input.sessionId)) { + return { error: 'A root Turn is still active' }; + } + const header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId); + const unavailableReason = runtimeHostExternalTurnUnavailableReason(header); + if (unavailableReason) return { error: unavailableReason }; + const reservation = this.reserveRootTurn(input.sessionId); + if (!reservation) return { error: 'Another root Turn is being admitted' }; + try { + const turnId = randomUUID(); + const admitted = await this.rootAdmissionOwner.admitRootTurn({ + sessionId: input.sessionId, + turnId, + proposedRunId: randomUUID(), + proposedUserMessageId: input.sources.length === 1 ? input.sources[0]!.messageId : null, + execution: { + kind: 'external_message', + inputDigest: messageContentDigest(input.submittedContent), + }, + normalizedInput: input.content, + sourceMessages: input.sources, + admittedAt: Date.now(), + }); + if (admitted.kind !== 'admitted') { + return { error: 'Recovered Message root identity already existed' }; + } + await this.messages.handoffRootSources({ + sessionId: input.sessionId, + turnId, + runId: admitted.admission.runId, + previousRootTurnId: admitted.admission.previousRootTurnId, + messageIds: input.sources.map((source) => source.messageId), + }); + const disposition = await this.prepareAdmittedTurn( + { sessionId: input.sessionId, turnId, content: input.content }, + admitted.admission, + this.acquireRecoveryResidency, + admissionLease, + undefined, + undefined, + reservation, + ); + if (disposition.kind !== 'await_start') { + return { error: 'Recovered Message root did not reserve execution' }; + } + return { turnId }; + } catch (error) { + this.#admissions.release(reservation); + throw error; + } + }); + } + prepareMessage( input: HostMessagePreparationInput, ): Promise< @@ -1810,7 +1897,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (active.startSettled.phase === 'rejected') { return { active, deliverStop: () => Promise.resolve() }; } - commitQueueFence(); + const fence = commitQueueFence(); + await this.messages.cancelMessages( + input.sessionId, + fence.retracted.map((message) => message.messageId), + ); await this.interactions.claimRunClosure(input, 'turn_stopped', admission); const shouldDeliverStop = !active.stopRequested; active.stopRequested = true; @@ -1845,7 +1936,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { const active = this.#executions.get(input.sessionId); if (isTerminalSnapshot(snapshot)) { if (active?.turnId === input.turnId && active.runId === input.runId) { - commitQueueFence(); + const fence = commitQueueFence(); + await this.messages.cancelMessages( + input.sessionId, + fence.retracted.map((message) => message.messageId), + ); active.stopRequested = true; return { kind: 'await_terminal', active }; } @@ -1870,7 +1965,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { }; } - commitQueueFence(); + const fence = commitQueueFence(); + await this.messages.cancelMessages( + input.sessionId, + fence.retracted.map((message) => message.messageId), + ); await this.interactions.claimRunClosure(input, 'turn_stopped', admissionLease); const shouldRequestStop = !active.stopRequested; active.stopRequested = true; @@ -1909,6 +2008,11 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (unavailableReason) { return completedStart(operationUnavailable(unavailableReason)); } + await this.clientCapabilities?.bindDurableRoot({ + sessionId: admission.sessionId, + userMessageId: admission.userMessageId, + execution: admission.execution, + }); const { runId } = admission; const existingRun = await this.readRunIfPresent(input.sessionId, runId); if (replacing && existingRun) { @@ -2097,7 +2201,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { }, { runId: active.runId, - userMessageId: active.userMessageId ?? undefined, + userMessageId: active.userMessageId, durability: 'required', onRunStarted: async (startedRunId) => { if (startedRunId !== active.runId) { @@ -2137,6 +2241,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } this.observeExecutionCompletion(active, { kind: 'terminal', snapshot }); await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status); + await this.settleExecutedMessageSources(active, snapshot.status); terminalTransitionStarted = true; await this.completeTerminalTransition(input.sessionId, active); } catch (error) { @@ -2160,6 +2265,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { snapshot, }); await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status); + await this.settleExecutedMessageSources(active, snapshot.status); terminalTransitionStarted = true; await this.completeTerminalTransition(input.sessionId, active); containedRunFailure = @@ -2215,6 +2321,25 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } } + private async settleExecutedMessageSources( + active: ActiveRootTurn, + terminalStatus: 'completed' | 'failed' | 'cancelled', + ): Promise { + const admission = await this.stores.agentRunStore.readRootTurnAdmission( + active.sessionId, + active.turnId, + ); + if (!admission) return; + await this.messages.settleMessagesAfterRoot({ + sessionId: active.sessionId, + turnId: active.turnId, + runId: active.runId, + admittedAt: admission.admittedAt, + messageIds: admission.sourceMessages.map((source) => source.messageId), + terminalStatus, + }); + } + private observeExecutionCompletion( active: ActiveRootTurn, completion: HostedExecutionCompletion, @@ -2276,15 +2401,12 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { admissionLease: SessionAdmissionLease, ): Promise { const initiatingConnectionId = batch.initiatingConnectionId; - if (!initiatingConnectionId) { - throw new RuntimeMessageAuthorityInvariantError( - 'Follow-up batch lost its initiating Client identity', - ); - } // A confirmed follow-up must become a durable root even when a Session // provider is unavailable. Lost tools are omitted while ephemeral // capabilities bind to the Client that submitted this follow-up. - await this.clientCapabilities?.bindConfirmedFollowup(batch.sessionId, initiatingConnectionId); + if (initiatingConnectionId) { + await this.clientCapabilities?.bindConfirmedFollowup(batch.sessionId, initiatingConnectionId); + } const turnId = randomUUID(); const header = await this.stores.sessionStore.readHeaderSnapshot(batch.sessionId); @@ -2293,7 +2415,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId: batch.sessionId, turnId, proposedRunId: randomUUID(), - proposedUserMessageId: randomUUID(), + proposedUserMessageId: batch.sources.length === 1 ? batch.sources[0]!.messageId : null, execution: { kind: 'external_message', inputDigest: messageContentDigest(batch.submittedContent), @@ -2307,6 +2429,13 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { 'Fresh follow-up root Turn identity already existed', ); } + await this.messages.handoffRootSources({ + sessionId: batch.sessionId, + turnId, + runId: admitted.admission.runId, + previousRootTurnId: admitted.admission.previousRootTurnId, + messageIds: batch.sources.map((source) => source.messageId), + }); const nextIdentity = { sessionId: batch.sessionId, @@ -2751,7 +2880,9 @@ function throwIfAborted(signal: AbortSignal): void { throw new DOMException('Agent graph supervisor Turn was aborted', 'AbortError'); } -function isTerminalSnapshot(snapshot: TurnSnapshot): boolean { +function isTerminalSnapshot( + snapshot: TurnSnapshot, +): snapshot is Extract { return ( snapshot.status === 'completed' || snapshot.status === 'failed' || diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index a3c83a20ea..d140ce9415 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -14801,930 +14801,6 @@ describe('SessionManager permission mode updates', () => { }); }); -describe('SessionManager steering and followup queues', () => { - function steeringManager() { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - return { manager, store }; - } - - // Run a turn and invoke `duringFirstDelta` synchronously the first time the - // turn streams text — the point at which a real user would type while the - // agent works. Returns every streamed event. - async function runTurnWith( - manager: SessionManager, - sessionId: string, - turnId: string, - duringFirstDelta: () => void, - ): Promise { - const events: SessionEvent[] = []; - let fired = false; - for await (const event of manager.sendMessage(sessionId, { turnId, text: 'hello' })) { - events.push(event); - if (!fired && event.type === 'text_delta') { - fired = true; - duringFirstDelta(); - } - } - return events; - } - - test('hosted root runs consume the Host owner and release it exactly once', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const identities: RuntimeMessageRunIdentity[] = []; - const acked: string[] = []; - const nacked: string[] = []; - let pulled = false; - let releases = 0; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - messageAuthority: { - bindRun: (identity) => { - identities.push(identity); - return { - ...identity, - pull: () => { - if (pulled) return []; - pulled = true; - return [ - { - id: 'host-lease-1', - messageId: 'host-message-1', - content: { text: 'host steer', displayText: 'visible host steer' }, - }, - ]; - }, - ack: (leaseIds) => acked.push(...leaseIds), - nack: (leaseIds) => nacked.push(...leaseIds), - release: () => { - releases += 1; - }, - }; - }, - }, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const events = await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-host-message', text: 'start' }), - ); - const [run] = await runStore.listSessionRuns(session.id); - expect(identities).toEqual([ - { sessionId: session.id, turnId: 'turn-host-message', runId: run?.runId }, - ]); - expect(acked).toEqual(['host-lease-1']); - expect(nacked).toEqual([]); - expect(releases).toBe(1); - expect( - events.some( - (event) => - event.type === 'steering_message' && - event.messageId === 'host-message-1' && - event.content.displayText === 'visible host steer', - ), - ).toBe(true); - expect(events.some((event) => event.type === 'queue_update')).toBe(false); - for (const operation of [ - () => manager.steer(session.id, 'runtime mirror'), - () => manager.queueMessage(session.id, 'runtime mirror'), - () => manager.drainFollowup(session.id), - () => manager.retractQueue(session.id), - ]) { - let error: unknown; - try { - operation(); - } catch (caught) { - error = caught; - } - expect(error instanceof RuntimeMessageAuthorityInvariantError).toBe(true); - } - }); - - test('hosted Interaction binds the durable Run identity and closes before release', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const identities: RuntimeInteractionRunIdentity[] = []; - const lifecycle: string[] = []; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - interactionAuthority: { - bindRun: (identity) => { - identities.push(identity); - return { - ...identity, - acceptSandboxBoundaryRequest: async () => {}, - acceptUserQuestionRequest: async () => {}, - close: async (reason) => { - lifecycle.push(`close:${reason}`); - }, - release: () => lifecycle.push('release'), - }; - }, - }, - canonicalPermissionOutcomes: noCanonicalPermissionOutcomes, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-host-interaction', text: 'go' }), - ); - const [run] = await runStore.listSessionRuns(session.id); - expect(identities).toEqual([ - { sessionId: session.id, turnId: 'turn-host-interaction', runId: run?.runId }, - ]); - expect(lifecycle).toEqual(['close:turn_terminal', 'release']); - }); - - test('hosted stopped question abandonment preserves stop closure before release', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new FakeBackend(ctx)); - const lifecycle: string[] = []; - let question: RuntimeUserQuestionContinuation | undefined; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - interactionAuthority: { - bindRun: (identity) => ({ - ...identity, - acceptSandboxBoundaryRequest: async () => {}, - acceptUserQuestionRequest: async ({ continuation }) => { - question = continuation; - }, - close: async (reason) => { - lifecycle.push(`close:${reason}`); - await question?.applyClosure(reason); - lifecycle.push('local-settled'); - }, - release: () => lifecycle.push('release'), - }), - }, - canonicalPermissionOutcomes: noCanonicalPermissionOutcomes, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'ask' })); - const iterator = manager - .sendMessage(session.id, { - turnId: 'turn-host-question-abandoned', - text: FAKE_ASK_USER_QUESTION_PROMPT, - }) - [Symbol.asyncIterator](); - - let request: SessionEvent | undefined; - while (request?.type !== 'user_question_request') { - const next = await iterator.next(); - if (next.done) break; - request = next.value; - } - expect(request?.type).toBe('user_question_request'); - await manager.stopSession(session.id, { source: 'stop_button' }); - expect(lifecycle).toEqual(['close:turn_stopped', 'local-settled']); - await iterator.return?.(undefined); - expect(lifecycle).toEqual(['close:turn_stopped', 'local-settled', 'release']); - }); - - test('hosted RuntimeKernel rejects a backend request without an admission receipt', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new UnadmittedQuestionBackend(ctx)); - let releases = 0; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - interactionAuthority: { - bindRun: (identity) => ({ - ...identity, - acceptSandboxBoundaryRequest: async () => {}, - acceptUserQuestionRequest: async () => {}, - close: async () => {}, - release: () => { - releases += 1; - }, - }), - }, - canonicalPermissionOutcomes: noCanonicalPermissionOutcomes, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'ask' })); - - let failure: unknown; - try { - await drainAll(manager.sendMessage(session.id, { turnId: 'turn-forged', text: 'start' })); - } catch (error) { - failure = error; - } - expect(failure instanceof RuntimeInteractionInvariantError).toBe(true); - expect(releases).toBe(1); - }); - - test('hosted owner is released when backend execution fails', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new ThrowBeforeTerminalBackend(ctx)); - let releases = 0; - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - messageAuthority: { - bindRun: (identity) => ({ - ...identity, - pull: () => [], - ack: () => {}, - nack: () => {}, - release: () => { - releases += 1; - }, - }), - }, - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - let failure: unknown; - try { - await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-host-failed', text: 'start' }), - ); - } catch (error) { - failure = error; - } - expect((failure as Error).message).toBe('backend failed before terminal'); - expect(releases).toBe(1); - }); - - test('a failed turn begin never leaks a steering owner', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let failBuilds = 1; - backends.register('ai-sdk', (ctx) => { - if (failBuilds > 0) { - failBuilds -= 1; - throw new Error('backend build failed'); - } - return new FakeBackend(ctx); - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - let failed: unknown; - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-fail', - text: 'hello', - })) { - // drain - } - } catch (error) { - failed = error; - } - expect((failed as Error).message).toBe('backend build failed'); - - // The failed begin must not have left a live owner: steering falls back - // instead of queueing a message no run will ever consume. - expect(manager.steer(session.id, 'orphaned')).toEqual({ kind: 'fallback' }); - expect(manager.queueMessage(session.id, 'orphaned too')).toEqual({ kind: 'fallback' }); - - // A later successful turn establishes ownership normally. - let outcome: QueueEnqueueOutcome | undefined; - const events = await runTurnWith(manager, session.id, 'turn-2', () => { - outcome = manager.steer(session.id, 'now consumed'); - }); - expect(outcome?.kind).toBe('queued'); - expect( - events.some( - (event) => event.type === 'steering_message' && event.content.text === 'now consumed', - ), - ).toBe(true); - }); - - test('an overlapping turn cannot drain steering queued for the current owner', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let backend: GatedSteeringBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new GatedSteeringBackend(ctx); - return backend; - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const first = drainAll(manager.sendMessage(session.id, { turnId: 'turn-a', text: 'first' })); - await waitUntil(() => backend?.gates.has('turn-a') === true); - const second = drainAll(manager.sendMessage(session.id, { turnId: 'turn-b', text: 'second' })); - await waitUntil(() => backend?.gates.has('turn-b') === true); - - // turn-b established ownership last, so the steer targets it. - expect(manager.steer(session.id, 'for the owner').kind).toBe('queued'); - - // The stale turn's pull hook fails the identity check and drains nothing. - backend?.release('turn-a'); - const firstEvents = await first; - expect(backend?.pulls.get('turn-a')).toEqual([[]]); - expect(firstEvents.some((event) => event.type === 'steering_message')).toBe(false); - - // The owner drains exactly the queued message. - backend?.release('turn-b'); - const secondEvents = await second; - expect(backend?.pulls.get('turn-b')).toEqual([['for the owner']]); - expect( - secondEvents.some( - (event) => event.type === 'steering_message' && event.content.text === 'for the owner', - ), - ).toBe(true); - }); - - test('a pulled lease is past the retract point: retract excludes it and it delivers exactly once', async () => { - // Round-5 F1/D1: pull() is the single atomic commit point. Once leased, - // the message belongs to this turn's delivery — a retract during the - // (slow) durable append returns only still-queued text, never the - // in-flight lease; otherwise the retracted text would ALSO be executed by - // the provider once the append lands (refill + execute = two copies). - const gate = makeGate(); - const parked = makeGate(); - class GatedRuntimeEventStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new GatedRuntimeEventStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - expect(manager.steer(sessionId, 'urgent steer').kind).toBe('queued'); - }, - ); - - const turnEvents: SessionEvent[] = []; - const turn = (async () => { - for await (const event of manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })) { - turnEvents.push(event); - } - })(); - await parked.promise; - // The steering append has not committed: the next provider request must - // not have started while the message is not durable. - await new Promise((resolve) => setTimeout(resolve, 25)); - expect(model.doStreamCalls.length).toBe(1); - // Pulled means committed to this turn: retract returns nothing. - expect(manager.retractQueue(session.id)).toBe(''); - gate.release(); - await turn; - // The message delivered exactly once: in the next provider request… - expect(model.doStreamCalls.length).toBe(2); - expect(JSON.stringify(model.doStreamCalls[1]?.prompt).includes('urgent steer')).toBe(true); - // …echoed once in the stream/ledger… - expect(turnEvents.filter((event) => event.type === 'steering_message').length).toBe(1); - // …and owned by no queue afterwards. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('an abort never converts a durably appended steering message into a redelivery', async () => { - // Round-5 F1/D3: abort does not settle a pushed lease — settlement is - // decided only by the persistence fact. Here the append is parked when - // the stop arrives; once it commits, the message belongs to the ledger - // (history replay presents it to the next turn) and must NOT also be - // nacked into the followup queue, which would put the same directive in - // the account twice. - const gate = makeGate(); - const parked = makeGate(); - class GatedRuntimeEventStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new GatedRuntimeEventStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - expect(manager.steer(sessionId, 'urgent steer').kind).toBe('queued'); - }, - ); - - const turn = (async () => { - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - // drain - } - } catch { - // the abort may end the stream abruptly - } - })(); - await parked.promise; - void manager.stopSession(session.id, { source: 'stop_button' }); - // Let the abort reach the backend's durability wait while the append is - // still parked — the exact window where an abort-settles-the-lease bug - // nacks a message that then also commits to the ledger. - await new Promise((resolve) => setTimeout(resolve, 25)); - gate.release(); - // Teardown converges: the parked append commits, the lease settles, and - // the aborted send terminates without hanging. - await turn; - - // The dying request was never sent… - expect(model.doStreamCalls.length).toBe(1); - // …the ledger owns the message (exactly one durable steering event)… - const runs = await runStore.listSessionRuns(session.id); - const steeringEvents: RuntimeEvent[] = []; - for (const run of runs) { - const events = await runStore.readRuntimeEvents(session.id, run.runId); - steeringEvents.push( - ...events.filter( - (event) => - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true, - ), - ); - } - expect(steeringEvents.length).toBe(1); - // …and no queue redelivers it. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('a nack that lands after the owner released folds into the followup queue, not an ownerless steering queue', async () => { - // Round-5 F3: turn A's append fails only after turn B took over and - // released. A's nack can no longer target A (it will never pull again) — - // the text's only safe home is the followup queue, exactly where a - // release-time fold would have put it. - const gate = makeGate(); - const parked = makeGate(); - class ParkThenFailStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - throw new Error('steering append failed'); - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new ParkThenFailStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - manager.steer(sessionId, 'urgent steer'); - }, - ); - - const turnA = (async () => { - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - // drain - } - } catch { - // the failed append ends the stream abruptly - } - })(); - await parked.promise; - // Turn B takes ownership and releases it while A is parked. - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-2', - text: 'second', - })) { - // drain - } - gate.release(); - await turnA; - - // The failed message is redeliverable exactly once, via followup. - expect(manager.drainFollowup(session.id)).toBe('urgent steer'); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('steer falls back when no RuntimeEventStore is configured', async () => { - // Round-5 F4: without a runtime event ledger, the steering durability ack - // has nothing to anchor to — the fail-closed persist contract cannot be - // honored. The fallback path opens a fresh turn whose user message is - // persisted by the SessionStore, keeping the same durability guarantee. - const store = new MemorySessionStore(); - const backends = new BackendRegistry(); - let backend: GatedSteeringBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new GatedSteeringBackend(ctx); - return backend; - }); - const manager = new SessionManager({ - store, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const turn = drainAll(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })); - await waitUntil(() => backend?.gates.has('turn-1') === true); - // A live turn exists, but steering cannot be made durable: fall back. - expect(manager.steer(session.id, 'no ledger')).toEqual({ kind: 'fallback' }); - // Followups are unaffected — they open a normal turn anyway. - expect(manager.queueMessage(session.id, 'later').kind).toBe('queued'); - backend?.gates.get('turn-1')?.release(); - backend?.pullDone.get('turn-1')?.release(); - await turn; - }); - - test('a failed steering append nacks the lease back to the queue and the request never carries it', async () => { - // Fail-CLOSED persistence: the steering append throws, the ack judgment - // propagates the failure (no fail-open swallow), the lease is nacked back - // to the queue (folded into followup at release), and neither the ledger - // nor the projection carries the undelivered message. - class FailingSteeringStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - throw new Error('steering append failed'); - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new FailingSteeringStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - expect(manager.steer(sessionId, 'urgent steer').kind).toBe('queued'); - }, - ); - - let failed: unknown; - try { - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - // drain - } - } catch (error) { - failed = error; - } - expect(failed instanceof Error).toBe(true); - // The dying request never carried the steering: no second provider call. - expect(model.doStreamCalls.length).toBe(1); - // Nacked back to the queue and folded into followup at release — the - // text is redeliverable, not lost. - expect(manager.drainFollowup(session.id)).toBe('urgent steer'); - // Ledger and projection agree: the message was never persisted. - const messages = await manager.getMessages(session.id); - expect( - messages.some((message) => message.type === 'user' && message.text === 'urgent steer'), - ).toBe(false); - }); - - test('an overlapping turn cannot turn a delivered lease into a followup redelivery', async () => { - // Round-4 V1: turn A leases the steer and parks in the (gated) durable - // append; turn B starts meanwhile and takes the owner slot. A's append - // then commits and A's provider request carries the message — so A's ack - // MUST still settle the lease (it is keyed by issuer, not by the current - // owner), and B's teardown must not fold A's in-flight lease into the - // followup queue, which would redeliver an already-executed directive. - const gate = makeGate(); - const parked = makeGate(); - class GatedRuntimeEventStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - parked.release(); - await gate.promise; - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new GatedRuntimeEventStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - manager.steer(sessionId, 'urgent steer'); - }, - ); - - const turnAEvents: SessionEvent[] = []; - const turnA = (async () => { - try { - for await (const event of manager.sendMessage(session.id, { - turnId: 'turn-1', - text: 'go', - })) { - turnAEvents.push(event); - } - } catch { - // A gated teardown may end the stream abruptly. - } - })(); - await parked.promise; - - // Turn B runs to completion while A is parked mid-lease. - for await (const _event of manager.sendMessage(session.id, { - turnId: 'turn-2', - text: 'second', - })) { - // drain - } - expect(model.doStreamCalls.length).toBe(2); - - gate.release(); - await turnA; - - // A's post-steer request went out carrying the directive exactly once… - expect(model.doStreamCalls.length).toBe(3); - expect(JSON.stringify(model.doStreamCalls[2]?.prompt).includes('urgent steer')).toBe(true); - // …B's request never did… - expect(JSON.stringify(model.doStreamCalls[1]?.prompt).includes('urgent steer')).toBe(false); - // …the ledger echoes it exactly once… - expect(turnAEvents.filter((event) => event.type === 'steering_message').length).toBe(1); - // …and NOTHING redelivers it: the delivered lease was acked by its - // issuer, so no queue still holds the text. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('a backend-forged queue_update never reaches the ledger or observers', async () => { - // Round-6 R3: the kernel is the only legal producer of queue_update (it - // pushes them directly into the turn stream). A backend that yields one - // is forging authoritative queue state; the flow drops it at the ingress - // — not mapped, not forwarded, not persisted. - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new ForgingQueueBackend(ctx)); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const events = await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' }), - ); - // Nothing was enqueued in this turn, so ANY queue_update in the stream - // is the forged one leaking through. - expect(events.some((event) => event.type === 'queue_update')).toBe(false); - const runs = await runStore.listSessionRuns(session.id); - const runtimeEvents = ( - await Promise.all(runs.map((run) => runStore.readRuntimeEvents(session.id, run.runId))) - ).flat(); - expect( - runtimeEvents.some( - (event) => - (event.actions?.stateDelta as { queueUpdate?: unknown } | undefined)?.queueUpdate !== - undefined, - ), - ).toBe(false); - }); - - test('provider retry progress reaches observers without becoming a durable runtime fact', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - backends.register('ai-sdk', (ctx) => new ProviderRetryProgressBackend(ctx)); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const events = await drainAll( - manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' }), - ); - expect( - events.filter((event) => event.type === 'provider_retry').map((event) => event.phase), - ).toEqual(['scheduled', 'started']); - - const runs = await runStore.listSessionRuns(session.id); - const runtimeEvents = ( - await Promise.all(runs.map((run) => runStore.readRuntimeEvents(session.id, run.runId))) - ).flat(); - expect( - runtimeEvents.some( - (event) => - (event.actions?.stateDelta as { providerRetry?: unknown } | undefined)?.providerRetry !== - undefined, - ), - ).toBe(false); - }); - - test('an append error after the write landed settles by the ledger read-back, not a duplicate nack', async () => { - // Round-6 R5: appendRuntimeEvent can fail AFTER the bytes landed (e.g. a - // close error). Treating every append error as not-durable would nack a - // message the ledger already owns — history replay plus the followup - // redelivery equals a double. The ambiguous failure is settled by reading - // the ledger back: present ⇒ durable ⇒ ack path. - class WriteThenThrowStore extends MemoryAgentRunStore { - override async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - if ( - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true - ) { - await super.appendRuntimeEvent(sessionId, runId, event); - throw new Error('close failed after the write landed'); - } - return super.appendRuntimeEvent(sessionId, runId, event); - } - } - const runStore = new WriteThenThrowStore(); - const model = steeringToolThenDoneModel(); - const { manager, session } = await steeringDeliverySession( - runStore, - model, - (manager, sessionId) => { - manager.steer(sessionId, 'urgent steer'); - }, - ); - - const turnEvents: SessionEvent[] = []; - for await (const event of manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })) { - turnEvents.push(event); - } - - // Delivered exactly once: the next request carries it… - expect(model.doStreamCalls.length).toBe(2); - expect(JSON.stringify(model.doStreamCalls[1]?.prompt).includes('urgent steer')).toBe(true); - // …the ledger owns exactly one copy… - const runs = await runStore.listSessionRuns(session.id); - const steeringEvents: RuntimeEvent[] = []; - for (const run of runs) { - const events = await runStore.readRuntimeEvents(session.id, run.runId); - steeringEvents.push( - ...events.filter( - (event) => - event.content?.kind === 'text' && - (event.content as { steering?: boolean }).steering === true, - ), - ); - } - expect(steeringEvents.length).toBe(1); - // …and no queue redelivers it. - expect(manager.drainFollowup(session.id)).toBe(null); - expect(manager.retractQueue(session.id)).toBe(''); - }); - - test('stranded steering emits a final queue snapshot when it folds into the followup queue', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let backend: GatedSteeringBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new GatedSteeringBackend(ctx); - return backend; - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(1_000), - }); - const session = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - - const turn = drainAll(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'go' })); - await waitUntil(() => backend?.gates.has('turn-1') === true); - backend?.gates.get('turn-1')?.release(); - // The turn's only step boundary has already pulled (empty)… - await waitUntil(() => backend?.pulls.has('turn-1') === true); - // …so this steer is stranded: no step is left to consume it. - expect(manager.steer(session.id, 'late').kind).toBe('queued'); - backend?.pullDone.get('turn-1')?.release(); - const events = await turn; - - // The stranded → followup migration is a queue change; the LAST snapshot - // in the stream reflects it, not the stale pre-fold state. - const updates = events.filter( - (event): event is Extract => - event.type === 'queue_update', - ); - expect(updates.at(-1)?.steering).toEqual([]); - expect(updates.at(-1)?.followup).toEqual(['late']); - expect(updates.at(-1)?.steeringEntries).toEqual([]); - expect(updates.at(-1)?.followupEntries).toHaveLength(1); - expect(updates.at(-1)?.followupEntries?.[0]?.content).toEqual({ text: 'late' }); - expect(updates.at(-1)?.followupEntries?.[0]?.placement).toBe('next_turn'); - expect(updates.at(-1)?.followupEntries?.[0]?.state).toBe('queued'); - // And the followup queue is the authoritative owner of the text. - expect(manager.drainFollowup(session.id)).toBe('late'); - }); -}); - async function drainAll(iterable: AsyncIterable): Promise { const events: SessionEvent[] = []; for await (const event of iterable) events.push(event); diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index fcbf0102ad..ee3cf91e6c 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -58,7 +58,7 @@ import { resolveEffectiveOrchestration, type EffectiveOrchestration, } from '@maka/core/orchestration'; -import type { SessionEvent } from '@maka/core/events'; +import { messageContentsEqual, type SessionEvent } from '@maka/core/events'; import type { AgentBackend, BackendSendInput } from '@maka/core/backend-types'; import type { RunTraceEvent } from './run-trace.js'; import type { StopSessionInput } from './session-manager.js'; @@ -145,7 +145,7 @@ export interface AgentRunInput { userInput: UserMessageInput; rootExecutionKind?: AgentRunHeader['rootExecutionKind']; runId?: string; - userMessageId?: string; + userMessageId?: string | null; durability?: AgentRunDurability; store: AgentRunSessionStore; runStore?: AgentRunStore; @@ -646,28 +646,32 @@ export class AgentRun { let initialRuntimeEventId: string; if (this.recordsSessionMessages()) { - const userMessageId = this.input.userMessageId ?? this.input.newId(); const userMessageTs = this.input.now(); - initialRuntimeEventId = userMessageId; - const userMsg = cloneAndFreezeRuntimeSnapshot({ - type: 'user', - id: userMessageId, - turnId: this.turnId, - ts: userMessageTs, - text: this.input.userInput.text, - ...(this.input.userInput.displayText !== undefined - ? { displayText: this.input.userInput.displayText } - : {}), - ...(this.input.userInput.attachments - ? { attachments: this.input.userInput.attachments } - : {}), - ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), - ...(this.input.userInput.inlineReferences - ? { inlineReferences: this.input.userInput.inlineReferences } - : {}), - ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), - }); - await this.input.store.appendMessage(this.sessionId, userMsg); + if (this.input.userMessageId === null) { + initialRuntimeEventId = this.input.newId(); + } else { + const userMessageId = this.input.userMessageId ?? this.input.newId(); + initialRuntimeEventId = userMessageId; + const userMsg = cloneAndFreezeRuntimeSnapshot({ + type: 'user', + id: userMessageId, + turnId: this.turnId, + ts: userMessageTs, + text: this.input.userInput.text, + ...(this.input.userInput.displayText !== undefined + ? { displayText: this.input.userInput.displayText } + : {}), + ...(this.input.userInput.attachments + ? { attachments: this.input.userInput.attachments } + : {}), + ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), + ...(this.input.userInput.inlineReferences + ? { inlineReferences: this.input.userInput.inlineReferences } + : {}), + ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), + }); + await appendUserMessageOnce(this.input.store, this.sessionId, userMsg); + } await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage); this.lastTs = userMessageTs; } else { @@ -1896,6 +1900,28 @@ function redactTraceString(value: string): string { function errorMessage(error: unknown): string { return redactTraceString(error instanceof Error ? error.message : String(error)); } + +async function appendUserMessageOnce( + store: AgentRunSessionStore, + sessionId: string, + message: UserMessage, +): Promise { + const existing = (await store.readMessages(sessionId)).find( + (candidate) => candidate.id === message.id, + ); + if (!existing) { + await store.appendMessage(sessionId, message); + return; + } + if ( + existing.type !== 'user' || + existing.turnId !== message.turnId || + !messageContentsEqual(existing, message) + ) { + throw new Error(`Durable UserMessage identity ${message.id} has conflicting content`); + } +} + function isInteractionResumeAck(event: SessionEvent): boolean { return ( event.type === 'sandbox_boundary_decision_ack' || event.type === 'user_question_answer_ack' diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index a9c0c9fefc..f93c8b7ee3 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -33,13 +33,12 @@ import type { RuntimeEventStore, } from '@maka/core/runtime-event-store'; import { isSessionInlineRun } from '@maka/core/agent-run'; -import type { - ActiveInteractionRequestEvent, - CompleteEvent, - QueueEnqueueOutcome, - QueueUpdateEvent, - SessionEvent, - TokenUsageEvent, +import { + type ActiveInteractionRequestEvent, + type CompleteEvent, + type QueueEnqueueOutcome, + type SessionEvent, + type TokenUsageEvent, } from '@maka/core/events'; import type { SessionBlockedReason, @@ -185,13 +184,10 @@ export interface RuntimeKernelLike { respondToSandboxBoundary(sessionId: string, response: SandboxBoundaryResponse): Promise; listActiveInteractions?(sessionId: string): ActiveInteractionRequestEvent[]; respondToUserQuestion?(sessionId: string, response: UserQuestionResponse): Promise; - /** Queue a user message for mid-turn injection at the next step boundary. */ + /** Compatibility surface; durable message admission belongs to Runtime Host. */ steer(sessionId: string, text: string): QueueEnqueueOutcome; - /** Queue a user message to open the turn after the current one finishes. */ queueMessage(sessionId: string, text: string): QueueEnqueueOutcome; - /** Drain the followup queue into one `\n\n`-joined prompt, or null if empty. */ drainFollowup(sessionId: string): string | null; - /** Take back every queued message (both queues) as one `\n\n`-joined string. */ retractQueue(sessionId: string): string; hasActiveRuns(sessionId: string): boolean; /** @@ -232,7 +228,7 @@ export class RuntimeContextCompactError extends Error { export interface TurnStartOptions { runId?: string; - userMessageId?: string; + userMessageId?: string | null; durability?: AgentRunDurability; /** * Resolve turn admission after this Session has registered a pending start @@ -271,44 +267,6 @@ export interface ChildAgentRetryInput { onRunStarted?: () => void | Promise; } -/** - * An embedded session's authoritative pending-message queues plus its event - * sink. Hosted composition never creates this state; its Host owns admission, - * snapshots, leases, and follow-up drain. - */ -interface PendingSteeringMessage extends SteeringLease {} - -/** - * A pulled lease is bound to the turn that pulled it: only the issuing turn's - * backend can settle it (ack/nack stay valid even after ownership moved to an - * overlapping turn — invalidating a delivered lease would leave it in-flight - * and redeliver an already-executed message), and no other turn's retract/ - * clear/release may reclaim it while its delivery is still undetermined. - */ -interface LeasedSteeringMessage extends PendingSteeringMessage { - issuingTurnId: string; -} - -interface SessionSteeringState { - /** Messages waiting to be injected into the running turn at a step boundary. */ - steering: PendingSteeringMessage[]; - /** - * Leased to the running turn's backend but not yet settled. pull() is the - * single atomic commit point: an in-flight lease is committed to that - * turn's delivery — retract/clear reclaim only QUEUED messages — and it - * settles exactly once, decided solely by the persistence fact: ack when - * the steering event is durably consumed (even under abort), nack when it - * provably never persisted. Snapshots count in-flight as still pending so - * the UI keeps showing the message until it lands in the transcript. - */ - inFlight: LeasedSteeringMessage[]; - /** Messages waiting to open the next turn. */ - followup: PendingSteeringMessage[]; - /** Pushes a `queue_update` into the active turn's stream; unset when idle. */ - sink?: (event: QueueUpdateEvent) => void; - activeTurnId?: string; -} - export type BackendActivationBoundary = (operation: () => Promise | T) => Promise; interface ChildToolActivation { @@ -458,7 +416,6 @@ export class RuntimeKernel implements RuntimeKernelLike { private readonly historyCompactCoordinator: HistoryCompactCheckpointCoordinator; private readonly pendingContinuationClaims = new Set(); private readonly pendingContinuationSessions = new Set(); - private readonly steeringBySession = new Map(); private readonly backendInvalidations = new Map(); private readonly interactionRequestOwners = new Map(); private nextBackendGeneration = 0; @@ -1560,11 +1517,6 @@ export class RuntimeKernel implements RuntimeKernelLike { const interactionRun = owners.interactionRun; const messageOwner = owners.messageOwner; - // Steering is a top-level-turn affordance only; child agent turns run - // without a queue. Hosted ownership is bound before begin so a pre-start - // cancellation can release the exact admitted owner. The pull hook still - // re-checks this run's turnId so stale or overlapping runs cannot drain - // messages queued for the current owner. let pullSteering: (() => readonly SteeringLease[]) | undefined; let ackSteering: ((leaseIds: readonly string[]) => void) | undefined; let nackSteering: ((leaseIds: readonly string[]) => void) | undefined; @@ -1572,71 +1524,6 @@ export class RuntimeKernel implements RuntimeKernelLike { pullSteering = () => messageOwner?.pull() ?? []; ackSteering = (leaseIds) => messageOwner?.ack(leaseIds); nackSteering = (leaseIds) => messageOwner?.nack(leaseIds); - } else if (steering) { - const state = this.ensureSteering(sessionId); - state.sink = (event) => { - void sessionEvents.push(event).catch(() => {}); - }; - state.activeTurnId = run.turnId; - // Lease, don't consume: pulled messages move to in-flight and only an - // ack (durable + injected) removes them; a nack or a retract/clear/ - // release reclaims them, so an abort window can never drop text. - pullSteering = () => { - const current = this.steeringBySession.get(sessionId); - if (!current || current.activeTurnId !== run.turnId) return []; - if (current.steering.length === 0) return []; - const leased = current.steering.splice(0); - current.inFlight.push( - ...leased.map((message) => ({ ...message, issuingTurnId: run.turnId })), - ); - return leased.map((message) => ({ ...message })); - }; - // Settlement is keyed by lease id + issuing turn, NOT by current - // ownership: an overlapping turn that takes the owner slot must not - // invalidate the issuer's ack (the message was delivered to ITS - // provider) or intercept its nack. A late settle for a reclaimed lease - // finds no match and is a no-op. - ackSteering = (leaseIds) => { - const current = this.steeringBySession.get(sessionId); - if (!current) return; - const ids = new Set(leaseIds); - const before = current.inFlight.length; - current.inFlight = current.inFlight.filter( - (message) => !(ids.has(message.id) && message.issuingTurnId === run.turnId), - ); - if (current.inFlight.length !== before) this.emitQueueUpdate(sessionId, current); - }; - nackSteering = (leaseIds) => { - const current = this.steeringBySession.get(sessionId); - if (!current) return; - const ids = new Set(leaseIds); - const returned = current.inFlight.filter( - (message) => ids.has(message.id) && message.issuingTurnId === run.turnId, - ); - if (returned.length === 0) return; - current.inFlight = current.inFlight.filter( - (message) => !(ids.has(message.id) && message.issuingTurnId === run.turnId), - ); - if (current.activeTurnId === run.turnId) { - // Back to the FRONT of the queue: a re-pull at the next step - // boundary preserves the user's original ordering. - current.steering = [ - ...returned.map(({ id, messageId, content }) => ({ id, messageId, content })), - ...current.steering, - ]; - } else { - // The issuer no longer owns the queue (an overlapping turn took - // over and possibly released): it will never pull again, so the - // steering queue would strand the text ownerless. The followup - // queue is its only safe home — the same direction a release-time - // fold takes. - current.followup = [ - ...returned.map(({ id, messageId, content }) => ({ id, messageId, content })), - ...current.followup, - ]; - } - this.emitQueueUpdate(sessionId, current); - }; } const stopBackend = this.stopBackendFor(begin.backend); @@ -1688,7 +1575,6 @@ export class RuntimeKernel implements RuntimeKernelLike { // under its Session admission gate. The outer finally remains an // idempotent backstop for paths that never reach this hook. if (messageOwner) owners.releaseMessage(); - else if (steering) this.releaseSteeringTurn(sessionId, run.turnId); sessionEvents.close(); } catch (error) { sessionEvents.fail(error); @@ -1734,7 +1620,6 @@ export class RuntimeKernel implements RuntimeKernelLike { finalizeRun: () => owners.finalize(), releaseOwner: () => { if (messageOwner) owners.releaseMessage(); - else if (steering) this.releaseSteeringTurn(sessionId, run.turnId); }, }); } finally { @@ -2189,10 +2074,6 @@ export class RuntimeKernel implements RuntimeKernelLike { } private async stopSessionAttempt(sessionId: string, intent: SessionStopIntent): Promise { - // Interrupt clears both queues before the abort lands; the emitted empty - // snapshot lets the UI collapse its pending bar, and callers refill their - // editor from the mirror captured before the clear. - this.clearSteering(sessionId); const failures: unknown[] = []; let operation = this.stopOperations.get(sessionId); try { @@ -2492,170 +2373,25 @@ export class RuntimeKernel implements RuntimeKernelLike { // -------------------------------------------------------------------------- steer(sessionId: string, text: string): QueueEnqueueOutcome { - this.assertEmbeddedMessageQueue('steer'); - // Steering's delivery contract is anchored to the runtime event ledger - // (fail-closed persist + durable-consume ack). Without a RuntimeEventStore - // that anchor does not exist — same condition as requireTerminalWrite — - // so fall back to a fresh turn, whose user message the SessionStore - // persists with the ordinary turn-open guarantee. - if (!this.deps.runtimeEventStore) return { kind: 'fallback' }; - // Double responsibility (codex): with no live steering owner to inject - // into — the turn just ended, begin() failed, or only child/compact runs - // are active (they never consume this queue) — tell the caller to open a - // fresh turn instead so the message is never dropped. - const state = this.liveSteeringState(sessionId); - if (!state) return { kind: 'fallback' }; - const messageId = this.deps.newId(); - state.steering.push({ id: messageId, messageId, content: { text } }); - this.emitQueueUpdate(sessionId, state); - return { kind: 'queued' }; + void sessionId; + void text; + return { kind: 'fallback' }; } queueMessage(sessionId: string, text: string): QueueEnqueueOutcome { - this.assertEmbeddedMessageQueue('queueMessage'); - const state = this.liveSteeringState(sessionId); - if (!state) return { kind: 'fallback' }; - const messageId = this.deps.newId(); - state.followup.push({ id: messageId, messageId, content: { text } }); - this.emitQueueUpdate(sessionId, state); - return { kind: 'queued' }; + void sessionId; + void text; + return { kind: 'fallback' }; } drainFollowup(sessionId: string): string | null { - this.assertEmbeddedMessageQueue('drainFollowup'); - const state = this.steeringBySession.get(sessionId); - if (!state || state.followup.length === 0) return null; - const drained = state.followup.splice(0); - this.emitQueueUpdate(sessionId, state); - return drained.map((message) => message.content.text).join('\n\n'); + void sessionId; + return null; } retractQueue(sessionId: string): string { - this.assertEmbeddedMessageQueue('retractQueue'); - const state = this.steeringBySession.get(sessionId); - if (!state) return ''; - // Retract reclaims QUEUED messages only. pull() is the single atomic - // commit point of delivery: an in-flight lease is already committed to - // the running turn — its durable append may land at any moment, so - // handing its text back to the user here would refill AND execute the - // same directive. An in-flight lease settles only by the persistence - // fact (ack when the ledger owns it, nack back to a queue otherwise). - const all = [ - ...state.steering.map((message) => message.content.text), - ...state.followup.map((message) => message.content.text), - ]; - state.steering = []; - state.followup = []; - this.emitQueueUpdate(sessionId, state); - return all.join('\n\n'); - } - - private ensureSteering(sessionId: string): SessionSteeringState { - const existing = this.steeringBySession.get(sessionId); - if (existing) return existing; - const created: SessionSteeringState = { steering: [], inFlight: [], followup: [] }; - this.steeringBySession.set(sessionId, created); - return created; - } - - private assertEmbeddedMessageQueue(operation: string): void { - if (this.deps.messageAuthority) { - throw new RuntimeMessageAuthorityInvariantError( - `Hosted Runtime cannot ${operation}; the Runtime Host owns message admission and queues`, - ); - } - } - - /** - * The session's steering state only while a steering-capable top-level run - * owns it (sink registered after begin() succeeded and not yet released). - * Child agent and compact runs never establish ownership, so their activity - * alone yields undefined — enqueue must fall back rather than strand text. - */ - private liveSteeringState(sessionId: string): SessionSteeringState | undefined { - const state = this.steeringBySession.get(sessionId); - return state?.sink ? state : undefined; - } - - private emitQueueUpdate(sessionId: string, state: SessionSteeringState): void { - state.sink?.({ - type: 'queue_update', - id: this.deps.newId(), - turnId: state.activeTurnId ?? '', - ts: this.deps.now(), - steering: [ - ...state.inFlight.map((message) => message.content.text), - ...state.steering.map((message) => message.content.text), - ], - followup: state.followup.map((message) => message.content.text), - steeringEntries: [ - ...state.inFlight.map((message) => ({ - entryId: message.id, - messageId: message.messageId, - content: message.content, - placement: 'current_turn' as const, - state: 'in_flight' as const, - })), - ...state.steering.map((message) => ({ - entryId: message.id, - messageId: message.messageId, - content: message.content, - placement: 'current_turn' as const, - state: 'queued' as const, - })), - ], - followupEntries: state.followup.map((message) => ({ - entryId: message.id, - messageId: message.messageId, - content: message.content, - placement: 'next_turn' as const, - state: 'queued' as const, - })), - }); - } - - private clearSteering(sessionId: string): void { - const state = this.steeringBySession.get(sessionId); - if (!state) return; - // Same commit-point rule as retractQueue: only QUEUED messages are - // clearable. An in-flight lease is already committed to the running - // turn's delivery and settles only by the persistence fact. - if (state.steering.length === 0 && state.followup.length === 0) return; - state.steering = []; - state.followup = []; - this.emitQueueUpdate(sessionId, state); - } - - private releaseSteeringTurn(sessionId: string, turnId: string): void { - const state = this.steeringBySession.get(sessionId); - if (!state) return; - // A release folds only the leases THIS turn issued; an overlapping turn's - // in-flight lease stays for its issuer to settle (acked = delivered, so - // folding it into followup would redeliver an already-executed message). - const own = state.inFlight.filter((message) => message.issuingTurnId === turnId); - if (state.activeTurnId !== turnId) { - // Not (or no longer) the owner. The issuer's backend settles every - // lease before its turn ends, so `own` is normally empty; this is a - // backstop that keeps a never-settled lease from stranding invisibly. - if (own.length === 0) return; - state.inFlight = state.inFlight.filter((message) => message.issuingTurnId !== turnId); - state.followup = [...own, ...state.followup]; - this.emitQueueUpdate(sessionId, state); - return; - } - // Stranded steering (arrived after the final step boundary, so no step is - // left to consume it) becomes the head of the followup queue instead of - // vanishing — the next turn opens with it first (grok-build safety). The - // migration is a queue change, so emit the final snapshot BEFORE the sink - // is cleared; otherwise observers stay on the stale pre-fold snapshot. - if (state.steering.length > 0 || own.length > 0) { - state.followup = [...own, ...state.steering, ...state.followup]; - state.inFlight = state.inFlight.filter((message) => message.issuingTurnId !== turnId); - state.steering = []; - this.emitQueueUpdate(sessionId, state); - } - state.sink = undefined; - state.activeTurnId = undefined; + void sessionId; + return ''; } hasActiveRuns(sessionId: string): boolean { @@ -2720,7 +2456,6 @@ export class RuntimeKernel implements RuntimeKernelLike { private async disposeBackendNow(sessionId: string): Promise { const generations = this.backendGenerationsFor(sessionId); - this.steeringBySession.delete(sessionId); this.historyCompactCoordinator.clear(sessionId); let disposalError: unknown; for (const active of generations) { diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 98e85f4105..6071b73d0b 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -38,6 +38,8 @@ export interface RuntimeLedgerRepairDeps { runStore: AgentRunStore; runtimeEventStore: RuntimeEventStore; readMessages(sessionId: string): Promise; + readPendingMessageIds?(sessionId: string): Promise; + readMessageLifecycleState?(sessionId: string, messageId: string): Promise; appendMessage(sessionId: string, message: StoredMessage): Promise; appendTurnState( sessionId: string, @@ -89,11 +91,24 @@ export class RuntimeLedgerRepair { this.deps.readMessages(sessionId), this.deps.runStore.listSessionRuns(sessionId), ]); + const pendingMessageIds = new Set((await this.deps.readPendingMessageIds?.(sessionId)) ?? []); + if (this.deps.readMessageLifecycleState) { + const lifecycleStates = await Promise.all( + messages.map(async (message) => ({ + messageId: message.id, + state: await this.deps.readMessageLifecycleState!(sessionId, message.id), + })), + ); + for (const { messageId, state } of lifecycleStates) { + if (state !== undefined) pendingMessageIds.add(messageId); + } + } + const ledgerMessages = messages.filter((message) => !pendingMessageIds.has(message.id)); const inlineRunsByTurn = new Map( runs.filter(isSessionInlineRun).map((run) => [run.turnId, run] as const), ); - const messagesByTurn = groupMessagesByTurn(messages); - const turns = deriveTurnRecords(messages).filter((turn) => + const messagesByTurn = groupMessagesByTurn(ledgerMessages); + const turns = deriveTurnRecords(ledgerMessages).filter((turn) => (messagesByTurn.get(turn.turnId) ?? []).some((message) => message.type === 'user'), ); if (turns.length === 0) return; diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 7693840d64..c19b6eaf11 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -671,6 +671,11 @@ export interface SessionStore { list(filter?: SessionListFilter): Promise; readHeader(sessionId: string): Promise; readMessages(sessionId: string): Promise; + listMessageAdmissions?(sessionId: string): Promise; + readMessageLifecycleState?( + sessionId: string, + messageId: string, + ): Promise<'accepted' | 'handed_off' | 'executed' | 'cancelled' | undefined>; readMessagesSnapshot?(sessionId: string): Promise; listTurns(sessionId: string): Promise; appendMessage(sessionId: string, m: StoredMessage): Promise; @@ -938,6 +943,12 @@ export class SessionManager { runStore: deps.runStore, runtimeEventStore: deps.runtimeEventStore, readMessages: (sessionId) => deps.store.readMessages(sessionId), + readPendingMessageIds: async (sessionId) => + (await deps.store.listMessageAdmissions?.(sessionId))?.map( + ({ messageId }) => messageId, + ) ?? [], + readMessageLifecycleState: async (sessionId, messageId) => + deps.store.readMessageLifecycleState?.(sessionId, messageId) ?? undefined, appendMessage: (sessionId, message) => deps.store.appendMessage(sessionId, message), appendTurnState: (sessionId, turnId, status, lineage, options) => this.appendTurnState(sessionId, turnId, status, lineage, options), diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 124f1e7d8a..3e3d0be12e 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -25,6 +25,7 @@ import { DatabaseSync } from 'node:sqlite'; import { describe, test } from 'node:test'; import { Worker } from 'node:worker_threads'; import { AgentGraphClientTerminalCursorError } from '@maka/core/agent-graph-client-projection'; +import { messageContentDigest } from '@maka/core/events'; import { canReadPath, createReadOnlyPermissionProfile, @@ -45,6 +46,7 @@ import { type SessionConfigurationMetadataUpdate, type SqliteSessionMetadataStoreFailpoint, } from '../sqlite-session-metadata-store.js'; +import type { PendingMessageAdmission } from '../message-receipt-store.js'; import { createSqliteRuntimeStore, SQLITE_RUNTIME_SCHEMA_VERSION, @@ -84,7 +86,7 @@ describe('SqliteSessionMetadataStore', () => { const migrated = createSqliteSessionMetadataStore(path); try { - assert.equal(migrated.schemaVersion(), 29); + assert.equal(migrated.schemaVersion(), SQLITE_SESSION_METADATA_SCHEMA_VERSION); assert.equal((await migrated.read(legacyHeader.id)).header.externalOrigin, undefined); } finally { migrated.close(); @@ -237,6 +239,165 @@ describe('SqliteSessionMetadataStore', () => { } }); + test('atomically accepts a steering message and its canonical transcript', async () => { + const store = createSqliteSessionMetadataStore(':memory:'); + try { + await store.create(fullHeader({ id: 'session-1' })); + const admission: PendingMessageAdmission = { + sessionId: 'session-1', + turnId: 'turn-1', + runId: 'run-1', + messageId: 'message-1', + content: { text: 'submitted', displayText: 'submitted' }, + submittedContentDigest: messageContentDigest({ text: 'submitted' }), + submittedPlacement: 'current_turn', + placement: 'current_turn', + disposition: 'steering', + admittedAt: 10, + }; + + const normalizedAdmission = { + ...admission, + content: { text: 'submitted' }, + }; + assert.deepEqual(await store.commitMessageAdmission(admission), normalizedAdmission); + assert.deepEqual( + await store.readMessageAdmission('session-1', 'message-1'), + normalizedAdmission, + ); + assert.deepEqual( + (await store.readMessages('session-1')).map((message) => ({ + id: message.id, + type: message.type, + turnId: message.turnId, + text: message.type === 'user' ? message.text : undefined, + steeringEventId: message.type === 'user' ? message.steeringEventId : undefined, + })), + [ + { + id: 'message-1', + type: 'user', + turnId: 'turn-1', + text: 'submitted', + steeringEventId: 'message-1', + }, + ], + ); + assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'accepted'); + assert.deepEqual( + (await store.listMessageAdmissions('session-1')).map((entry) => entry.messageId), + ['message-1'], + ); + await store.markMessagesHandedOff('session-1', ['message-1']); + assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'handed_off'); + assert.deepEqual(await store.listMessageAdmissions('session-1'), []); + assert.deepEqual( + (await store.listUnsettledMessageAdmissions('session-1')).map((entry) => entry.messageId), + ['message-1'], + ); + await store.markMessagesExecuted('session-1', ['message-1']); + assert.equal(await store.readMessageLifecycleState('session-1', 'message-1'), 'executed'); + assert.deepEqual(await store.listUnsettledMessageAdmissions('session-1'), []); + } finally { + store.close(); + } + }); + + test('atomically accepts a follow-up message and its canonical transcript', async () => { + const store = createSqliteSessionMetadataStore(':memory:'); + try { + await store.create(fullHeader({ id: 'session-followup-admission' })); + const admission = await store.commitMessageAdmission({ + sessionId: 'session-followup-admission', + turnId: 'turn-current', + runId: 'run-current', + messageId: 'message-followup', + content: { text: 'queued before the successor root' }, + submittedContentDigest: messageContentDigest({ + text: 'queued before the successor root', + }), + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: 11, + }); + assert.equal(admission.disposition, 'followup'); + assert.deepEqual( + (await store.readMessages('session-followup-admission')).map((message) => ({ + id: message.id, + turnId: message.turnId, + })), + [{ id: 'message-followup', turnId: 'turn-current' }], + ); + await store.rebindMessageAdmissionTranscript({ + sessionId: 'session-followup-admission', + messageIds: ['message-followup'], + turnId: 'turn-successor', + previousRootTurnId: 'turn-current', + }); + await store.rebindMessageAdmissionTranscript({ + sessionId: 'session-followup-admission', + messageIds: ['message-followup'], + turnId: 'turn-successor', + previousRootTurnId: 'turn-current', + }); + assert.deepEqual( + (await store.readMessages('session-followup-admission')).map((message) => ({ + id: message.id, + turnId: message.turnId, + })), + [{ id: 'message-followup', turnId: 'turn-successor' }], + ); + } finally { + store.close(); + } + }); + + test('persists a follow-up reorder across SQLite restart', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-message-reorder-')); + const path = join(root, 'state.sqlite'); + try { + const store = createSqliteSessionMetadataStore(path); + try { + await store.create(fullHeader({ id: 'session-reorder' })); + for (const [index, messageId] of ['message-first', 'message-second'].entries()) { + await store.commitMessageAdmission({ + sessionId: 'session-reorder', + turnId: 'turn-current', + runId: 'run-current', + messageId, + content: { text: messageId }, + submittedContentDigest: messageContentDigest({ text: messageId }), + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: 20 + index, + }); + } + await store.reorderMessageAdmissions('session-reorder', [ + 'message-second', + 'message-first', + ]); + } finally { + store.close(); + } + + const reopened = createSqliteSessionMetadataStore(path); + try { + assert.deepEqual( + (await reopened.listMessageAdmissions('session-reorder')).map( + (admission) => admission.messageId, + ), + ['message-second', 'message-first'], + ); + } finally { + reopened.close(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + test('migrates v24 legacy session statuses to active exactly once', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-session-status-v24-')); const path = join(root, 'state.sqlite'); diff --git a/packages/storage/src/agent-run-store.ts b/packages/storage/src/agent-run-store.ts index 069635e85d..f11839265a 100644 --- a/packages/storage/src/agent-run-store.ts +++ b/packages/storage/src/agent-run-store.ts @@ -1731,7 +1731,8 @@ function assertRootTurnAdmissionContract(admission: RootTurnAdmission): void { const providerRetry = execution.kind === 'linked_child_provider_retry'; const inputlessExecution = execution.kind === 'safe_boundary_continuation' || execution.kind === 'context_compact'; - const messageLessExecution = inputlessExecution || providerRetry; + const sourceBatch = execution.kind === 'external_message' && admission.sourceMessages.length > 1; + const messageLessExecution = inputlessExecution || providerRetry || sourceBatch; if (execution.kind === 'agent_graph_supervisor_wake') { if ( admission.turnOrchestration?.mode !== 'graph' || diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index ea8e49bcda..fcbae1a961 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -115,9 +115,12 @@ export type { RuntimeEventScanResult, } from './agent-run-store.js'; export type { + MessageLifecycleState, + MessageLifecycleStore, MessageOperationReceipt, MessageReceiptOperation, MessageReceiptStore, + PendingMessageAdmission, } from './message-receipt-store.js'; export type { ProbeSessionRemovalResult, @@ -173,6 +176,10 @@ export interface ExecutionSessionReader { list(filter?: SessionListFilter): Promise; readHeader(sessionId: string): Promise; readMessages(sessionId: string): Promise; + readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise; listTurns(sessionId: string): Promise; close?(): Promise; } @@ -421,6 +428,28 @@ async function createExecutionStoresForWrite sessionStore.appendMessage(sessionId, message)), appendMessages: (sessionId, messages) => run(() => sessionStore.appendMessages(sessionId, messages)), + commitMessageAdmission: (admission) => + run(() => sessionStore.commitMessageAdmission(admission)), + readMessageAdmission: (sessionId, messageId) => + run(() => sessionStore.readMessageAdmission(sessionId, messageId)), + readMessageLifecycleState: (sessionId, messageId) => + run(() => sessionStore.readMessageLifecycleState(sessionId, messageId)), + listMessageAdmissions: (sessionId) => + run(() => sessionStore.listMessageAdmissions(sessionId)), + listUnsettledMessageAdmissions: (sessionId) => + run(() => sessionStore.listUnsettledMessageAdmissions(sessionId)), + rebindMessageAdmissionTranscript: (input) => + run(() => sessionStore.rebindMessageAdmissionTranscript(input)), + updateMessageAdmission: (admission) => + run(() => sessionStore.updateMessageAdmission(admission)), + reorderMessageAdmissions: (sessionId, messageIds) => + run(() => sessionStore.reorderMessageAdmissions(sessionId, messageIds)), + cancelMessageAdmissions: (sessionId, messageIds) => + run(() => sessionStore.cancelMessageAdmissions(sessionId, messageIds)), + markMessagesHandedOff: (sessionId, messageIds) => + run(() => sessionStore.markMessagesHandedOff(sessionId, messageIds)), + markMessagesExecuted: (sessionId, messageIds) => + run(() => sessionStore.markMessagesExecuted(sessionId, messageIds)), subscribeTranscriptChanges: (listener) => sessionStore.subscribeTranscriptChanges(listener), updateHeader: (sessionId, patch) => run(() => sessionStore.updateHeader(sessionId, patch)), updateHeaderVersioned: (sessionId, patch, expectedRevision) => @@ -608,6 +637,8 @@ async function openExecutionStoresForRead run(() => sessionStore.list(filter)), readHeader: (sessionId) => run(() => sessionStore.readHeaderSnapshot(sessionId)), readMessages: (sessionId) => run(() => sessionStore.readMessagesSnapshot(sessionId)), + readMessageLifecycleState: (sessionId, messageId) => + run(() => sessionStore.readMessageLifecycleState(sessionId, messageId)), listTurns: (sessionId) => run(() => sessionStore.listTurnsSnapshot(sessionId)), close: () => closeExecutionStorePersistence(sessionStore, runtimePersistence, { diff --git a/packages/storage/src/message-receipt-store.ts b/packages/storage/src/message-receipt-store.ts index 82bcd27277..8fafab521a 100644 --- a/packages/storage/src/message-receipt-store.ts +++ b/packages/storage/src/message-receipt-store.ts @@ -20,6 +20,7 @@ import { resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import type { DatabaseSync } from 'node:sqlite'; +import { normalizeMessageContent, type MessageContent } from '@maka/core/events'; import { acquireOperationalStateDatabase, type OperationalStateDatabaseLease, @@ -29,6 +30,99 @@ const SAFE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; const RECEIPT_SCHEMA_VERSION = 1 as const; const RECEIPT_MAX_BYTES = 64 * 1024; +export type MessageLifecycleState = 'accepted' | 'handed_off' | 'executed' | 'cancelled'; + +export interface PendingMessageAdmission { + readonly sessionId: string; + readonly turnId: string; + readonly runId: string; + readonly messageId: string; + readonly content: MessageContent; + readonly submittedContentDigest: `sha256:${string}`; + readonly submittedPlacement: 'current_turn' | 'next_turn'; + readonly placement: 'current_turn' | 'next_turn'; + readonly disposition: 'steering' | 'followup'; + readonly admittedAt: number; +} + +export interface MessageLifecycleStore { + commitMessageAdmission(admission: PendingMessageAdmission): Promise; + readMessageAdmission( + sessionId: string, + messageId: string, + ): Promise; + readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise; + listMessageAdmissions(sessionId: string): Promise; + listUnsettledMessageAdmissions(sessionId: string): Promise; + rebindMessageAdmissionTranscript(input: { + sessionId: string; + messageIds: readonly string[]; + turnId: string; + previousRootTurnId: string | null; + }): Promise; + updateMessageAdmission(admission: PendingMessageAdmission): Promise; + reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; + cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise; + markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise; + markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise; +} + +export function normalizePendingMessageAdmission( + admission: PendingMessageAdmission, +): PendingMessageAdmission { + for (const [name, value] of [ + ['Session', admission.sessionId], + ['Turn', admission.turnId], + ['Run', admission.runId], + ['Message', admission.messageId], + ] as const) { + assertSafeId(value, `Invalid ${name} identity`); + } + if ( + (admission.submittedPlacement !== 'current_turn' && + admission.submittedPlacement !== 'next_turn') || + (admission.placement !== 'current_turn' && admission.placement !== 'next_turn') || + (admission.disposition !== 'steering' && admission.disposition !== 'followup') || + (admission.placement === 'current_turn') !== (admission.disposition === 'steering') + ) { + throw new Error('Invalid pending Message placement'); + } + if (!Number.isSafeInteger(admission.admittedAt) || admission.admittedAt < 0) { + throw new Error('Invalid message admission timestamp'); + } + const normalized = Object.freeze({ + ...admission, + content: normalizeMessageContent(admission.content), + }); + if (!/^sha256:[a-f0-9]{64}$/u.test(normalized.submittedContentDigest)) { + throw new Error('Invalid pending Message submitted content digest'); + } + return normalized; +} + +export function samePendingMessageAdmission( + left: PendingMessageAdmission, + right: PendingMessageAdmission, +): boolean { + const a = normalizePendingMessageAdmission(left); + const b = normalizePendingMessageAdmission(right); + return ( + a.sessionId === b.sessionId && + a.turnId === b.turnId && + a.runId === b.runId && + a.messageId === b.messageId && + a.submittedContentDigest === b.submittedContentDigest && + a.submittedPlacement === b.submittedPlacement && + a.placement === b.placement && + a.disposition === b.disposition && + a.admittedAt === b.admittedAt && + isDeepStrictEqual(a.content, b.content) + ); +} + export type MessageReceiptOperation = | 'submit' | 'retract' diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index a731808e92..c6bc1a59f8 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -80,6 +80,7 @@ import { type TurnStateMessage, type UserMessage, } from '@maka/core/session'; +import type { MessageLifecycleStore, PendingMessageAdmission } from './message-receipt-store.js'; const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -299,7 +300,7 @@ export interface SessionStore { close?(): Promise; } -export interface SessionAuthorityStore extends SessionStore { +export interface SessionAuthorityStore extends SessionStore, MessageLifecycleStore { /** Read a bounded set of durable messages at an inclusive transcript watermark. */ readTranscriptMessagesSnapshot( sessionId: string, @@ -857,6 +858,80 @@ class SqliteSessionStore implements SessionAuthorityStore { for (const listener of this.transcriptChangeListeners) listener(sessionId); } + async commitMessageAdmission( + admission: PendingMessageAdmission, + ): Promise { + await this.ensureReady(); + const committed = await this.metadata.commitMessageAdmission(admission); + for (const listener of this.transcriptChangeListeners) listener(admission.sessionId); + return committed; + } + + async readMessageAdmission( + sessionId: string, + messageId: string, + ): Promise { + await this.ensureReady(); + return this.metadata.readMessageAdmission(sessionId, messageId); + } + + async listMessageAdmissions(sessionId: string): Promise { + await this.ensureReady(); + return this.metadata.listMessageAdmissions(sessionId); + } + + async listUnsettledMessageAdmissions( + sessionId: string, + ): Promise { + await this.ensureReady(); + return this.metadata.listUnsettledMessageAdmissions(sessionId); + } + + async rebindMessageAdmissionTranscript(input: { + sessionId: string; + messageIds: readonly string[]; + turnId: string; + previousRootTurnId: string | null; + }): Promise { + await this.ensureReady(); + await this.metadata.rebindMessageAdmissionTranscript(input); + for (const listener of this.transcriptChangeListeners) listener(input.sessionId); + } + + async readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise { + await this.ensureReady(); + return this.metadata.readMessageLifecycleState(sessionId, messageId); + } + + async updateMessageAdmission(admission: PendingMessageAdmission): Promise { + await this.ensureReady(); + await this.metadata.updateMessageAdmission(admission); + for (const listener of this.transcriptChangeListeners) listener(admission.sessionId); + } + + async reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.reorderMessageAdmissions(sessionId, messageIds); + } + + async cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.cancelMessageAdmissions(sessionId, messageIds); + } + + async markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.markMessagesHandedOff(sessionId, messageIds); + } + + async markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise { + await this.ensureReady(); + await this.metadata.markMessagesExecuted(sessionId, messageIds); + } + subscribeTranscriptChanges(listener: (sessionId: string) => void): () => void { this.transcriptChangeListeners.add(listener); return () => this.transcriptChangeListeners.delete(listener); diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index c7b92e8e1a..d1ccc46a65 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -19,7 +19,7 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 29; +export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 30; export const SQLITE_SESSION_MESSAGE_CHUNK_BYTES = 64 * 1024; export const SQLITE_SESSION_MESSAGE_CHUNK_MARKER = '{"$maka":"session-message-chunks-v1"}'; @@ -820,6 +820,33 @@ const MIGRATIONS: ReadonlyMap = new Map([ ON session_messages(session_id, message_ts, sequence); `, ], + [ + 30, + ` + CREATE TABLE IF NOT EXISTS message_admissions ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + run_id TEXT NOT NULL, + message_id TEXT NOT NULL, + content_json TEXT NOT NULL, + submitted_content_digest TEXT NOT NULL, + submitted_placement TEXT NOT NULL + CHECK (submitted_placement IN ('current_turn', 'next_turn')), + placement TEXT NOT NULL CHECK (placement IN ('current_turn', 'next_turn')), + disposition TEXT NOT NULL CHECK (disposition IN ('steering', 'followup')), + lifecycle_state TEXT NOT NULL + CHECK (lifecycle_state IN ('accepted', 'handed_off', 'executed', 'cancelled')), + queue_order INTEGER NOT NULL CHECK (queue_order >= 0), + admitted_at INTEGER NOT NULL CHECK (admitted_at >= 0), + UNIQUE (session_id, message_id), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS message_admissions_by_session_order + ON message_admissions(session_id, lifecycle_state, queue_order, sequence); + `, + ], [ 21, ` diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 3e13d67460..6954c2b593 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -96,6 +96,13 @@ import { decodeStoredMessage as decodePersistedStoredMessage, } from '@maka/core/session'; import { markPersisted } from '@maka/core/persisted-value'; +import { + normalizePendingMessageAdmission, + samePendingMessageAdmission, + type MessageLifecycleState, + type PendingMessageAdmission, +} from './message-receipt-store.js'; +import { messageContentsEqual, normalizeMessageContent } from '@maka/core/events'; import { type AgentGraphIntentAdmissionSnapshot, type AgentGraphTimelineMetadataSnapshot, @@ -219,6 +226,60 @@ export interface SessionCatalogMessageProjection { readonly lastMessagePreview?: string; } +interface MessageAdmissionRow { + readonly turn_id?: unknown; + readonly run_id?: unknown; + readonly message_id?: unknown; + readonly content_json?: unknown; + readonly submitted_content_digest?: unknown; + readonly submitted_placement?: unknown; + readonly placement?: unknown; + readonly disposition?: unknown; + readonly lifecycle_state?: unknown; + readonly queue_order?: unknown; + readonly admitted_at?: unknown; +} + +function decodeMessageAdmissionRow( + sessionId: string, + row: MessageAdmissionRow, +): { readonly admission: PendingMessageAdmission; readonly lifecycleState: MessageLifecycleState } { + if ( + typeof row.turn_id !== 'string' || + typeof row.run_id !== 'string' || + typeof row.message_id !== 'string' || + typeof row.content_json !== 'string' || + typeof row.submitted_content_digest !== 'string' || + (row.submitted_placement !== 'current_turn' && row.submitted_placement !== 'next_turn') || + (row.placement !== 'current_turn' && row.placement !== 'next_turn') || + (row.disposition !== 'steering' && row.disposition !== 'followup') || + (row.lifecycle_state !== 'accepted' && + row.lifecycle_state !== 'handed_off' && + row.lifecycle_state !== 'executed' && + row.lifecycle_state !== 'cancelled') || + typeof row.queue_order !== 'number' || + !Number.isSafeInteger(row.queue_order) || + row.queue_order < 0 || + typeof row.admitted_at !== 'number' + ) { + throw new SessionMetadataConflictError(`Invalid Message admission row for ${sessionId}`); + } + const admission = normalizePendingMessageAdmission({ + sessionId, + turnId: row.turn_id, + runId: row.run_id, + messageId: row.message_id, + content: JSON.parse(row.content_json) as PendingMessageAdmission['content'], + submittedContentDigest: + row.submitted_content_digest as PendingMessageAdmission['submittedContentDigest'], + submittedPlacement: row.submitted_placement, + placement: row.placement, + disposition: row.disposition, + admittedAt: row.admitted_at, + }); + return { admission, lifecycleState: row.lifecycle_state }; +} + export interface SessionAuthoritySnapshot { record: SessionMetadataRecord; boundary: ExecutionBoundary; @@ -1478,6 +1539,515 @@ export class SqliteSessionMetadataStore { }); } + async commitMessageAdmission( + admission: PendingMessageAdmission, + ): Promise { + this.assertOpen(); + const stored = normalizePendingMessageAdmission(admission); + return this.transaction(() => { + const record = this.readRecordSync(stored.sessionId); + if (!record) throw new SessionNotFoundError(stored.sessionId); + const existingRow = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(stored.sessionId, stored.messageId) as MessageAdmissionRow | undefined; + if (existingRow) { + const existing = decodeMessageAdmissionRow(stored.sessionId, existingRow); + if (!samePendingMessageAdmission(existing.admission, stored)) { + throw new SessionMetadataConflictError('Message admission identity conflict'); + } + if (existing.lifecycleState !== 'accepted') { + throw new SessionMetadataConflictError('Message admission identity is already settled'); + } + return existing.admission; + } + const orderRow = this.db + .prepare( + ` + SELECT COALESCE(MAX(queue_order), -1) + 1 AS next_order + FROM message_admissions + WHERE session_id = ? AND lifecycle_state = 'accepted' + `, + ) + .get(stored.sessionId) as { next_order?: unknown }; + if (typeof orderRow.next_order !== 'number' || !Number.isSafeInteger(orderRow.next_order)) { + throw new SessionMetadataConflictError('Invalid message admission order'); + } + this.db + .prepare( + ` + INSERT INTO message_admissions( + session_id, turn_id, run_id, message_id, content_json, submitted_content_digest, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'accepted', ?, ?) + `, + ) + .run( + stored.sessionId, + stored.turnId, + stored.runId, + stored.messageId, + JSON.stringify(stored.content), + stored.submittedContentDigest, + stored.submittedPlacement, + stored.placement, + stored.disposition, + orderRow.next_order, + stored.admittedAt, + ); + + if (stored.disposition === 'steering' || stored.disposition === 'followup') { + const message = decodeCanonicalMessage({ + type: 'user', + id: stored.messageId, + turnId: stored.turnId, + ts: stored.admittedAt, + ...stored.content, + steeringEventId: stored.messageId, + }); + const existingMessages = this.readMessagesWith( + stored.sessionId, + decodeStoredMessage, + ).filter((candidate) => candidate.id === stored.messageId); + if (existingMessages.length > 1) { + throw new SessionMetadataConflictError( + 'Message admission transcript identity is ambiguous', + ); + } + const existingMessage = existingMessages[0]; + if (existingMessage && !isDeepStrictEqual(existingMessage, message)) { + throw new SessionMetadataConflictError('Message admission transcript identity conflict'); + } + if (!existingMessage) { + const row = this.db + .prepare( + 'SELECT COALESCE(MAX(sequence), -1) AS last_sequence FROM session_messages WHERE session_id = ?', + ) + .get(stored.sessionId) as { last_sequence?: unknown }; + if (typeof row.last_sequence !== 'number' || !Number.isSafeInteger(row.last_sequence)) { + throw new SessionMetadataConflictError('Invalid Session message sequence'); + } + const json = JSON.stringify(message); + this.insertSessionMessagesSync(stored.sessionId, row.last_sequence + 1, [ + { message, json }, + ]); + this.updateCatalogProjectionSync( + stored.sessionId, + { + lastMessageAt: stored.admittedAt, + lastMessagePreview: message.type === 'user' ? message.displayText : undefined, + }, + false, + !record.header.connectionLocked, + ); + } + } + return stored; + }); + } + + async readMessageAdmission( + sessionId: string, + messageId: string, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + assertSafeSessionId(messageId); + return this.readTransaction(() => { + const row = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + AND lifecycle_state = 'accepted' + `, + ) + .get(sessionId, messageId) as MessageAdmissionRow | undefined; + return row ? decodeMessageAdmissionRow(sessionId, row).admission : undefined; + }); + } + + async listMessageAdmissions(sessionId: string): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + return this.readTransaction(() => { + const rows = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND lifecycle_state = 'accepted' + ORDER BY queue_order, sequence + `, + ) + .all(sessionId) as MessageAdmissionRow[]; + return rows.map((row) => decodeMessageAdmissionRow(sessionId, row).admission); + }); + } + + async listUnsettledMessageAdmissions( + sessionId: string, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + return this.readTransaction(() => { + const rows = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND lifecycle_state IN ('accepted', 'handed_off') + ORDER BY queue_order, sequence + `, + ) + .all(sessionId) as MessageAdmissionRow[]; + return rows.map((row) => decodeMessageAdmissionRow(sessionId, row).admission); + }); + } + + async readMessageLifecycleState( + sessionId: string, + messageId: string, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + assertSafeSessionId(messageId); + return this.readTransaction(() => { + const row = this.db + .prepare( + ` + SELECT lifecycle_state + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(sessionId, messageId) as { lifecycle_state?: unknown } | undefined; + if (!row) return undefined; + if ( + row.lifecycle_state !== 'accepted' && + row.lifecycle_state !== 'handed_off' && + row.lifecycle_state !== 'executed' && + row.lifecycle_state !== 'cancelled' + ) { + throw new SessionMetadataConflictError('Invalid Message admission lifecycle state'); + } + return row.lifecycle_state; + }); + } + + async rebindMessageAdmissionTranscript(input: { + sessionId: string; + messageIds: readonly string[]; + turnId: string; + previousRootTurnId: string | null; + }): Promise { + this.assertOpen(); + assertSafeSessionId(input.sessionId); + assertSafeSessionId(input.turnId); + if (input.previousRootTurnId !== null) { + assertSafeSessionId(input.previousRootTurnId); + } + const unique = [...new Set(input.messageIds)]; + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + for (const messageId of unique) { + const admissionRow = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(input.sessionId, messageId) as MessageAdmissionRow | undefined; + if (!admissionRow) { + throw new SessionMetadataConflictError('Message admission does not exist'); + } + const admission = decodeMessageAdmissionRow(input.sessionId, admissionRow); + if ( + admission.lifecycleState !== 'accepted' && + admission.lifecycleState !== 'handed_off' && + admission.lifecycleState !== 'executed' + ) { + throw new SessionMetadataConflictError('Message admission is already cancelled'); + } + const rows = this.db + .prepare( + ` + SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 + FROM session_messages AS message + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE message.session_id = ? AND message.message_id = ? + `, + ) + .all(input.sessionId, messageId) as Array<{ + sequence?: unknown; + record_json?: unknown; + record_bytes?: unknown; + sha256?: unknown; + }>; + if (rows.length !== 1) { + throw new SessionMetadataConflictError( + rows.length === 0 + ? 'Message admission transcript is missing' + : 'Message admission transcript identity is ambiguous', + ); + } + const sequence = rows[0]?.sequence; + if (typeof sequence !== 'number' || !Number.isSafeInteger(sequence)) { + throw new SessionMetadataConflictError('Invalid Message transcript sequence'); + } + const row = rows[0]!; + const recordJson = readStoredMessageRecordJson(this.db, input.sessionId, sequence, row); + const message = decodeStoredMessage(JSON.parse(recordJson) as unknown); + if ( + message.type !== 'user' || + message.id !== messageId || + !messageContentsEqual(normalizeMessageContent(message), admission.admission.content) + ) { + throw new SessionMetadataConflictError('Message admission transcript identity conflict'); + } + if (message.turnId === input.turnId) continue; + if ( + message.turnId !== admission.admission.turnId && + (input.previousRootTurnId === null || message.turnId !== input.previousRootTurnId) + ) { + throw new SessionMetadataConflictError('Message admission transcript Turn conflict'); + } + const rebound = decodeCanonicalMessage({ ...message, turnId: input.turnId }); + this.replaceSessionMessageSync(input.sessionId, sequence, rebound); + } + }); + } + + async updateMessageAdmission(admission: PendingMessageAdmission): Promise { + this.assertOpen(); + const stored = normalizePendingMessageAdmission(admission); + this.transaction(() => { + const currentRow = this.db + .prepare( + ` + SELECT turn_id, run_id, message_id, content_json, submitted_content_digest, + submitted_placement, placement, disposition, lifecycle_state, queue_order, admitted_at + FROM message_admissions + WHERE session_id = ? AND message_id = ? + `, + ) + .get(stored.sessionId, stored.messageId) as MessageAdmissionRow | undefined; + if (!currentRow) throw new SessionMetadataConflictError('Message admission does not exist'); + const current = decodeMessageAdmissionRow(stored.sessionId, currentRow); + if (current.lifecycleState !== 'accepted') { + throw new SessionMetadataConflictError('Message admission is already settled'); + } + if ( + current.admission.turnId !== stored.turnId || + current.admission.runId !== stored.runId || + current.admission.submittedPlacement !== stored.submittedPlacement || + current.admission.admittedAt !== stored.admittedAt + ) { + throw new SessionMetadataConflictError('Message admission update identity conflict'); + } + this.db + .prepare( + ` + UPDATE message_admissions + SET content_json = ?, submitted_content_digest = ?, placement = ?, disposition = ? + WHERE session_id = ? AND message_id = ? AND lifecycle_state = 'accepted' + `, + ) + .run( + JSON.stringify(stored.content), + stored.submittedContentDigest, + stored.placement, + stored.disposition, + stored.sessionId, + stored.messageId, + ); + if (stored.disposition !== 'steering' && stored.disposition !== 'followup') return; + const message = decodeCanonicalMessage({ + type: 'user', + id: stored.messageId, + turnId: stored.turnId, + ts: stored.admittedAt, + ...stored.content, + steeringEventId: stored.messageId, + }); + const rows = this.db + .prepare( + ` + SELECT sequence, record_json + FROM session_messages + WHERE session_id = ? AND message_id = ? + `, + ) + .all(stored.sessionId, stored.messageId) as Array<{ + sequence?: unknown; + record_json?: unknown; + }>; + if (rows.length > 1) { + throw new SessionMetadataConflictError( + 'Message admission transcript identity is ambiguous', + ); + } + const json = JSON.stringify(message); + if (rows.length === 0) { + const row = this.db + .prepare( + 'SELECT COALESCE(MAX(sequence), -1) AS last_sequence FROM session_messages WHERE session_id = ?', + ) + .get(stored.sessionId) as { last_sequence?: unknown }; + if (typeof row.last_sequence !== 'number' || !Number.isSafeInteger(row.last_sequence)) { + throw new SessionMetadataConflictError('Invalid Session message sequence'); + } + this.insertSessionMessagesSync(stored.sessionId, row.last_sequence + 1, [ + { message, json }, + ]); + } else { + const sequence = rows[0]?.sequence; + if (typeof sequence !== 'number' || !Number.isSafeInteger(sequence)) { + throw new SessionMetadataConflictError('Invalid Message transcript sequence'); + } + this.replaceSessionMessageSync(stored.sessionId, sequence, message, json); + } + this.updateCatalogProjectionSync( + stored.sessionId, + { + lastMessageAt: stored.admittedAt, + lastMessagePreview: message.type === 'user' ? message.displayText : undefined, + }, + true, + ); + }); + } + + async cancelMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + const unique = [...new Set(messageIds)]; + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + const statement = this.db.prepare( + ` + UPDATE message_admissions + SET lifecycle_state = 'cancelled' + WHERE session_id = ? AND message_id = ? AND lifecycle_state IN ('accepted', 'handed_off') + `, + ); + for (const messageId of unique) { + const result = statement.run(sessionId, messageId); + if (result.changes !== 1) { + const existing = this.db + .prepare( + 'SELECT lifecycle_state FROM message_admissions WHERE session_id = ? AND message_id = ?', + ) + .get(sessionId, messageId) as { lifecycle_state?: unknown } | undefined; + if (existing?.lifecycle_state !== 'cancelled') { + throw new SessionMetadataConflictError( + 'Message admission cancellation identity conflict', + ); + } + } + } + }); + } + + async reorderMessageAdmissions(sessionId: string, messageIds: readonly string[]): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + const unique = [...new Set(messageIds)]; + if (unique.length !== messageIds.length) { + throw new SessionMetadataConflictError( + 'Message admission reorder contains duplicate identities', + ); + } + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + const rows = this.db + .prepare( + ` + SELECT message_id + FROM message_admissions + WHERE session_id = ? AND lifecycle_state = 'accepted' AND disposition = 'followup' + ORDER BY queue_order, sequence + `, + ) + .all(sessionId) as Array<{ message_id?: unknown }>; + const current = rows.map((row) => row.message_id); + const currentIds = new Set(current); + if ( + current.length !== unique.length || + unique.some((messageId) => !currentIds.has(messageId)) + ) { + throw new SessionMetadataConflictError('Message admission reorder identity conflict'); + } + const update = this.db.prepare( + ` + UPDATE message_admissions + SET queue_order = ? + WHERE session_id = ? AND message_id = ? AND lifecycle_state = 'accepted' + `, + ); + unique.forEach((messageId, index) => update.run(index, sessionId, messageId)); + }); + } + + async markMessagesHandedOff(sessionId: string, messageIds: readonly string[]): Promise { + return this.markMessageLifecycle(sessionId, messageIds, 'handed_off'); + } + + async markMessagesExecuted(sessionId: string, messageIds: readonly string[]): Promise { + return this.markMessageLifecycle(sessionId, messageIds, 'executed'); + } + + private markMessageLifecycle( + sessionId: string, + messageIds: readonly string[], + state: 'handed_off' | 'executed', + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + const unique = [...new Set(messageIds)]; + for (const messageId of unique) assertSafeSessionId(messageId); + this.transaction(() => { + const allowedPreviousStates = + state === 'handed_off' ? "lifecycle_state = 'accepted'" : "lifecycle_state = 'handed_off'"; + const statement = this.db.prepare( + ` + UPDATE message_admissions + SET lifecycle_state = ? + WHERE session_id = ? AND message_id = ? + AND ${allowedPreviousStates} + `, + ); + for (const messageId of unique) { + const result = statement.run(state, sessionId, messageId); + if (result.changes !== 1) { + const existing = this.db + .prepare( + 'SELECT lifecycle_state FROM message_admissions WHERE session_id = ? AND message_id = ?', + ) + .get(sessionId, messageId) as { lifecycle_state?: unknown } | undefined; + if (existing?.lifecycle_state !== state) { + throw new SessionMetadataConflictError('Message admission lifecycle identity conflict'); + } + } + } + }); + return Promise.resolve(); + } + async readMessages(sessionId: string): Promise { return this.readMessagesWith(sessionId, decodeStoredMessage); } @@ -4189,6 +4759,60 @@ export class SqliteSessionMetadataStore { } } + private replaceSessionMessageSync( + sessionId: string, + sequence: number, + message: StoredMessage, + json = JSON.stringify(message), + ): void { + const encoded = Buffer.from(json, 'utf8'); + this.db + .prepare('DELETE FROM session_message_chunks WHERE session_id = ? AND sequence = ?') + .run(sessionId, sequence); + this.db + .prepare('DELETE FROM session_message_payloads WHERE session_id = ? AND sequence = ?') + .run(sessionId, sequence); + if (encoded.byteLength <= SQLITE_SESSION_MESSAGE_CHUNK_BYTES) { + this.db + .prepare( + 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?', + ) + .run(json, sessionId, sequence); + return; + } + this.db + .prepare('UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = ?') + .run(SQLITE_SESSION_MESSAGE_CHUNK_MARKER, sessionId, sequence); + this.db + .prepare( + 'INSERT INTO session_message_payloads(session_id, sequence, record_bytes, sha256) VALUES (?, ?, ?, ?)', + ) + .run( + sessionId, + sequence, + encoded.byteLength, + createHash('sha256').update(encoded).digest('hex'), + ); + for ( + let offset = 0; + offset < encoded.byteLength; + offset += SQLITE_SESSION_MESSAGE_CHUNK_BYTES + ) { + const chunk = encoded.subarray(offset, offset + SQLITE_SESSION_MESSAGE_CHUNK_BYTES); + this.db + .prepare( + 'INSERT INTO session_message_chunks(session_id, sequence, chunk_index, data, sha256) VALUES (?, ?, ?, ?, ?)', + ) + .run( + sessionId, + sequence, + offset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES, + chunk, + createHash('sha256').update(chunk).digest('hex'), + ); + } + } + private readMessagesWith( sessionId: string, decode: (value: unknown) => StoredMessage,