diff --git a/docs/desktop-message-queue.md b/docs/desktop-message-queue.md index 62954b30b3..2fc30d8722 100644 --- a/docs/desktop-message-queue.md +++ b/docs/desktop-message-queue.md @@ -26,7 +26,7 @@ Desktop used a rendered `streaming` prop to decide whether a composer submit sta Runtime Host already owns the durable message semantics: - `current_turn` queues steering for the next provider boundary. -- `next_turn` queues a successor turn. +- `next_turn` queues one successor turn per accepted message. - queue projections are authoritative. - queue projections carry the canonical queued message content; mutation results return only queue state. diff --git a/packages/runtime-host/src/__tests__/execution-host-message.test.ts b/packages/runtime-host/src/__tests__/execution-host-message.test.ts index 51058a93e0..f11a478e0d 100644 --- a/packages/runtime-host/src/__tests__/execution-host-message.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-message.test.ts @@ -287,38 +287,41 @@ test('steering becomes durable and ordered followups automatically start the nex } const chain = await fixture.readAdmissionChain(); - assert.equal(chain.length, 2); + assert.equal(chain.length, 3); assert.equal(chain[1]?.previousRootTurnId, firstTurnId); - assert.equal(chain[1]?.userMessageId, null); - assert.deepEqual( - chain[1]?.sourceMessages.map(({ messageId, content, placement, disposition }) => ({ - messageId, - content, - placement, - disposition, - })), - orderedFollowupSources.map((source) => ({ - ...source, - placement: 'next_turn', - disposition: 'followup', - })), + assert.equal(chain[2]?.previousRootTurnId, chain[1]?.turnId); + for (const [index, source] of orderedFollowupSources.entries()) { + const admission = chain[index + 1]; + assert.equal(admission?.userMessageId, source.messageId); + assert.deepEqual( + admission?.sourceMessages.map(({ messageId, content, placement, disposition }) => ({ + messageId, + content, + placement, + disposition, + })), + [{ ...source, placement: 'next_turn', disposition: 'followup' }], + ); + } + const followupTurnIds = chain.slice(1).map((admission) => admission.turnId); + const followupLedgers = await Promise.all( + followupTurnIds.map((turnId) => fixture.readTurn(turnId)), ); - assert.deepEqual(chain[1]?.normalizedInput, { - text: `${orderedFollowupSources[0].content.text}\n\n${orderedFollowupSources[1].content.text}`, - displayText: `${orderedFollowupSources[0].content.text}\n\n${orderedFollowupSources[1].content.displayText}`, - attachments: orderedFollowupSources[1].content.attachments, - quotes: orderedFollowupSources.flatMap((source) => source.content.quotes ?? []), - }); - const followupTurnId = chain[1]?.turnId; - assert.ok(followupTurnId); - const followupLedger = await fixture.readTurn(followupTurnId); const expectedQuotes = orderedFollowupSources.flatMap((source) => source.content.quotes ?? []); - assert.equal(followupLedger.userMessages.length, followupSources.length); assert.deepEqual( - followupLedger.userMessages.flatMap((message) => message.quotes ?? []), + followupLedgers.map((ledger) => ledger.userMessages.length), + [1, 1], + ); + assert.deepEqual( + followupLedgers.flatMap((ledger) => + ledger.userMessages.flatMap((message) => message.quotes ?? []), + ), + expectedQuotes, + ); + assert.deepEqual( + followupLedgers.flatMap((ledger) => userRuntimeContent(ledger.runtimeEvents)?.quotes ?? []), expectedQuotes, ); - assert.deepEqual(userRuntimeContent(followupLedger.runtimeEvents)?.quotes, expectedQuotes); const sessionUserMessages = await fixture.readSessionUserMessages(); for (const source of orderedFollowupSources) { assert.equal( @@ -327,13 +330,15 @@ test('steering becomes durable and ordered followups automatically start the nex ); } assert.equal( - sessionUserMessages.filter((message) => message.turnId === followupTurnId).length, + sessionUserMessages.filter((message) => followupTurnIds.includes(message.turnId)).length, orderedFollowupSources.length, ); assert.deepEqual( - sessionUserMessages - .filter((message) => message.turnId === followupTurnId) - .map((message) => message.id), + followupTurnIds.flatMap((turnId) => + sessionUserMessages + .filter((message) => message.turnId === turnId) + .map((message) => message.id), + ), orderedFollowupSources.map((source) => source.messageId), ); }); diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index 40f8c22a18..559ee3585e 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -217,17 +217,18 @@ test('subscribed Clients share one canonical queue and ordered root handoff', as await tui.close(); await fixture.stopHost(host); const chain = await fixture.readAdmissionChain(); + assert.equal(chain.length, 3); assert.deepEqual( - chain.map((admission) => admission.turnId), + chain.slice(0, 2).map((admission) => admission.turnId), [firstTurnId, successor.snapshot.rootTurn.turnId], ); + assert.equal(chain[2]?.previousRootTurnId, successor.snapshot.rootTurn.turnId); assert.deepEqual( - chain[1]?.sourceMessages.map((source) => source.messageId), - [desktopFollowupId, tuiFollowupId], + chain.slice(1).map((admission) => admission.sourceMessages.map((source) => source.messageId)), + [[desktopFollowupId], [tuiFollowupId]], ); - assert.deepEqual(chain[1]?.normalizedInput, { - text: `${desktopFollowupContent.text}\n\n${tuiFollowupContent.text}`, - }); + assert.deepEqual(chain[1]?.normalizedInput, desktopFollowupContent); + assert.deepEqual(chain[2]?.normalizedInput, tuiFollowupContent); }); }); diff --git a/packages/runtime-host/src/__tests__/message-coordinator.test.ts b/packages/runtime-host/src/__tests__/message-coordinator.test.ts index 48f24ad74a..cb9c21cd51 100644 --- a/packages/runtime-host/src/__tests__/message-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/message-coordinator.test.ts @@ -341,7 +341,7 @@ test('invalidates the canonical projection after each observable queue mutation' await fixture.coordinator.close(); }); -test('hands a mixed-Client queue to one Session successor', async () => { +test('hands each explicit follow-up to its own Session successor', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); const owner = fixture.coordinator.bindRun(ROOT); @@ -353,41 +353,55 @@ test('hands a mixed-Client queue to one Session successor', async () => { placement, }); - const steering = await fixture.coordinator.handlers['turn.message.submit']( - input('steering-from-b', 'first aggregate source', 'current_turn'), + const first = await fixture.coordinator.handlers['turn.message.submit']( + input('followup-from-b', 'first successor', 'next_turn'), operationContext('connection-b'), ); - const followup = await fixture.coordinator.handlers['turn.message.submit']( - input('followup-from-c', 'second aggregate source', 'next_turn'), + const second = await fixture.coordinator.handlers['turn.message.submit']( + input('followup-from-c', 'second successor', 'next_turn'), operationContext('connection-c'), ); - assert.equal(steering.ok, true); - assert.equal(followup.ok, true); + assert.equal(first.ok, true); + assert.equal(second.ok, true); owner.release(); - const batch = fixture.coordinator.beginTerminalTransition(ROOT); + const firstBatch = fixture.coordinator.beginTerminalTransition(ROOT); assert.deepEqual( - batch.sources.map((source) => source.messageId), - ['steering-from-b', 'followup-from-c'], + firstBatch.sources.map((source) => source.messageId), + ['followup-from-b'], ); - fixture.coordinator.commitNextRoot(batch, { + const secondRoot = { sessionId: ROOT.sessionId, turnId: 'turn-2', runId: 'run-2', + }; + fixture.coordinator.commitNextRoot(firstBatch, secondRoot); + assert.equal(fixture.liveResidencies(), 1); + const nextOwner = fixture.coordinator.bindRun(secondRoot); + nextOwner.release(); + const secondBatch = fixture.coordinator.beginTerminalTransition(secondRoot); + assert.deepEqual( + secondBatch.sources.map((source) => source.messageId), + ['followup-from-c'], + ); + fixture.coordinator.commitNextRoot(secondBatch, { + sessionId: ROOT.sessionId, + turnId: 'turn-3', + runId: 'run-3', }); assert.equal(fixture.liveResidencies(), 0); - const nextOwner = fixture.coordinator.bindRun({ + const finalOwner = fixture.coordinator.bindRun({ sessionId: ROOT.sessionId, - turnId: 'turn-2', - runId: 'run-2', + turnId: 'turn-3', + runId: 'run-3', }); - nextOwner.release(); + finalOwner.release(); fixture.coordinator.completeIdle( fixture.coordinator.beginTerminalTransition({ sessionId: ROOT.sessionId, - turnId: 'turn-2', - runId: 'run-2', + turnId: 'turn-3', + runId: 'run-3', }), ); await fixture.coordinator.close(); @@ -438,6 +452,107 @@ test('recovered followups without a connection owner still form one successor ba ); }); +test('recovery starts one explicit follow-up and keeps later messages queued', async () => { + const fixture = createFixture(); + for (const [index, messageId] of ['recovered-first', 'recovered-second'].entries()) { + const content = { text: messageId }; + await fixture.admissions.commitMessageAdmission({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + messageId, + content, + submittedContentDigest: messageContentDigest(content), + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: index + 1, + }); + } + + fixture.setRootState({ kind: 'idle' }); + await fixture.coordinator.recoverPendingAfterHostRestart([ROOT.sessionId]); + + assert.deepEqual( + fixture.recoveredBatches.map((batch) => batch.sources.map((source) => source.messageId)), + [['recovered-first']], + ); + assert.deepEqual( + fixture.coordinator.projection(ROOT.sessionId).followup.map((entry) => entry.messageId), + ['recovered-second'], + ); + + const projection = fixture.coordinator.projection(ROOT.sessionId); + const remainingEntryId = projection.followup[0]?.entryId; + assert.ok(remainingEntryId); + const updated = await fixture.coordinator.handlers['queue.entry.update']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + entryId: remainingEntryId, + updateId: 'update-recovered-second', + expectedQueueRevision: projection.queueRevision, + text: 'edited after recovery', + }, + operationContext(), + ); + assert.equal(updated.ok, true); + assert.deepEqual(fixture.readMessageAdmission('recovered-second'), { + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + messageId: 'recovered-second', + content: { text: 'edited after recovery' }, + submittedContentDigest: messageContentDigest({ text: 'edited after recovery' }), + submittedPlacement: 'next_turn', + placement: 'next_turn', + disposition: 'followup', + admittedAt: 2, + }); +}); + +test('recovery folds later steering ahead of an earlier explicit follow-up', async () => { + const fixture = createFixture(); + for (const admission of [ + { + messageId: 'recovered-followup', + content: { text: 'future work' }, + submittedPlacement: 'next_turn' as const, + placement: 'next_turn' as const, + disposition: 'followup' as const, + admittedAt: 1, + }, + { + messageId: 'recovered-steering', + content: { text: 'correct the current work' }, + submittedPlacement: 'current_turn' as const, + placement: 'current_turn' as const, + disposition: 'steering' as const, + admittedAt: 2, + }, + ]) { + await fixture.admissions.commitMessageAdmission({ + sessionId: ROOT.sessionId, + turnId: ROOT.turnId, + runId: ROOT.runId, + ...admission, + submittedContentDigest: messageContentDigest(admission.content), + }); + } + + fixture.setRootState({ kind: 'idle' }); + await fixture.coordinator.recoverPendingAfterHostRestart([ROOT.sessionId]); + + assert.deepEqual( + fixture.recoveredBatches.map((batch) => batch.sources.map((source) => source.messageId)), + [['recovered-steering']], + ); + assert.deepEqual( + fixture.coordinator.projection(ROOT.sessionId).followup.map((entry) => entry.messageId), + ['recovered-followup'], + ); +}); + // The Host stopped after the Message admission committed and before the root // admission that carries the exact-Turn intent was written. async function recoverExactTurnAcrossHostStop(): Promise> { @@ -641,24 +756,24 @@ test('full snapshot preflight rejection leaves queue, replay outcome, residency, await fixture.coordinator.close(); }); -test('queue admission rejects content that cannot form a durable follow-up Turn', async () => { +test('separate follow-ups do not share one root-admission capacity budget', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); const first = await submit(fixture, 'large-followup', 'x'.repeat(40 * 1024), 'next_turn'); assert.equal(first.ok && first.result.disposition, 'followup'); - const projectionBefore = structuredClone(fixture.coordinator.projection(ROOT.sessionId)); - - const rejected = await submitContent( + const second = await submitContent( fixture, 'display-followup', { text: 'model', displayText: 'human' }, 'next_turn', ); - assert.equal(rejected.ok, false); - if (!rejected.ok) assert.equal(rejected.error.code, 'session_busy'); - assert.deepEqual(fixture.coordinator.projection(ROOT.sessionId), projectionBefore); - assert.equal(fixture.liveResidencies(), 1); + assert.equal(second.ok && second.result.disposition, 'followup'); + assert.deepEqual( + fixture.coordinator.projection(ROOT.sessionId).followup.map((entry) => entry.messageId), + ['large-followup', 'display-followup'], + ); + assert.equal(fixture.liveResidencies(), 2); const retracted = await fixture.coordinator.handlers['queue.retract']( { originHostEpoch: 'epoch-1', sessionId: ROOT.sessionId, retractId: 'cleanup-large' }, @@ -1104,6 +1219,92 @@ test('entry promote moves a follow-up into the steering queue', async () => { assert.equal(fixture.liveResidencies(), 0); }); +test('a carried follow-up promoted in its successor requeues after nack', async () => { + const fixture = createFixture(); + fixture.coordinator.reserveRootTurn(ROOT); + const firstOwner = fixture.coordinator.bindRun(ROOT); + await submit(fixture, 'first-successor', 'first', 'next_turn'); + await submit(fixture, 'carried-followup', 'second', 'next_turn'); + firstOwner.release(); + const firstBatch = fixture.coordinator.beginTerminalTransition(ROOT); + const successor = { sessionId: ROOT.sessionId, turnId: 'turn-2', runId: 'run-2' }; + fixture.coordinator.commitNextRoot(firstBatch, successor); + fixture.setRootState({ kind: 'active', ...successor }); + + const owner = fixture.coordinator.bindRun(successor); + const entryId = fixture.coordinator.projection(ROOT.sessionId).followup[0]?.entryId; + assert.ok(entryId); + const promoted = await fixture.coordinator.handlers['queue.entry.promote']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + entryId, + promoteId: 'promote-carried-for-nack', + }, + operationContext(), + ); + assert.equal(promoted.ok, true); + const leases = owner.pull(); + assert.deepEqual( + leases.map((lease) => lease.messageId), + ['carried-followup'], + ); + owner.nack(leases.map((lease) => lease.id)); + assert.deepEqual( + fixture.coordinator.projection(ROOT.sessionId).steering.map((entry) => entry.messageId), + ['carried-followup'], + ); +}); + +test('an acked carried follow-up is not redelivered after restart', async () => { + const fixture = createFixture(); + fixture.coordinator.reserveRootTurn(ROOT); + const firstOwner = fixture.coordinator.bindRun(ROOT); + await submit(fixture, 'first-successor', 'first', 'next_turn'); + await submit(fixture, 'carried-followup', 'second', 'next_turn'); + firstOwner.release(); + const firstBatch = fixture.coordinator.beginTerminalTransition(ROOT); + const successor = { sessionId: ROOT.sessionId, turnId: 'turn-2', runId: 'run-2' }; + fixture.coordinator.commitNextRoot(firstBatch, successor); + fixture.setRootState({ kind: 'active', ...successor }); + + const owner = fixture.coordinator.bindRun(successor); + const entryId = fixture.coordinator.projection(ROOT.sessionId).followup[0]?.entryId; + assert.ok(entryId); + const promoted = await fixture.coordinator.handlers['queue.entry.promote']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + entryId, + promoteId: 'promote-carried-for-ack', + }, + operationContext(), + ); + assert.equal(promoted.ok, true); + const leases = owner.pull(); + assert.equal(leases.length, 1); + owner.ack(leases.map((lease) => lease.id)); + fixture.events.push({ + ...steeringEvent('carried-followup', 'second'), + turnId: successor.turnId, + runId: successor.runId, + }); + + await fixture.coordinator.materializeMessageHandoffsForRun({ + ...successor, + messageIds: [], + }); + assert.equal(fixture.readMessageAdmission('carried-followup'), undefined); + await fixture.admissions.markMessagesHandedOff({ + sessionId: ROOT.sessionId, + messageIds: ['first-successor'], + turnId: successor.turnId, + }); + fixture.setRootState({ kind: 'idle' }); + await fixture.coordinator.recoverPendingAfterHostRestart([ROOT.sessionId]); + assert.deepEqual(fixture.recoveredBatches, []); +}); + test('editing a promoted entry preserves its original submitted placement', async () => { const fixture = createFixture(); fixture.coordinator.reserveRootTurn(ROOT); @@ -1277,11 +1478,19 @@ test('queued mutations reject a queue that is draining into the next Turn', asyn originHostEpoch: 'epoch-1', sessionId: ROOT.sessionId, reorderId: 'reorder-after-commit', - entryIds: [], + entryIds: ['id-2'], }, operationContext(), ); assert.equal(after.ok, true); + await fixture.coordinator.handlers['queue.retract']( + { + originHostEpoch: 'epoch-1', + sessionId: ROOT.sessionId, + retractId: 'cleanup-after-commit', + }, + operationContext(), + ); fixture.coordinator.abandonRootReservation({ sessionId: ROOT.sessionId, turnId: 'turn-2', @@ -1645,10 +1854,10 @@ test('release folds unpulled steering ahead of follow-up without changing source owner.release(); const batch = fixture.coordinator.beginTerminalTransition(ROOT); assert.deepEqual(batch.content, { - text: 'first\n\nsecond\n\nthird', - displayText: 'first\n\nsecond\n\nthird', - attachments: [firstAttachment, secondAttachment, thirdAttachment], - quotes: [...firstQuotes, ...secondQuotes, ...thirdQuotes], + text: 'first\n\nsecond', + displayText: 'first\n\nsecond', + attachments: [firstAttachment, secondAttachment], + quotes: [...firstQuotes, ...secondQuotes], }); assert.deepEqual(batch.sources, [ { @@ -1679,42 +1888,42 @@ test('release folds unpulled steering ahead of follow-up without changing source placement: 'current_turn', disposition: 'steering', }, - { - messageId: 'follow-1', - content: { - text: 'third', - displayText: 'third', - attachments: [thirdAttachment], - quotes: thirdQuotes, - }, - submittedContentDigest: messageContentDigest({ - text: 'third', - displayText: 'third', - attachments: [thirdAttachment], - quotes: thirdQuotes, - }), - placement: 'next_turn', - disposition: 'followup', - }, ]); assert.equal(fixture.liveResidencies(), 3); - fixture.coordinator.commitNextRoot(batch, { + const steeringSuccessor = { sessionId: ROOT.sessionId, turnId: 'turn-2', runId: 'run-2', + }; + fixture.coordinator.commitNextRoot(batch, steeringSuccessor); + assert.equal(fixture.liveResidencies(), 1); + const next = fixture.coordinator.bindRun(steeringSuccessor); + next.release(); + const followupBatch = fixture.coordinator.beginTerminalTransition(steeringSuccessor); + assert.deepEqual(followupBatch.content, { + text: 'third', + displayText: 'third', + attachments: [thirdAttachment], + quotes: thirdQuotes, }); - assert.equal(fixture.liveResidencies(), 0); - const next = fixture.coordinator.bindRun({ + assert.deepEqual( + followupBatch.sources.map((source) => source.messageId), + ['follow-1'], + ); + const followupSuccessor = { sessionId: ROOT.sessionId, - turnId: 'turn-2', - runId: 'run-2', - }); - next.release(); + turnId: 'turn-3', + runId: 'run-3', + }; + fixture.coordinator.commitNextRoot(followupBatch, followupSuccessor); + assert.equal(fixture.liveResidencies(), 0); + const final = fixture.coordinator.bindRun(followupSuccessor); + final.release(); const empty = fixture.coordinator.beginTerminalTransition({ sessionId: ROOT.sessionId, - turnId: 'turn-2', - runId: 'run-2', + turnId: 'turn-3', + runId: 'run-3', }); fixture.coordinator.completeIdle(empty); }); @@ -2548,6 +2757,14 @@ function memoryMessageAdmissionStore( updateMessageAdmission: async (admission) => { const existing = admissions.get(admission.messageId); if (!existing) throw new Error(`Missing admission ${admission.messageId}`); + if ( + existing.admission.turnId !== admission.turnId || + existing.admission.runId !== admission.runId || + existing.admission.submittedPlacement !== admission.submittedPlacement || + existing.admission.admittedAt !== admission.admittedAt + ) { + throw new Error(`Message admission update identity conflict: ${admission.messageId}`); + } existing.admission = admission; }, reorderMessageAdmissions: async () => undefined, diff --git a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts index 0d04bd9c60..7de0689e58 100644 --- a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts @@ -3447,7 +3447,7 @@ test('an exact active retry preserves the Client Capability admission binding', } }); -test('mixed-Client queued follow-ups use one Session successor without connection-local tools', { +test('mixed-Client queued follow-ups use separate Session successors without connection-local tools', { timeout: 20_000, }, async () => { const clientCapabilities = new HostClientCapabilityCoordinator({ @@ -3557,6 +3557,8 @@ test('mixed-Client queued follow-ups use one Session successor without connectio await waitUntil(() => backend?.sendCount === 2); backend?.release(); + await waitUntil(() => backend?.sendCount === 3); + backend?.release(); await waitUntil( () => fixture.coordinator.readRootState(fixture.sessionId).kind === 'idle', 5_000, @@ -3566,7 +3568,7 @@ test('mixed-Client queued follow-ups use one Session successor without connectio ); assert.deepEqual( admissions.map((admission) => admission.sourceMessages.map((source) => source.messageId)), - [[], ['followup-from-provider-b', 'followup-from-provider-a']], + [[], ['followup-from-provider-b'], ['followup-from-provider-a']], ); assert.deepEqual( (await fixture.stores.sessionStore.readMessages(fixture.sessionId)) diff --git a/packages/runtime-host/src/server/message-coordinator.ts b/packages/runtime-host/src/server/message-coordinator.ts index 1db629328d..b85dc4935f 100644 --- a/packages/runtime-host/src/server/message-coordinator.ts +++ b/packages/runtime-host/src/server/message-coordinator.ts @@ -221,15 +221,15 @@ export type CandidateSnapshotPreflight = ( interface LiveEntry { readonly entryId: string; readonly messageId: string; - readonly turnId: string; - readonly runId: string; + readonly admissionTurnId: string; + readonly admissionRunId: string; readonly admittedAt: number; content: MessageContent; modelContent: MessageContent; submittedContentDigest: `sha256:${string}`; readonly placement: MessagePlacement; readonly disposition: 'steering' | 'followup'; - readonly generation: number; + generation: number; readonly residency: RuntimeHostResidency; state: 'queued' | 'in_flight' | 'released'; leaseId?: string; @@ -515,7 +515,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { this.#mutated(state); } state.run = undefined; - const entries = [...state.followup]; + const entries = nextSuccessorItems(state.followup); const followup = canonicalFollowupBatch(entries); const transition: TerminalTransition = { transitionId: this.#createId(), @@ -540,6 +540,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } this.#commitTransition(state); state.generation += 1; + for (const entry of allLiveEntries(state)) entry.generation = state.generation; state.reservedRoot = { ...identity }; state.phase = 'open'; this.#mutated(state); @@ -620,11 +621,7 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } } for (const admission of admissions) { - if ( - admission.turnId !== input.turnId || - admission.runId !== input.runId || - admission.disposition !== 'steering' - ) { + if (admission.disposition !== 'steering') { continue; } const proof = await this.#durableProof.readImmutableSteeringMessageProof( @@ -687,14 +684,11 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { sessionId, admission.messageId, ); - if ( - steering?.event.turnId === admission.turnId && - steering.event.runId === admission.runId - ) { + if (steering) { await this.materializeMessageHandoffsForRun({ sessionId, - turnId: admission.turnId, - runId: admission.runId, + turnId: steering.event.turnId, + runId: steering.event.runId, messageIds: [admission.messageId], }); } else { @@ -711,14 +705,19 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { 'Message recovery authority is unavailable', ); } + const recoveryOrder = [ + ...pending.filter((entry) => entry.disposition === 'steering'), + ...pending.filter((entry) => entry.disposition !== 'steering'), + ]; + const recoveryBatch = nextSuccessorItems(recoveryOrder); const started = await this.#root.startRecoveredMessages( { sessionId, - content: aggregateMessageContents(pending.map((entry) => entry.content)), - submittedContent: aggregateMessageContents(pending.map((entry) => entry.content)), - sources: pending.map(pendingMessageSource), - ...(pending.length === 1 && pending[0]!.submittedIntent - ? { submittedIntent: pending[0]!.submittedIntent } + content: aggregateMessageContents(recoveryBatch.map((entry) => entry.content)), + submittedContent: aggregateMessageContents(recoveryBatch.map((entry) => entry.content)), + sources: recoveryBatch.map(pendingMessageSource), + ...(recoveryBatch.length === 1 && recoveryBatch[0]!.submittedIntent + ? { submittedIntent: recoveryBatch[0]!.submittedIntent } : {}), }, admissionLease, @@ -728,14 +727,32 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { `Durable Message recovery failed: ${started.error}`, ); } + const recoveredMessageIds = new Set(recoveryBatch.map((entry) => entry.messageId)); + const remaining = pending.filter((entry) => !recoveredMessageIds.has(entry.messageId)); + if (remaining.length > 0) { + const active = await this.#root.readRootState(sessionId); + if (active.kind !== 'active') { + throw new RuntimeMessageAuthorityInvariantError( + 'Recovered successor did not become the active root Turn', + ); + } + this.#restorePendingAdmissions(sessionId, active, remaining); + } return; } + this.#restorePendingAdmissions(sessionId, rootState, pending); + } + + #restorePendingAdmissions( + sessionId: string, + rootState: RuntimeMessageRunIdentity & { readonly kind: 'active' }, + pending: readonly PendingMessageAdmission[], + ): void { if (!this.#sessions.has(sessionId)) this.#state(sessionId); const state = this.#requireState(sessionId); if (!state.reservedRoot) this.reserveRootTurn(rootState); if (!sameRun(state.reservedRoot!, rootState)) return; for (const admission of pending) { - if (admission.turnId !== rootState.turnId || admission.runId !== rootState.runId) continue; const existing = allLiveEntries(state).find( (entry) => entry.messageId === admission.messageId, ); @@ -744,8 +761,8 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { const entry: LiveEntry = { entryId: this.#createId(), messageId: admission.messageId, - turnId: admission.turnId, - runId: admission.runId, + admissionTurnId: admission.turnId, + admissionRunId: admission.runId, admittedAt: admission.admittedAt, content: submittedProjectionContent(admission.content), modelContent: admission.content, @@ -1023,19 +1040,20 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { if (!interruptResultFits(candidate, rootState)) { return failure('session_busy', 'Message queue interrupt result capacity is full'); } - const prospectiveSources = [ - ...[...state.inFlight.values(), ...state.steering, ...state.followup].map( - sourceFromEntry, - ), - { - messageId: input.messageId, - content: prepared.content, - submittedContentDigest: messageContentDigest(payload.content), - placement: input.placement, - disposition, - }, - ] satisfies RootTurnSourceMessage[]; - if (!rootAdmissionPayloadFits(prospectiveSources)) { + const candidateSource = { + messageId: input.messageId, + content: prepared.content, + submittedContentDigest: messageContentDigest(payload.content), + placement: input.placement, + disposition, + } satisfies RootTurnSourceMessage; + const prospectiveSteering = [...state.inFlight.values(), ...state.steering].map( + sourceFromEntry, + ); + const prospectiveFollowup = state.followup.map(sourceFromEntry); + if (disposition === 'steering') prospectiveSteering.push(candidateSource); + else prospectiveFollowup.push(candidateSource); + if (!successorAdmissionsFit(prospectiveSteering, prospectiveFollowup)) { return failure('session_busy', 'Message queue cannot form a durable follow-up Turn'); } if ( @@ -1068,8 +1086,8 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { const entry: LiveEntry = { entryId, messageId: input.messageId, - turnId: rootState.turnId, - runId: rootState.runId, + admissionTurnId: rootState.turnId, + admissionRunId: rootState.runId, admittedAt: messageAdmission.admittedAt, content: payload.content, modelContent: prepared.content, @@ -1357,10 +1375,25 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { } return failure('not_found', 'Message queue entry does not exist'); } + const promotedSource = { + ...sourceFromEntry(entry), + placement: 'current_turn', + disposition: 'steering', + } satisfies RootTurnSourceMessage; + const prospectiveSteering = [...state.inFlight.values(), ...state.steering].map( + sourceFromEntry, + ); + prospectiveSteering.push(promotedSource); + const prospectiveFollowup = state.followup + .filter((queued) => queued !== entry) + .map(sourceFromEntry); + if (!successorAdmissionsFit(prospectiveSteering, prospectiveFollowup)) { + return failure('session_busy', 'Promoted Message exceeds steering admission capacity'); + } await this.#admissions.updateMessageAdmission({ sessionId: input.sessionId, - turnId: entry.turnId, - runId: entry.runId, + turnId: entry.admissionTurnId, + runId: entry.admissionRunId, messageId: entry.messageId, content: entry.modelContent, submittedContentDigest: entry.submittedContentDigest, @@ -1432,16 +1465,17 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { if (!projectionFitsEveryEntryState(updatedProjection)) { return failure('session_busy', 'Message queue projection capacity is full'); } - const sources = allLiveEntries(state).map((entry) => + const updatedSource = (entry: LiveEntry): RootTurnSourceMessage => entry === queued.entry ? { ...sourceFromEntry(entry), content: modelContent, submittedContentDigest: messageContentDigest(content), } - : sourceFromEntry(entry), - ) satisfies RootTurnSourceMessage[]; - if (!rootAdmissionPayloadFits(sources)) { + : sourceFromEntry(entry); + const steeringSources = [...state.inFlight.values(), ...state.steering].map(updatedSource); + const followupSources = state.followup.map(updatedSource); + if (!successorAdmissionsFit(steeringSources, followupSources)) { return failure('session_busy', 'Message queue mutation exceeds root admission capacity'); } if (!(await this.#preflightSessionSnapshot(input.sessionId, { queue: updatedProjection }))) { @@ -1459,8 +1493,8 @@ export class HostMessageCoordinator implements RuntimeMessageAuthority { ); await this.#admissions.updateMessageAdmission({ sessionId: input.sessionId, - turnId: queued.entry.turnId, - runId: queued.entry.runId, + turnId: queued.entry.admissionTurnId, + runId: queued.entry.admissionRunId, messageId: queued.entry.messageId, content: modelContent, submittedContentDigest: messageContentDigest(content), @@ -2390,6 +2424,25 @@ function canonicalFollowupBatch(entries: readonly LiveEntry[]): { } } +/** + * One explicit next-turn Message owns one successor root Turn. Steering that + * missed the final provider boundary is different: those entries all targeted + * the finishing Turn, so keep their correction context together in the first + * successor rather than turning each interjection into unrelated future work. + */ +function nextSuccessorItems< + T extends { readonly disposition: 'steering' | 'followup' | 'turn_started' }, +>(entries: readonly T[]): T[] { + if (entries.length === 0) return []; + if (entries[0]!.disposition !== 'steering') return [entries[0]!]; + const steering: T[] = []; + for (const entry of entries) { + if (entry.disposition !== 'steering') break; + steering.push(entry); + } + return steering; +} + function rootAdmissionPayloadFits(sources: readonly RootTurnSourceMessage[]): boolean { try { const content = aggregateMessageContent(sources.map((source) => source.content)); @@ -2400,6 +2453,16 @@ function rootAdmissionPayloadFits(sources: readonly RootTurnSourceMessage[]): bo } } +function successorAdmissionsFit( + steering: readonly RootTurnSourceMessage[], + followup: readonly RootTurnSourceMessage[], +): boolean { + return ( + (steering.length === 0 || rootAdmissionPayloadFits(steering)) && + followup.every((source) => rootAdmissionPayloadFits([source])) + ); +} + function interruptResultFits( projection: SessionMessageQueueProjection, identity: RuntimeMessageRunIdentity,