From 2f21782419f77972d2131ec0c34adeda9dc02ff8 Mon Sep 17 00:00:00 2001 From: Michael Fox <85814106+q1@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:05:45 -0700 Subject: [PATCH 1/2] fix(server): resume failed Claude sessions before retrying Restart idle Claude runtimes with a recorded turn failure on the next explicit turn while retaining the resume cursor and model options. Cover unchanged retries and the native terminal 429 result shape. Fork-Feature: base Upstream: candidate --- .../Layers/ProviderCommandReactor.test.ts | 134 ++++++++++++++++++ .../Layers/ProviderCommandReactor.ts | 13 +- .../src/provider/Layers/ClaudeAdapter.test.ts | 63 ++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index d93fec5a3cf6..7bd09e62367e 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -2866,6 +2866,140 @@ describe("ProviderCommandReactor", () => { }); }); + it.each([ + { + name: "omitted selection", + instanceId: "claudeAgent", + failed: true, + running: false, + selection: false, + restart: true, + }, + { + name: "unchanged selection", + instanceId: "claudeAgent", + failed: true, + running: false, + selection: true, + restart: true, + }, + { + name: "named Claude instance", + instanceId: "claude_work", + failed: true, + running: false, + selection: false, + restart: true, + }, + { + name: "healthy Claude session", + instanceId: "claudeAgent", + failed: false, + running: false, + selection: false, + restart: false, + }, + { + name: "active Claude turn", + instanceId: "claudeAgent", + failed: true, + running: true, + selection: false, + restart: false, + }, + { + name: "another provider", + instanceId: "codex", + failed: true, + running: false, + selection: false, + restart: false, + }, + ])("restarts only failed idle Claude sessions on retry: $name", async (scenario) => { + const instanceId = ProviderInstanceId.make(scenario.instanceId); + const modelSelection = createModelSelection( + instanceId, + scenario.instanceId === "codex" ? "gpt-5-codex" : "claude-fable-5-1", + scenario.instanceId === "codex" ? [] : [{ id: "effort", value: "high" }], + ); + const harness = await createHarness({ threadModelSelection: modelSelection }); + const threadId = ThreadId.make("thread-1"); + const now = "2026-01-01T00:00:00.000Z"; + const send = async (id: string, selection?: ModelSelection) => { + const sent = await harness.runEffect(Deferred.make()); + const sendTurn = harness.sendTurn.getMockImplementation()!; + harness.sendTurn.mockImplementationOnce((input) => + sendTurn(input).pipe(Effect.tap(() => Deferred.succeed(sent, undefined))), + ); + await harness.runEffect( + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`cmd-${id}`), + threadId, + message: { + messageId: asMessageId(`message-${id}`), + role: "user", + text: "Continue", + attachments: [], + }, + ...(selection ? { modelSelection: selection } : {}), + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + createdAt: now, + }), + ); + await harness.runEffect(Deferred.await(sent)); + await harness.drain(); + }; + await send("initial", modelSelection); + expect(harness.startSession).toHaveBeenCalledTimes(1); + const original = harness.runtimeSessions[0]!; + const lastError = scenario.failed ? "Claude gave up after repeated API errors." : undefined; + harness.runtimeSessions[0] = { + ...original, + status: scenario.running ? "running" : "ready", + ...(scenario.running ? { activeTurnId: asTurnId("active-turn") } : {}), + ...(lastError ? { lastError } : {}), + }; + await harness.runEffect( + harness.engine.dispatch({ + type: "thread.session.set", + commandId: CommandId.make("cmd-failed-session"), + threadId, + session: { + threadId, + providerName: original.provider, + providerInstanceId: instanceId, + status: scenario.running ? "running" : scenario.failed ? "error" : "ready", + runtimeMode: "approval-required", + activeTurnId: scenario.running ? asTurnId("active-turn") : null, + lastError: lastError ?? null, + updatedAt: now, + }, + createdAt: now, + }), + ); + await harness.drain(); + + await send("retry", scenario.selection ? modelSelection : undefined); + + expect(harness.startSession).toHaveBeenCalledTimes(scenario.restart ? 2 : 1); + expect(harness.sendTurn).toHaveBeenCalledTimes(2); + if (scenario.restart) { + expect(harness.startSession.mock.calls[1]?.[1]).toMatchObject({ + threadId, + providerInstanceId: instanceId, + modelSelection, + resumeCursor: original.resumeCursor, + runtimeMode: original.runtimeMode, + cwd: original.cwd, + }); + expect(harness.startSession.mock.invocationCallOrder[1]).toBeLessThan( + harness.sendTurn.mock.invocationCallOrder[1]!, + ); + } + }); + it("restarts claude sessions when claude effort changes", async () => { const harness = await createHarness({ threadModelSelection: { diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 5c1086b9e29c..81c7464202c9 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -783,13 +783,23 @@ const make = Effect.gen(function* () { preferredProvider === "claudeAgent" && requestedModelSelection !== undefined && !Equal.equals(previousModelSelection, requestedModelSelection); + // Claude leaves its CLI alive after a failed result, including terminal + // usage-limit errors. Resume the next user turn in a fresh runtime so + // process-local failure state cannot keep rejecting an unchanged retry. + // Keep the conversation cursor and never interrupt an active turn. + const shouldRestartAfterFailure = + preferredProvider === "claudeAgent" && + activeSession?.status === "ready" && + activeSession.activeTurnId === undefined && + activeSession.lastError !== undefined; if ( !runtimeModeChanged && !cwdChanged && !instanceChanged && !shouldRestartForModelChange && - !shouldRestartForModelSelectionChange + !shouldRestartForModelSelectionChange && + !shouldRestartAfterFailure ) { yield* refreshWorkspaceSnapshot; return existingSessionThreadId; @@ -815,6 +825,7 @@ const make = Effect.gen(function* () { instanceChanged, shouldRestartForModelChange, shouldRestartForModelSelectionChange, + shouldRestartAfterFailure, hasResumeCursor: resumeCursor !== undefined, }); const restartedSession = yield* startProviderSession( diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts index 13b44c1669fa..ba3ba28344ee 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.test.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.test.ts @@ -3746,6 +3746,69 @@ describe("ClaudeAdapterLive", () => { return { runtimeEvents, runtimeEventsFiber, drainSdkMessages }; }); + it.effect( + "retains the failed result and resume cursor after a zero-API-time limit rejection", + () => { + const harness = makeHarness(); + return Effect.gen(function* () { + const adapter = yield* ClaudeAdapter; + const { runtimeEvents, runtimeEventsFiber, drainSdkMessages } = + yield* observeUsageLimitEvents(adapter, harness.query); + yield* adapter.startSession({ + threadId: THREAD_ID, + provider: ProviderDriverKind.make("claudeAgent"), + runtimeMode: "full-access", + }); + yield* adapter.sendTurn({ threadId: THREAD_ID, input: "Continue", attachments: [] }); + harness.query.emit({ + type: "rate_limit_event", + rate_limit_info: { + status: "rejected", + rateLimitType: "seven_day_overage_included", + resetsAt: 1788825600, + overageStatus: "rejected", + isUsingOverage: false, + }, + session_id: "sdk-session-limit", + uuid: "limit-rejected", + } as unknown as SDKMessage); + harness.query.emit({ + type: "result", + subtype: "success", + is_error: true, + terminal_reason: "api_error", + api_error_status: 429, + duration_ms: 500, + duration_api_ms: 0, + total_cost_usd: 0, + num_turns: 1, + errors: [], + result: "", + session_id: "sdk-session-limit", + uuid: "result-limit-rejected", + } as unknown as SDKMessage); + yield* drainSdkMessages; + const sessions = yield* adapter.listSessions(); + assert.equal(sessions.length, 1); + assert.equal(sessions[0]?.status, "ready"); + assert.equal(sessions[0]?.activeTurnId, undefined); + assert.equal(sessions[0]?.lastError, "Claude gave up after repeated API errors."); + assert.equal( + (sessions[0]?.resumeCursor as { resume?: string })?.resume, + "sdk-session-limit", + ); + assert.equal(harness.query.closeCalls, 0); + assert.equal(runtimeEvents.filter((event) => event.type === "runtime.warning").length, 1); + const completed = runtimeEvents.find((event) => event.type === "turn.completed"); + assert.equal(completed?.type === "turn.completed" && completed.payload.state, "failed"); + yield* Fiber.interrupt(runtimeEventsFiber); + }).pipe( + Effect.provideService(Random.Random, makeDeterministicRandomService()), + Effect.provide(harness.layer), + ); + }, + ); + it.effect("surfaces a rejected Claude usage limit once per turn", () => { const harness = makeHarness(); return Effect.gen(function* () { From fcd986d315c7d62769e97cca0360d542d9bdead8 Mon Sep 17 00:00:00 2001 From: Michael Fox <85814106+q1@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:05:46 -0700 Subject: [PATCH 2/2] fix(ios): show runtime limit warnings and error messages Keep runtime warnings visible in loaded history and live transcript updates. Render the provider message instead of only the generic runtime error summary. Fork-Feature: swift-ios Upstream: no --- apps/swift-ios/App/NativeFeatureClient.swift | 32 +++++++++---- .../FeatureTests/RuntimeNoticeTests.swift | 48 +++++++++++++++++++ 2 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 apps/swift-ios/Tests/FeatureTests/RuntimeNoticeTests.swift diff --git a/apps/swift-ios/App/NativeFeatureClient.swift b/apps/swift-ios/App/NativeFeatureClient.swift index 5ee47a5d1068..67da6256ebe8 100644 --- a/apps/swift-ios/App/NativeFeatureClient.swift +++ b/apps/swift-ios/App/NativeFeatureClient.swift @@ -4110,7 +4110,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, var changedIDs = Set(mutations.messages.map(\.id)) for activity in mutations.activities { - if activity.tone == "error" { + if Self.activityNoticeText(activity) != nil { changedIDs.insert("activity-\(activity.id)") } else if NativeWorkLogAccumulator.accepts(activity) { changedIDs.insert("work-log-\(activity.turnId ?? "unscoped")") @@ -4527,10 +4527,10 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, } cache.approvals = pendingApprovals(thread, environment: environment) cache.userInputs = pendingUserInputs(thread, environment: environment) - let errors = thread.activities.compactMap(mapErrorActivity) + let notices = thread.activities.compactMap(mapNoticeActivity) let sessionIsLive = thread.session?.status == "starting" || thread.session?.status == "running" - let activityMessages = (errors + collapsedWorkLogs( + let activityMessages = (notices + collapsedWorkLogs( thread.activities, sessionIsLive: sessionIsLive )) @@ -4743,7 +4743,7 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, let workIsLive = thread.session?.status == "starting" || thread.session?.status == "running" || backgroundLiveness(threadID: thread.id, environmentID: environmentID) == .working - let activities = thread.activities.compactMap(mapErrorActivity) + let activities = thread.activities.compactMap(mapNoticeActivity) + collapsedWorkLogs(thread.activities, sessionIsLive: workIsLive) return (messages + activities).sorted { $0.createdAt < $1.createdAt } } @@ -4838,8 +4838,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, environment: environment, cache: cache ) - if let error = mapErrorActivity(activity) { - upsertMergedMessage(error, cache: cache) + if let notice = mapNoticeActivity(activity) { + upsertMergedMessage(notice, cache: cache) } guard NativeWorkLogAccumulator.accepts(activity), cache.workLogActivityIDs.insert(activity.id).inserted else { @@ -5012,10 +5012,22 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging, ) } - private func mapErrorActivity(_ activity: OrchestrationActivity) -> FeatureMessage? { - guard activity.tone == "error" else { return nil } - let detail = activity.payload["detail"]?.stringValue - let text = detail.map { "\(activity.summary)\n\($0)" } ?? activity.summary + static func activityNoticeText(_ activity: OrchestrationActivity) -> String? { + guard activity.tone == "error" || activity.kind == "runtime.warning" else { return nil } + if let message = activity.payload["message"]?.stringValue, + !message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return message + } + if let detail = activity.payload["detail"]?.stringValue, + !detail.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + detail != activity.summary { + return "\(activity.summary)\n\(detail)" + } + return activity.summary + } + + private func mapNoticeActivity(_ activity: OrchestrationActivity) -> FeatureMessage? { + guard let text = Self.activityNoticeText(activity) else { return nil } return FeatureMessage( id: "activity-\(activity.id)", role: .system, diff --git a/apps/swift-ios/Tests/FeatureTests/RuntimeNoticeTests.swift b/apps/swift-ios/Tests/FeatureTests/RuntimeNoticeTests.swift new file mode 100644 index 000000000000..7dc23119607c --- /dev/null +++ b/apps/swift-ios/Tests/FeatureTests/RuntimeNoticeTests.swift @@ -0,0 +1,48 @@ +import Foundation +import Testing +@testable import T3Code + +@MainActor +@Suite("Runtime notices") +struct RuntimeNoticeTests { + @Test + func showsTheUsageLimitWarningWithoutAnAssistantMessage() { + let message = "Claude usage limit reached. This turn is paused until the 7-day Fable limit resets in 27h 18m." + let warning = activity(tone: "warning", kind: "runtime.warning", summary: message, + payload: ["message": .string(message), "detail": .object(["status": .string("rejected")])]) + #expect(NativeFeatureClient.activityNoticeText(warning) == message) + } + + @Test + func showsTheRuntimeErrorMessageInsteadOfItsGenericSummary() { + let error = activity(tone: "error", kind: "runtime.error", summary: "Runtime error", + payload: ["message": .string("Claude gave up after repeated API errors.")]) + #expect(NativeFeatureClient.activityNoticeText(error) == "Claude gave up after repeated API errors.") + } + + @Test + func preservesLegacyDetailsAndAvoidsDuplicatingTheSummary() { + let legacy = activity(tone: "error", kind: "provider.turn.start.failed", summary: "Turn failed", + payload: ["detail": .string("Provider unavailable")]) + #expect(NativeFeatureClient.activityNoticeText(legacy) == "Turn failed\nProvider unavailable") + let repeated = activity(tone: "warning", kind: "runtime.warning", summary: "Limit reached", + payload: ["message": .string(" "), "detail": .string("Limit reached")]) + #expect(NativeFeatureClient.activityNoticeText(repeated) == "Limit reached") + } + + @Test + func keepsOrdinaryActivitiesOutOfTheTranscriptAndDoesNotDumpStructuredDetails() { + let progress = activity(tone: "info", kind: "tool.updated", summary: "Running", + payload: ["message": .string("Tool progress")]) + #expect(NativeFeatureClient.activityNoticeText(progress) == nil) + let diagnostic = activity(tone: "error", kind: "runtime.error", summary: "Runtime error", + payload: ["detail": .object(["internal": .string("diagnostic")])]) + #expect(NativeFeatureClient.activityNoticeText(diagnostic) == "Runtime error") + } + + private func activity(tone: String, kind: String, summary: String, payload: [String: JSONValue]) -> OrchestrationActivity { + OrchestrationActivity(id: "notice", tone: tone, kind: kind, summary: summary, + payload: .object(payload), turnId: "turn", sequence: 1, + createdAt: "2026-09-06T20:42:22Z") + } +}