diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 8a467fd92..23dd81e89 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -14,6 +14,7 @@ import { createSidecarAllocationStore, createSignalCorrelationStore, createWorkflowRunDispatchStore, + createWorkflowRunStore, listVisibleOfferings, resolveCredentialByName, resolveCredentialRequirement, @@ -118,6 +119,7 @@ import type { FinalizedTurnToolCall } from "@corbits/turn-artifacts"; import { decodedOrNull } from "@corbits/url-path"; import { createCryptoProviderCache, + findFoldedRunByAddress, lookupFoldedRunReconnectKey, } from "@corbits/folded-runs"; import { createTopLevelRunRoutes } from "@corbits/run-scope"; @@ -209,6 +211,7 @@ import { resolveLaunchableDefinition, routine as routineTable, routineRun as routineRunTable, + settleRoutineFireFromTurn, } from "@corbits/routines"; import { createSidecarProvisioner as createE2BSidecarProvisioner, @@ -817,6 +820,31 @@ export async function createHub(config: HubConfig) { store: insightsUsage.store, generateId: () => generateId("inferenceTurn"), }); + // CL-6778: a routine fire's folded delivery agent stays deployed after + // it replies, so `workflow_run.status` would linger on `running` unless + // something stamps it terminal. `onTurnFinalized` is that stamp — + // `markTerminal` + `endedAt`, gated on a `routine_run` row so a + // workbench host or invited agent is never settled by accident. + const workflowRunStore = createWorkflowRunStore(db); + const routineFireSettlePort = { + async lookupRunByAddress(address: string) { + const row = await findFoldedRunByAddress(db, address); + return row === undefined ? undefined : { id: row.id }; + }, + async isRoutineFire(runId: string) { + const rows = await db + .select({ runId: routineRunTable.runId }) + .from(routineRunTable) + .where(eq(routineRunTable.runId, runId)) + .limit(1); + return rows.length > 0; + }, + markTerminal: ( + runId: string, + status: "completed" | "failed" | "cancelled", + endedAt: Date, + ) => workflowRunStore.markTerminal(runId, status, endedAt), + }; // CL-6257: per-message-run stage latency (message-received → // reactor.start → inference.start → first-token → reply-posted). The // vendored event collector never persists the events this reads (see @@ -835,8 +863,19 @@ export async function createHub(config: HubConfig) { // surviving loss loud (error-level cause, counted) instead of a // swallowed WRN. db: withTurnPartPersistGuard(withTurnPartWriteDefaults(db)), - onTurnFinalized: (agentAddress, turn) => - artifactDeliveryHandlerRef.current?.(agentAddress, turn), + onTurnFinalized: (agentAddress, turn) => { + artifactDeliveryHandlerRef.current?.(agentAddress, turn); + void settleRoutineFireFromTurn(routineFireSettlePort, agentAddress, { + status: turn.status, + hadReply: turn.hadReply, + }).catch((err: unknown) => { + log.warn`Failed to settle routine fire for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`; + reportError(err, { + operation: "routine_fire_settle_from_turn", + agentId: agentAddress, + }); + }); + }, // Per-turn usage, emitted once when the collector finalizes a turn. onUsage: (_agentAddress, usage) => { void usageSink diff --git a/apps/hub/src/routine-run-summary.ts b/apps/hub/src/routine-run-summary.ts index c064611e9..98f89e624 100644 --- a/apps/hub/src/routine-run-summary.ts +++ b/apps/hub/src/routine-run-summary.ts @@ -8,6 +8,7 @@ import { and, eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { workflowRun } from "@intx/db/schema"; import type { RunSummaryResolver } from "@corbits/routines"; +import { listingTurnsByRunId } from "@corbits/run-scope"; export function createHubRunSummaryResolver(db: DB["db"]): RunSummaryResolver { return { @@ -19,10 +20,13 @@ export function createHubRunSummaryResolver(db: DB["db"]): RunSummaryResolver { ), }); if (row === undefined) return undefined; + const turns = (await listingTurnsByRunId(db, [runId])).get(runId) ?? []; return { status: row.status, createdAt: row.createdAt.toISOString(), endedAt: row.endedAt?.toISOString() ?? null, + hasInFlightTurn: turns.length > 0, + turns, }; }, }; diff --git a/apps/web/src/agents-api.ts b/apps/web/src/agents-api.ts index 5cc0d76e9..4bcff2657 100644 --- a/apps/web/src/agents-api.ts +++ b/apps/web/src/agents-api.ts @@ -31,7 +31,15 @@ export type AgentInstance = typeof WorkflowRunResponse.infer; export type CatalogModel = typeof ModelResponse.infer; const RunFireResponse = WorkflowRunResponse.and( - type({ routineId: "string | null", routineName: "string | null" }), + type({ + routineId: "string | null", + routineName: "string | null", + "hasInFlightTurn?": "boolean", + "turns?": type({ + status: "string", + "endedAt?": "string | null", + }).array(), + }), ); /** A `feed=fires` row: every `AgentInstance` field plus the routine that * fired it (both `null` for a directly-triggered deployment with no diff --git a/apps/web/src/insights-api.ts b/apps/web/src/insights-api.ts index 35a954009..63cd69268 100644 --- a/apps/web/src/insights-api.ts +++ b/apps/web/src/insights-api.ts @@ -224,17 +224,28 @@ export function insightsScopePath(tenantId: string): string { return `/api/tenants/${tenantId}/insights/scope`; } +export const ListingTurnSchema = type({ + status: "string", + "endedAt?": "string | null", +}); + /** * `WorkflowRunResponse` plus the two fields only the `feed=fires` mode of * `/top-level-runs` reports (CL-6249): the routine that fired this run, * when it has one. Both are `null` for a run with no routine/task * parent — a directly launched workflow — so a caller falls back to * `definitionName` honestly instead of inventing a routine. + * `turns` / `hasInFlightTurn` are this build's listing of in-flight + * inference turns for the run (not Interchange fields) so a live + * tool-loop can stay running past the abandoned-fire window. Omitting + * them is not "no in-flight turn". */ export const InsightsRunSchema = WorkflowRunResponse.and( type({ routineId: "string | null", routineName: "string | null", + "hasInFlightTurn?": "boolean", + "turns?": ListingTurnSchema.array(), }), ); export type InsightsRun = typeof InsightsRunSchema.infer; diff --git a/apps/web/src/insights-stats.test.ts b/apps/web/src/insights-stats.test.ts index 6d8818a20..420d378cf 100644 --- a/apps/web/src/insights-stats.test.ts +++ b/apps/web/src/insights-stats.test.ts @@ -1,10 +1,13 @@ import { describe, expect, test } from "bun:test"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; + import { computeInsightsStats, computeTraceStats, filterRunsByCreatedAt, groupRunsByDefinition, + INSIGHTS_RECENT_LIMIT, purposeRunsForInsights, runDisplayName, } from "./insights-stats"; @@ -94,6 +97,8 @@ describe("computeInsightsStats", () => { routine({ id: "r1", enabled: true }), routine({ id: "r2", enabled: false }), ], + INSIGHTS_RECENT_LIMIT, + Date.parse("2026-01-03T00:01:00.000Z"), ); expect(stats.totalRuns).toBe(3); @@ -105,6 +110,37 @@ describe("computeInsightsStats", () => { expect(stats.recentRuns.map((r) => r.id)).toEqual(["1", "2", "3"]); }); + test("a live running run past the fire window is still counted as running", () => { + const stats = computeInsightsStats( + [ + run({ + id: "stale", + status: "running", + createdAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + }), + ], + [], + ); + expect(stats.running).toBe(1); + }); + + test("endedAt drops a just-finished running run from the running count immediately", () => { + const stats = computeInsightsStats( + [ + run({ + id: "just-finished", + status: "running", + createdAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + }), + ], + [], + ); + expect(stats.running).toBe(0); + }); + test("limits recent runs", () => { const runs = Array.from({ length: 5 }, (_, i) => run({ diff --git a/apps/web/src/insights-stats.ts b/apps/web/src/insights-stats.ts index 71ce56340..761ed5948 100644 --- a/apps/web/src/insights-stats.ts +++ b/apps/web/src/insights-stats.ts @@ -3,6 +3,10 @@ // existing endpoints. import { isWorkbenchHostDefinitionName } from "@corbits/chat/workbench-host-naming"; +import { + runOutcomeStatus, + withListingAbandoned, +} from "@corbits/routines/client"; import type { InsightsRun, RunTraceSpan } from "./insights-api"; import type { Routine } from "./routines-api"; @@ -158,6 +162,7 @@ export function computeInsightsStats( runs: readonly InsightsRun[], routines: readonly Routine[], recentLimit: number = INSIGHTS_RECENT_LIMIT, + now: number = Date.now(), ): InsightsStats { const purposeful = purposeRunsForInsights(runs); let running = 0; @@ -165,7 +170,9 @@ export function computeInsightsStats( let stopped = 0; let deployed = 0; for (const run of purposeful) { - switch (run.status) { + const outcome = + runOutcomeStatus(withListingAbandoned(run, now), now) ?? run.status; + switch (outcome) { case "running": case "updating": running += 1; @@ -174,6 +181,7 @@ export function computeInsightsStats( errored += 1; break; case "stopped": + case "completed": stopped += 1; break; case "deployed": diff --git a/apps/web/src/pages/insights-page-render.test.tsx b/apps/web/src/pages/insights-page-render.test.tsx index 4be3a9fa3..ab33245e6 100644 --- a/apps/web/src/pages/insights-page-render.test.tsx +++ b/apps/web/src/pages/insights-page-render.test.tsx @@ -138,7 +138,8 @@ describe("InsightsPage 'Running now' strip", () => { // Liveness is not a windowed property: a run that started long before the // 7-day window and is still running must not disappear from the strip or // read 0 in the "Running now" KPI just because its start time falls - // outside `range`. + // outside `range`. Persist has not settled (`endedAt` absent), so the + // fire is live — not remapped to completed by the abandoned-fire window. test("a run started 8 days ago that is still running stays in the strip and the KPI", () => { const eightDaysAgo = new Date( Date.now() - 8 * 24 * 60 * 60 * 1000, diff --git a/apps/web/src/pages/insights-page.test.ts b/apps/web/src/pages/insights-page.test.ts index b0927ea12..f0fb0168a 100644 --- a/apps/web/src/pages/insights-page.test.ts +++ b/apps/web/src/pages/insights-page.test.ts @@ -9,10 +9,12 @@ // so this landing scope never takes a workbench override anymore. import { describe, expect, test } from "bun:test"; import type { DayActivity } from "@corbits/insights/client"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; import { costPerDay, elapsedLabel, + isRunningNow, resolveInsightsScope, runsPerDay, } from "./insights-page"; @@ -178,3 +180,72 @@ describe("elapsedLabel", () => { expect(elapsedLabel("not-a-date", Date.now())).toBe("—"); }); }); + +describe("isRunningNow", () => { + test("a running run still inside the fire window is in flight", () => { + expect( + isRunningNow( + run({ + id: "fresh", + createdAt: new Date().toISOString(), + status: "running", + }), + ), + ).toBe(true); + }); + + test("updating is in flight", () => { + expect( + isRunningNow( + run({ + id: "updating", + createdAt: new Date().toISOString(), + status: "updating", + }), + ), + ).toBe(true); + }); + + test("a stopped run is not in flight", () => { + expect( + isRunningNow( + run({ + id: "done", + createdAt: new Date().toISOString(), + status: "stopped", + }), + ), + ).toBe(false); + }); + + // Warm-keep (CL-6681 / CL-6778): a routine's delivery agent stays deployed + // after it replies, so workflow_run.status lingers on `running`. Past the + // fire window that is not an in-flight job — Insights must not keep it in + // "Running now" forever. + test("endedAt drops in-flight immediately, even while status is still running inside the window", () => { + expect( + isRunningNow( + run({ + id: "just-finished", + createdAt: new Date().toISOString(), + status: "running", + endedAt: new Date().toISOString(), + }), + ), + ).toBe(false); + }); + + test("a live running run past the fire window is still in flight", () => { + expect( + isRunningNow( + run({ + id: "stale", + createdAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + status: "running", + }), + ), + ).toBe(true); + }); +}); diff --git a/apps/web/src/pages/insights-page.tsx b/apps/web/src/pages/insights-page.tsx index b646ea638..6bc61583f 100644 --- a/apps/web/src/pages/insights-page.tsx +++ b/apps/web/src/pages/insights-page.tsx @@ -13,7 +13,6 @@ import { PageShell, RichEmptyState, RUN_STATUS_DOT_TONE, - RUN_STATUS_LABEL, RUN_STATUS_TONE, Skeleton, StatGrid, @@ -33,6 +32,11 @@ import { type TraceSpan, } from "@corbits/react-ui"; import { ChartBar } from "@corbits/icons"; +import { + runOutcomeStatus, + runStatusLabel, + withListingAbandoned, +} from "@corbits/routines/client"; import type * as React from "react"; import { useEffect, useMemo, useState } from "react"; @@ -54,7 +58,7 @@ import { type OverallUsage, } from "@corbits/insights/client"; -import type { WorkflowRunStatus } from "@intx/types"; +import { workflowRunStatuses, type WorkflowRunStatus } from "@intx/types"; import { SignedOutNotice, type APIQuery } from "@corbits/api-query"; import { workbenchesQueryKey, @@ -145,6 +149,18 @@ export function statusTone(status: WorkflowRunStatus): BadgeTone { return RUN_STATUS_TONE[WORKFLOW_RUN_STATUS_ALIAS[status]]; } +function isWorkflowRunStatus(status: string): status is WorkflowRunStatus { + return workflowRunStatuses.some((value) => value === status); +} + +function insightsStatusTone(status: string): BadgeTone { + if (status === "completed") return RUN_STATUS_TONE.completed; + if (status === "failed") return RUN_STATUS_TONE.failed; + if (status === "cancelled") return RUN_STATUS_TONE.stopped; + if (isWorkflowRunStatus(status)) return statusTone(status); + return "neutral"; +} + function tileValue(value: string | number | null, loading: boolean): string { if (loading) return ""; if (value === null) return "—"; @@ -455,9 +471,21 @@ function useTickingNow(enabled: boolean): number { /** A run actually in flight right now (`status: running | updating`) — * liveness is not a windowed property, so this filters the full run set, - * never the range-filtered one. */ -function isRunningNow(run: InsightsRun): boolean { - return run.status === "running" || run.status === "updating"; + * never the range-filtered one. A persisted `endedAt` means the fire + * already finished, even if `status` still reads `running`. A live fire + * with an in-flight turn stays in flight however old it is; without an + * explicit no-in-flight signal, missing `turns` is not abandonment. + */ +export function isRunningNow( + run: InsightsRun, + now: number = Date.now(), +): boolean { + const outcome = runOutcomeStatus(withListingAbandoned(run, now), now); + return outcome === "running" || outcome === "updating"; +} + +function insightsRunStatus(run: InsightsRun, now: number = Date.now()): string { + return runOutcomeStatus(withListingAbandoned(run, now), now) ?? run.status; } /** @@ -475,8 +503,11 @@ function RunningNowStrip({ readonly runs: readonly InsightsRun[]; readonly onOpenRun: (id: string) => void; }) { - const running = runs.filter(isRunningNow); - const now = useTickingNow(running.length > 0); + const maybeLive = runs.some( + (run) => run.status === "running" || run.status === "updating", + ); + const now = useTickingNow(maybeLive); + const running = runs.filter((run) => isRunningNow(run, now)); if (running.length === 0) return null; return ( @@ -496,7 +527,7 @@ function RunningNowStrip({ onClick={() => onOpenRun(run.id)} > @@ -507,7 +538,7 @@ function RunningNowStrip({ {elapsedLabel(run.createdAt, now)} - {RUN_STATUS_LABEL.running} + {runStatusLabel("running")} @@ -612,7 +643,9 @@ function RecentRunRows({ - {row.status} + + {runStatusLabel(insightsRunStatus(row))} + ))} @@ -680,7 +713,9 @@ function InsightsLanding({ // `range.from` and is still going is running right now regardless of when // it started, so "Running now" reads off every fetched run, never the // range-filtered subset above. - const runningNow = purposeRunsForInsights(runs).filter(isRunningNow); + const runningNow = purposeRunsForInsights(runs).filter((run) => + isRunningNow(run), + ); // Absent usage → zeros at the client boundary (never demo peaks / em-dash // for "no spend"). Real fetched summary is preserved when present. @@ -999,7 +1034,9 @@ function DefinitionRunTable({ {...onRowActivate(() => onOpenRun(row.id))} > - {row.status} + + {runStatusLabel(insightsRunStatus(row))} + {formatWhen(row.createdAt)} {runDurationLabel(row)} diff --git a/apps/web/src/pages/mission-control-page.tsx b/apps/web/src/pages/mission-control-page.tsx index 5c7909040..ef4a6f4c8 100644 --- a/apps/web/src/pages/mission-control-page.tsx +++ b/apps/web/src/pages/mission-control-page.tsx @@ -32,6 +32,11 @@ import { useQueryClient } from "@tanstack/react-query"; import { formatUsd } from "@corbits/insights/client"; import { CHAT_STRINGS, type Workbench } from "@corbits/chat-ui"; +import { + runOutcomeStatus, + runStatusLabel, + withListingAbandoned, +} from "@corbits/routines/client"; import { approveApproval, rejectApproval, useAPIQuery } from "../api"; import { useBench } from "../bench-context"; @@ -63,13 +68,33 @@ type InFlightRow = { readonly steps: string; }; -function routineInFlightRow(routine: RoutineActivityItem): InFlightRow { +function listingFromRoutine(routine: RoutineActivityItem, now: number) { + return withListingAbandoned( + { + createdAt: routine.startedAt, + status: routine.status, + ...(routine.endedAt !== undefined ? { endedAt: routine.endedAt } : {}), + ...(routine.hasInFlightTurn !== undefined + ? { hasInFlightTurn: routine.hasInFlightTurn } + : {}), + ...(routine.turns !== undefined ? { turns: routine.turns } : {}), + }, + now, + ); +} + +function routineInFlightRow( + routine: RoutineActivityItem, + now: number, +): InFlightRow { + const status = + runOutcomeStatus(listingFromRoutine(routine, now), now) ?? routine.status; return { key: `routine:${routine.id}`, label: routine.name, context: "routine", createdAt: routine.startedAt, - statusLabel: "Running", + statusLabel: runStatusLabel(status), statusTone: RUN_STATUS_TONE.running, // The routine feed carries no step count — an honest dash, not a guess. steps: "—", @@ -79,10 +104,14 @@ function routineInFlightRow(routine: RoutineActivityItem): InFlightRow { /** Every routine this bench is actively running right now, newest first. */ export function computeInFlightRows( routines: readonly RoutineActivityItem[], + now: number = Date.now(), ): readonly InFlightRow[] { const rows = routines - .filter((routine) => routine.status === "running") - .map(routineInFlightRow); + .filter( + (routine) => + runOutcomeStatus(listingFromRoutine(routine, now), now) === "running", + ) + .map((routine) => routineInFlightRow(routine, now)); return rows.sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); } @@ -227,10 +256,7 @@ export function MissionControlRoute({ const inFlightRows = activity.kind === "ready" ? computeInFlightRows(activity.routines) : []; const activeRunsCount = - activity.kind === "ready" - ? activity.routines.filter((routine) => routine.status === "running") - .length - : null; + activity.kind === "ready" ? inFlightRows.length : null; const jumpBackRows = activity.kind === "ready" diff --git a/apps/web/src/pages/routine-detail-page.tsx b/apps/web/src/pages/routine-detail-page.tsx index 161f6e5b0..3a9ea3895 100644 --- a/apps/web/src/pages/routine-detail-page.tsx +++ b/apps/web/src/pages/routine-detail-page.tsx @@ -46,6 +46,7 @@ import { routineHealth, routineScheduleSentence, timezoneForTrigger, + withListingAbandoned, } from "@corbits/routines/client"; import type { RoutineHealth } from "@corbits/routines/client"; @@ -352,7 +353,11 @@ export function RoutineDetailPage({ readonly onSaveSchedule: (expression: string) => Promise; readonly onEdit: () => void; }) { - const health = routineHealth(row.routine, row.runs, now); + const health = routineHealth( + row.routine, + row.runs.map((run) => withListingAbandoned(run, now)), + now, + ); const latestRunId = row.runs.find((run) => !fireNeverStarted(run.triggeredBy))?.runId ?? null; return ( diff --git a/apps/web/src/pages/routines-page.tsx b/apps/web/src/pages/routines-page.tsx index 8fd0d2f60..4b6e35edf 100644 --- a/apps/web/src/pages/routines-page.tsx +++ b/apps/web/src/pages/routines-page.tsx @@ -41,6 +41,7 @@ import { routineScheduleSentence, runStatusLabel, triggeredByLabel, + withListingAbandoned, } from "@corbits/routines/client"; import type { RoutineHealth } from "@corbits/routines/client"; @@ -189,7 +190,7 @@ export function RunStatusCell({ readonly run: RoutineRun; readonly now: number; }) { - const status = fireOutcomeStatus(run, now); + const status = fireOutcomeStatus(withListingAbandoned(run, now), now); if (status === null) { return —; } @@ -203,7 +204,11 @@ export function routineRowHealth( row: GlobalRoutineRow, now: number, ): RoutineHealth { - return routineHealth(row.routine, row.runs, now); + return routineHealth( + row.routine, + row.runs.map((run) => withListingAbandoned(run, now)), + now, + ); } /** "At 09:00, Monday through Friday (UTC)" — the schedule as a sentence, diff --git a/apps/web/src/shell/routine-activity.ts b/apps/web/src/shell/routine-activity.ts index 9b7646289..518706f84 100644 --- a/apps/web/src/shell/routine-activity.ts +++ b/apps/web/src/shell/routine-activity.ts @@ -10,6 +10,12 @@ // toward Mission Control's "Active runs" — exactly CL-6595's desync // between the Routines page's own "Running now" pill and Mission Control's // "0 / nothing running". +import { + runOutcomeStatus, + withListingAbandoned, +} from "@corbits/routines/client"; +import type { ListingTurn } from "@corbits/routines/client"; + import { listRoutineRunFires } from "../agents-api"; import type { RunFire } from "../agents-api"; @@ -18,21 +24,44 @@ export type RoutineActivityItem = { readonly name: string; readonly status: string; readonly startedAt: string; + readonly endedAt?: string | null; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; }; -function toRoutineActivityItem(run: RunFire): RoutineActivityItem { +function toRoutineActivityItem(run: RunFire, now: number): RoutineActivityItem { + const listing = withListingAbandoned( + { + createdAt: run.createdAt, + status: run.status, + ...(run.endedAt !== undefined ? { endedAt: run.endedAt } : {}), + ...(run.hasInFlightTurn !== undefined + ? { hasInFlightTurn: run.hasInFlightTurn } + : {}), + ...(run.turns !== undefined ? { turns: run.turns } : {}), + }, + now, + ); return { id: run.id, name: run.routineName ?? run.definitionName, - status: run.status, + status: runOutcomeStatus(listing, now) ?? run.status, startedAt: run.createdAt, + ...(run.endedAt !== undefined ? { endedAt: run.endedAt } : {}), + ...(run.hasInFlightTurn !== undefined + ? { hasInFlightTurn: run.hasInFlightTurn } + : {}), + ...(run.turns !== undefined ? { turns: run.turns } : {}), }; } export function listRoutineActivity( tenantId: string, + now: number = Date.now(), ): Promise { return listRoutineRunFires(tenantId).then((runs) => - runs.filter((run) => run.routineId !== null).map(toRoutineActivityItem), + runs + .filter((run) => run.routineId !== null) + .map((run) => toRoutineActivityItem(run, now)), ); } diff --git a/apps/web/test/insights-page.test.tsx b/apps/web/test/insights-page.test.tsx index 02a205194..ed59876ff 100644 --- a/apps/web/test/insights-page.test.tsx +++ b/apps/web/test/insights-page.test.tsx @@ -563,7 +563,7 @@ describe("InsightsRunsHistory definition grouping", () => { expect(groups[0]?.getAttribute("data-definition-group")).toBe("wfd_a"); const firstGroupRows = groups[0]?.querySelectorAll("tbody tr") ?? []; expect(firstGroupRows.length).toBe(2); - expect(firstGroupRows[0]?.textContent).toContain("error"); + expect(firstGroupRows[0]?.textContent).toContain("Failed"); expect(el.textContent).not.toContain("Showing the 100 most recent runs."); }); diff --git a/apps/web/test/mission-control-page.test.tsx b/apps/web/test/mission-control-page.test.tsx index 8b95c66e1..3384e2fb6 100644 --- a/apps/web/test/mission-control-page.test.tsx +++ b/apps/web/test/mission-control-page.test.tsx @@ -4,6 +4,7 @@ import { createRoot } from "react-dom/client"; import type { Root } from "react-dom/client"; import type { Workbench } from "@corbits/chat-ui"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; import { BenchContext, type BenchState } from "../src/bench-context"; import { NavigationProvider } from "../src/navigation"; @@ -26,7 +27,7 @@ function routine(overrides: Partial): RoutineActivityItem { id: "rtn_1", name: "Weekly digest", status: "running", - startedAt: "2026-08-19T09:00:00.000Z", + startedAt: new Date().toISOString(), ...overrides, }; } @@ -57,16 +58,56 @@ describe("computeInFlightRows", () => { routine({ id: "old", status: "running", - startedAt: "2026-08-19T08:00:00.000Z", + startedAt: new Date(Date.now() - 60_000).toISOString(), }), routine({ id: "new", status: "running", - startedAt: "2026-08-19T11:00:00.000Z", + startedAt: new Date().toISOString(), }), ]); expect(rows.map((row) => row.key)).toEqual(["routine:new", "routine:old"]); }); + + test("a running routine still inside the fire window stays in-flight", () => { + const rows = computeInFlightRows([ + routine({ + id: "fresh", + status: "running", + startedAt: new Date().toISOString(), + }), + ]); + expect(rows.map((row) => row.key)).toEqual(["routine:fresh"]); + expect(rows[0]?.statusLabel).toBe("Running now"); + }); + + // Warm-keep (CL-6681 / CL-6778): Mission Control's active-run count and + // in-flight table must not keep a finished fire as "Running" forever just + // because the delivery agent is still deployed. + test("endedAt drops a just-finished running routine from in-flight immediately", () => { + const rows = computeInFlightRows([ + routine({ + id: "just-finished", + status: "running", + startedAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + }), + ]); + expect(rows).toEqual([]); + }); + + test("a live running routine past the fire window stays in-flight", () => { + const rows = computeInFlightRows([ + routine({ + id: "stale", + status: "running", + startedAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + }), + ]); + expect(rows.map((row) => row.key)).toEqual(["routine:stale"]); + }); }); describe("computeJumpBackRows", () => { diff --git a/apps/web/test/routine-activity.test.ts b/apps/web/test/routine-activity.test.ts index 86a0cc293..eee605d1a 100644 --- a/apps/web/test/routine-activity.test.ts +++ b/apps/web/test/routine-activity.test.ts @@ -9,6 +9,8 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; + import { listRoutineActivity } from "../src/shell/routine-activity"; const realFetch = globalThis.fetch; @@ -38,8 +40,8 @@ const runningFire = { tenantId: "tnt_1", address: "run_1@tnt1.example", status: "running", - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), routineId: "rtn_1", routineName: "Weekly digest", }; @@ -81,7 +83,7 @@ describe("listRoutineActivity", () => { id: "run_1", name: "Weekly digest", status: "running", - startedAt: "2026-01-01T00:00:00.000Z", + startedAt: runningFire.createdAt, }, ]); }); @@ -98,6 +100,36 @@ describe("listRoutineActivity", () => { expect(item?.status).not.toBe("running"); }); + test("endedAt drops a just-finished running fire from running immediately", async () => { + stubTopLevelRunsFetch([ + { + ...runningFire, + endedAt: new Date().toISOString(), + }, + ]); + + const [item] = await listRoutineActivity("tnt_1"); + + expect(item?.status).not.toBe("running"); + }); + + // A live fire still in a tool loop can outlast the abandoned-fire + // window. Persist has not settled, so the row stays running. + test("a live running fire past the window still reads as running", async () => { + stubTopLevelRunsFetch([ + { + ...runningFire, + createdAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + }, + ]); + + const [item] = await listRoutineActivity("tnt_1"); + + expect(item?.status).toBe("running"); + }); + // A directly-triggered deployment run is also a `feed=fires` row (it is // not a folded run at all), but it has no routine parent -- it must not // be counted as routine activity. diff --git a/apps/web/test/routines-page.test.tsx b/apps/web/test/routines-page.test.tsx index 62e306bff..989acb70f 100644 --- a/apps/web/test/routines-page.test.tsx +++ b/apps/web/test/routines-page.test.tsx @@ -223,7 +223,7 @@ describe("GlobalRoutinesList", () => { expect(markup).not.toContain(">running<"); }); - test("warm-keep (CL-6681): a fire whose 'running' status is stale reads as finished, not stuck Running now forever", () => { + test("a live fire whose 'running' status is older than the window still reads as Running now", () => { const markup = renderList([ row({ runs: [ @@ -236,8 +236,8 @@ describe("GlobalRoutinesList", () => { ], }), ]); - expect(markup).not.toContain("Running now"); - expect(markup).toContain("Finished"); + expect(markup).toContain("Running now"); + expect(markup).not.toContain("Finished"); }); test("a failing routine states its failure count in words, not only in colour", () => { diff --git a/packages/routines/src/client.ts b/packages/routines/src/client.ts index b3534c098..20d7071c7 100644 --- a/packages/routines/src/client.ts +++ b/packages/routines/src/client.ts @@ -29,10 +29,16 @@ export { fireFailed, fireOutcomeStatus, lastFailedFire, + listingAbandoned, + listingHasInFlightTurn, medianFireDurationMs, routineHealth, + runOutcomeStatus, + withListingAbandoned, } from "./health"; export type { + ListingRun, + ListingTurn, RoutineFire, RoutineHealth, RoutineHealthState, diff --git a/packages/routines/src/health.test.ts b/packages/routines/src/health.test.ts index 48c518448..3fdf9bff3 100644 --- a/packages/routines/src/health.test.ts +++ b/packages/routines/src/health.test.ts @@ -5,8 +5,11 @@ import { FIRE_RUNNING_WINDOW_MS, fireOutcomeStatus, lastFailedFire, + listingAbandoned, + listingHasInFlightTurn, medianFireDurationMs, routineHealth, + runOutcomeStatus, } from "./health"; import type { RoutineFire, RoutineHealthSubject } from "./health"; @@ -40,13 +43,40 @@ describe("fireOutcomeStatus", () => { ); }); - test("warm-keep (CL-6681): a running fire past its window reads as completed", () => { + test("a live fire past the window still reads as running until persist settles", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; expect( fireOutcomeStatus(fire("r1", {}, { status: "running" }), staleNow), + ).toBe("running"); + }); + + test("an 8-day still-running live fire stays running", () => { + const eightDays = Date.parse(FIRE_CREATED_AT) + 8 * 24 * 60 * 60 * 1000; + expect( + fireOutcomeStatus(fire("r1", {}, { status: "running" }), eightDays), + ).toBe("running"); + }); + + test("abandoned: a running fire past its window reads as completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus( + fire("r1", { abandoned: true }, { status: "running" }), + staleNow, + ), ).toBe("completed"); }); + test("abandoned: a running fire one millisecond inside its window still reads as running", () => { + const stillInside = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS; + expect( + fireOutcomeStatus( + fire("r1", { abandoned: true }, { status: "running" }), + stillInside, + ), + ).toBe("running"); + }); + test("every terminal status passes through unchanged, however old", () => { const longAfter = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS * 10; expect( @@ -60,6 +90,227 @@ describe("fireOutcomeStatus", () => { test("null when the platform has no run to report", () => { expect(fireOutcomeStatus(fire("r1", {}, {}), NOW)).toBeNull(); }); + + test("endedAt on a still-running column is finished immediately, not at T+window", () => { + expect( + fireOutcomeStatus( + fire( + "r1", + {}, + { status: "running", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + NOW, + ), + ).toBe("completed"); + }); + + test("a persisted failed fire stays failed past the window — never remapped to completed", () => { + const longAfter = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS * 10; + expect( + fireOutcomeStatus( + fire( + "r1", + {}, + { status: "failed", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + longAfter, + ), + ).toBe("failed"); + }); + + test("listing-shaped empty turns past the window settles to completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus( + fire("r1", { turns: [] }, { status: "running" }), + staleNow, + ), + ).toBe("completed"); + }); + + test("listing-shaped hasInFlightTurn true past the window stays running", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus( + fire("r1", { hasInFlightTurn: true }, { status: "running" }), + staleNow, + ), + ).toBe("running"); + }); +}); + +describe("runOutcomeStatus", () => { + test("a top-level running status still inside its window reads as running", () => { + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "running" }, NOW), + ).toBe("running"); + }); + + test("a live top-level running status past the window still reads as running", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running" }, + staleNow, + ), + ).toBe("running"); + }); + + test("abandoned: a top-level running status past its window reads as completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running", abandoned: true }, + staleNow, + ), + ).toBe("completed"); + }); + + test("endedAt drops a top-level running status immediately", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + endedAt: "2026-01-01T00:00:30.000Z", + }, + NOW, + ), + ).toBe("completed"); + }); + + test("updating and terminal platform statuses pass through", () => { + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "updating" }, NOW), + ).toBe("updating"); + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "error" }, NOW), + ).toBe("error"); + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "stopped" }, NOW), + ).toBe("stopped"); + }); +}); + +describe("listingAbandoned", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const listing = { createdAt: FIRE_CREATED_AT }; + + test("omitted turns and hasInFlightTurn is not abandoned past the window", () => { + expect(listingAbandoned(listing, staleNow)).toBe(false); + expect(listingHasInFlightTurn(listing)).toBe(false); + }); + + test("empty turns after a real query past the window is abandoned", () => { + expect(listingAbandoned({ ...listing, turns: [] }, staleNow)).toBe(true); + }); + + test("hasInFlightTurn false past the window is abandoned", () => { + expect( + listingAbandoned({ ...listing, hasInFlightTurn: false }, staleNow), + ).toBe(true); + }); + + test("hasInFlightTurn true past the window is not abandoned", () => { + expect( + listingAbandoned({ ...listing, hasInFlightTurn: true }, staleNow), + ).toBe(false); + expect(listingHasInFlightTurn({ ...listing, hasInFlightTurn: true })).toBe( + true, + ); + }); + + test("a running turn past the window is not abandoned", () => { + expect( + listingAbandoned( + { ...listing, turns: [{ status: "running", endedAt: null }] }, + staleNow, + ), + ).toBe(false); + }); + + test("finished turns only past the window is abandoned", () => { + expect( + listingAbandoned( + { + ...listing, + turns: [{ status: "completed", endedAt: "2026-01-01T00:01:00.000Z" }], + }, + staleNow, + ), + ).toBe(true); + }); + + test("empty nested run.turns past the window is abandoned", () => { + expect(listingAbandoned({ ...listing, run: { turns: [] } }, staleNow)).toBe( + true, + ); + }); + + test("empty turns inside the window is not abandoned", () => { + expect(listingAbandoned({ ...listing, turns: [] }, NOW)).toBe(false); + }); +}); + +describe("runOutcomeStatus listing-shaped", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + + test("missing turns past the window stays running", () => { + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running" }, + staleNow, + ), + ).toBe("running"); + }); + + test("empty turns past the window settles to completed", () => { + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running", turns: [] }, + staleNow, + ), + ).toBe("completed"); + }); + + test("hasInFlightTurn false past the window settles to completed", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + hasInFlightTurn: false, + }, + staleNow, + ), + ).toBe("completed"); + }); + + test("hasInFlightTurn true past the window stays running", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + hasInFlightTurn: true, + }, + staleNow, + ), + ).toBe("running"); + }); + + test("a running listing turn past the window stays running", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + turns: [{ status: "running", endedAt: null }], + }, + staleNow, + ), + ).toBe("running"); + }); }); describe("cleanFireStreak", () => { @@ -83,13 +334,23 @@ describe("cleanFireStreak", () => { ).toBe(1); }); - test("a stale running fire (warm-keep) counts as a success, not in-flight forever", () => { + test("a live running fire past the window stays in-flight and does not extend the streak", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; expect( cleanFireStreak( [fire("r2", {}, { status: "running" }), fire("r1")], staleNow, ), + ).toBe(1); + }); + + test("an abandoned stale running fire counts as a success, not in-flight forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + cleanFireStreak( + [fire("r2", { abandoned: true }, { status: "running" }), fire("r1")], + staleNow, + ), ).toBe(2); }); @@ -187,17 +448,61 @@ describe("routineHealth", () => { ).toBe("running"); }); - test("warm-keep (CL-6681): a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => { + test("endedAt on the latest fire is not Running now, even inside the window", () => { + expect( + routineHealth( + healthy, + [ + fire( + "r1", + {}, + { status: "running", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + ], + NOW, + ).state, + ).not.toBe("running"); + }); + + test("a live latest run past the window still reports Running now", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; const health = routineHealth( healthy, [fire("r1", {}, { status: "running" })], staleNow, ); + expect(health.state).toBe("running"); + expect(health.label).toBe("Running now"); + }); + + test("abandoned: a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const health = routineHealth( + healthy, + [fire("r1", { abandoned: true }, { status: "running" })], + staleNow, + ); expect(health.state).toBe("ok"); expect(health.label).not.toBe("Running now"); }); + test("a persisted failed latest fire is Last run failed, even past the window", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const health = routineHealth( + healthy, + [ + fire( + "r1", + {}, + { status: "failed", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + ], + staleNow, + ); + expect(health.state).toBe("failing"); + expect(health.label).toBe("Last run failed"); + }); + test("consecutive failures are stated in the caption, not just the pill", () => { const health = routineHealth({ ...healthy, consecutiveFailures: 2 }, [ fire("r1", { error: "boom" }, {}), diff --git a/packages/routines/src/health.ts b/packages/routines/src/health.ts index 0fa8691a1..ec0e53ae8 100644 --- a/packages/routines/src/health.ts +++ b/packages/routines/src/health.ts @@ -9,6 +9,12 @@ // routine's condition have to agree, and they can only agree if there is // one function that decides. +/** One turn on a listing-shaped payload — enough to tell in-flight from settled. */ +export type ListingTurn = { + readonly status: string; + readonly endedAt?: string | null; +}; + /** The per-fire history row this module reads — structurally the subset of * `./client`'s `RoutineRun` that says whether a fire worked and how long * it took. Declared here rather than imported so `./client` can re-export @@ -19,6 +25,31 @@ export type RoutineFire = { readonly createdAt: string; readonly error?: string | null; readonly run?: Record; + /** + * True when this fire is already known abandoned (sidecar dead, no + * in-flight turn). The running-window is only applied then — a live + * fire still doing work stays `running` however old it is. + */ + readonly abandoned?: boolean; + /** + * True when the listing already resolved an in-flight turn for this + * fire. Equivalent to `turns` containing a running turn; either form + * keeps a live tool-loop `running` past `FIRE_RUNNING_WINDOW_MS`. + */ + readonly hasInFlightTurn?: boolean; + /** Listing-shaped turns for this fire, when the producer attached them. */ + readonly turns?: readonly ListingTurn[]; +}; + +/** A listing row (Insights fire, shell activity, reconstructed Mission + * Control row, or a routine fire) from which abandonment can be derived + * without a dedicated Interchange field. */ +export type ListingRun = { + readonly createdAt: string; + readonly abandoned?: boolean; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; + readonly run?: Record; }; /** The routine fields health depends on — every one of them already @@ -65,42 +96,172 @@ export type RoutineHealth = { readonly medianDurationMs: number | null; }; -function statusOf(fire: RoutineFire): string | null { +function statusOf(fire: { + readonly run?: Record; +}): string | null { const status = fire.run?.status; return typeof status === "string" ? status : null; } +function isInFlightTurn(turn: unknown): boolean { + if (typeof turn !== "object" || turn === null) return false; + const status = "status" in turn ? turn.status : undefined; + if (status !== "running") return false; + const endedAt = "endedAt" in turn ? turn.endedAt : undefined; + return endedAt === undefined || endedAt === null || endedAt === ""; +} + +function nestedTurns( + run: Record | undefined, +): readonly unknown[] | undefined { + const turns = run?.turns; + return Array.isArray(turns) ? turns : undefined; +} + /** - * How long a fire may credibly still be doing work before its lingering - * `running` status is read as warm-keep (CL-6681) — a routine's delivery - * agent deliberately stays deployed after it replies, so - * `workflow_run.status` never settles out of `running` on its own. Past - * this window a `running` fire is presumed to have already delivered its - * reply, so every surface badging its status reads it through - * `fireOutcomeStatus` rather than the raw column. + * How long an *abandoned* fire may linger as `running` with no `endedAt` + * before it is read as completed (warm-keep CL-6681 / CL-6778). A finished + * fire is supposed to land `completed`/`failed`/`cancelled` plus `endedAt` + * via `markTerminal`; this window is last-resort for a fire already known + * abandoned that never got that write. It is never applied to a live + * in-flight fire — a tool loop can outlast ten minutes, and persist has + * not settled yet. */ export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; +type InFlightSignal = "yes" | "no" | "unknown"; + +function runHasInFlightTurn(run: Record | undefined): boolean { + return run?.hasInFlightTurn === true; +} + +function runHasNoInFlightTurn( + run: Record | undefined, +): boolean { + return run?.hasInFlightTurn === false; +} + /** - * A fire's status the way this build should show it: the raw `run.status` - * for every terminal value, but a `running` status older than - * `FIRE_RUNNING_WINDOW_MS` is read as `completed` instead of taken - * literally — see that constant's own comment. This is the one place that - * tells a fire still doing work apart from one merely staying warm; every - * caller that needs a fire's displayed status goes through here, never - * `fire.run?.status` directly. + * Absent `turns` / `hasInFlightTurn` is unknown, not "no in-flight turn". + * Treating omit as empty reverts the live-tool-loop 10-minute false-complete. + */ +function listingInFlightSignal(listing: ListingRun): InFlightSignal { + if (listing.hasInFlightTurn === true || runHasInFlightTurn(listing.run)) { + return "yes"; + } + if (listing.turns?.some(isInFlightTurn) === true) return "yes"; + const nested = nestedTurns(listing.run); + if (nested?.some(isInFlightTurn) === true) return "yes"; + if (listing.turns !== undefined) return "no"; + if (nested !== undefined) return "no"; + if (listing.hasInFlightTurn === false || runHasNoInFlightTurn(listing.run)) { + return "no"; + } + return "unknown"; +} + +/** + * A listing row has an in-flight turn when the producer said so + * (`hasInFlightTurn`) or attached a running turn (top-level `turns` or + * nested on `run`). A finished turn (`endedAt` set, or status other than + * `running`) does not count. Omitted fields are unknown, not false. + */ +export function listingHasInFlightTurn(listing: ListingRun): boolean { + return listingInFlightSignal(listing) === "yes"; +} + +/** + * Abandoned only when the producer explicitly said there is no in-flight + * turn — an empty `turns` array after a real query, or `hasInFlightTurn: + * false` — and the fire is older than `FIRE_RUNNING_WINDOW_MS`. Omitting + * those fields is not a no; a live tool-loop listing that has not attached + * them must stay running. + */ +export function listingAbandoned(listing: ListingRun, now: number): boolean { + if (listingInFlightSignal(listing) !== "no") return false; + const startedAt = Date.parse(listing.createdAt); + if (Number.isNaN(startedAt)) return false; + return now - startedAt > FIRE_RUNNING_WINDOW_MS; +} + +/** Pass `abandoned: true` into outcome helpers when the listing is abandoned. */ +export function withListingAbandoned( + listing: T, + now: number, +): T { + if (!listingAbandoned(listing, now)) return listing; + return { ...listing, abandoned: true }; +} + +/** + * A fire's status the way this build should show it: the raw + * `run.status` for every non-running value; a `running` row that + * already carries `endedAt` is finished immediately (the persist + * path); a `running` row with an in-flight turn stays `running` + * however old it is; a `running` row whose listing explicitly says + * there is no in-flight turn is abandoned past `FIRE_RUNNING_WINDOW_MS` + * and remapped to `completed`. Omitting `turns`/`hasInFlightTurn` is + * not that signal — the fire stays `running`. + * Every caller that needs a fire's displayed status goes through here, + * never `fire.run?.status` directly. */ export function fireOutcomeStatus( - fire: RoutineFire, + fire: Pick & { + readonly run?: Record; + readonly abandoned?: boolean; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; + }, now: number, ): string | null { const status = statusOf(fire); if (status !== "running") return status; + const endedAt = fire.run?.endedAt; + if (typeof endedAt === "string" && endedAt !== "") return "completed"; + if (listingHasInFlightTurn(fire)) return status; + const abandoned = fire.abandoned === true || listingAbandoned(fire, now); + if (!abandoned) return status; const startedAt = Date.parse(fire.createdAt); if (Number.isNaN(startedAt)) return status; return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running"; } +/** + * `fireOutcomeStatus` for a platform run whose status lives at the top + * level (`workflow_run.status`), not nested on a routine fire's `run`. + * Insights, Mission Control, and the shell activity feed all see that + * shape. `endedAt` is the persist-path signal that the fire already + * finished. A listing-shaped payload with an in-flight turn stays + * `running` however old it is; one that explicitly says there is no + * in-flight turn is abandoned past `FIRE_RUNNING_WINDOW_MS` and + * remapped to `completed`. Omitting those fields is not a no. + */ +export function runOutcomeStatus( + run: { + readonly createdAt: string; + readonly status: string; + readonly endedAt?: string | null; + readonly abandoned?: boolean; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; + }, + now: number, +): string | null { + const listing = withListingAbandoned(run, now); + return fireOutcomeStatus( + { + createdAt: listing.createdAt, + ...(listing.abandoned === true ? { abandoned: true } : {}), + ...(listing.hasInFlightTurn !== undefined + ? { hasInFlightTurn: listing.hasInFlightTurn } + : {}), + ...(listing.turns !== undefined ? { turns: listing.turns } : {}), + run: { status: listing.status, endedAt: listing.endedAt }, + }, + now, + ); +} + /** A fire failed when it recorded a launch error (the synthetic * `schedule-failed` row) or its run settled as failed. */ export function fireFailed(fire: RoutineFire): boolean { diff --git a/packages/routines/src/index.ts b/packages/routines/src/index.ts index 0d0ab56ef..565009004 100644 --- a/packages/routines/src/index.ts +++ b/packages/routines/src/index.ts @@ -69,6 +69,13 @@ export type { RunSummaryResolver, } from "./routes"; +export { settleRoutineFire, settleRoutineFireFromTurn } from "./settle-fire"; +export type { + RoutineFireSettlePort, + RoutineFireSettleStatus, + RoutineFireTurnSettlePort, +} from "./settle-fire"; + export { createWorkflowRoutineRoutes } from "./workflow-routine-routes"; export type { CreateWorkflowRoutineRoutesDeps, diff --git a/packages/routines/src/run-language.test.ts b/packages/routines/src/run-language.test.ts index 1c70c0dea..aeeb372ff 100644 --- a/packages/routines/src/run-language.test.ts +++ b/packages/routines/src/run-language.test.ts @@ -9,8 +9,10 @@ import { describe("runStatusLabel", () => { test("every status a routine surface can show reads as words", () => { expect(runStatusLabel("running")).toBe("Running now"); + expect(runStatusLabel("updating")).toBe("Running now"); expect(runStatusLabel("completed")).toBe("Finished"); expect(runStatusLabel("failed")).toBe("Failed"); + expect(runStatusLabel("error")).toBe("Failed"); expect(runStatusLabel("cancelled")).toBe("Cancelled"); expect(runStatusLabel("queued")).toBe("Waiting to start"); }); diff --git a/packages/routines/src/run-language.ts b/packages/routines/src/run-language.ts index 9157b773f..97e7d4851 100644 --- a/packages/routines/src/run-language.ts +++ b/packages/routines/src/run-language.ts @@ -11,8 +11,10 @@ const RUN_STATUS_WORDS: Readonly> = { running: "Running now", + updating: "Running now", completed: "Finished", failed: "Failed", + error: "Failed", cancelled: "Cancelled", queued: "Waiting to start", pending: "Waiting to start", diff --git a/packages/routines/src/settle-fire.test.ts b/packages/routines/src/settle-fire.test.ts new file mode 100644 index 000000000..894509e8d --- /dev/null +++ b/packages/routines/src/settle-fire.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, test } from "bun:test"; + +import { settleRoutineFire, settleRoutineFireFromTurn } from "./settle-fire"; + +describe("settleRoutineFire", () => { + test("persists completed + endedAt via markTerminal after a finished fire", async () => { + const endedAt = new Date("2026-01-01T00:01:00.000Z"); + const calls: { + runId: string; + status: string; + endedAt: Date; + }[] = []; + + const won = await settleRoutineFire( + { + isRoutineFire: async () => true, + markTerminal: async (runId, status, at) => { + calls.push({ runId, status, endedAt: at }); + return { id: runId, status, endedAt: at }; + }, + }, + { runId: "run_fire1", status: "completed", endedAt }, + ); + + expect(won).toBe(true); + expect(calls).toEqual([ + { runId: "run_fire1", status: "completed", endedAt }, + ]); + }); + + test("a failed fire persists failed, never completed", async () => { + const endedAt = new Date("2026-01-01T00:01:00.000Z"); + const calls: string[] = []; + + await settleRoutineFire( + { + isRoutineFire: async () => true, + markTerminal: async (_runId, status, _at) => { + calls.push(status); + return { status }; + }, + }, + { runId: "run_fire1", status: "failed", endedAt }, + ); + + expect(calls).toEqual(["failed"]); + }); + + test("does not markTerminal a run that is not a routine fire", async () => { + let marked = false; + + const won = await settleRoutineFire( + { + isRoutineFire: async () => false, + markTerminal: async () => { + marked = true; + return {}; + }, + }, + { runId: "run_chat1", status: "completed" }, + ); + + expect(won).toBe(false); + expect(marked).toBe(false); + }); + + test("a losing markTerminal (already terminal) is not a second persist", async () => { + const won = await settleRoutineFire( + { + isRoutineFire: async () => true, + markTerminal: async () => null, + }, + { runId: "run_fire1", status: "completed" }, + ); + + expect(won).toBe(false); + }); +}); + +describe("settleRoutineFireFromTurn", () => { + test("looks up the run by address and persists the turn's terminal status", async () => { + const endedAt = new Date("2026-01-01T00:02:00.000Z"); + const calls: { + runId: string; + status: string; + endedAt: Date; + }[] = []; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async (address) => + address === "run_fire1@acme.workbench.test" + ? { id: "run_fire1" } + : undefined, + isRoutineFire: async (runId) => runId === "run_fire1", + markTerminal: async (runId, status, at) => { + calls.push({ runId, status, endedAt: at }); + return { id: runId, status, endedAt: at }; + }, + }, + "run_fire1@acme.workbench.test", + { status: "completed", hadReply: true }, + endedAt, + ); + + expect(won).toBe(true); + expect(calls).toEqual([ + { runId: "run_fire1", status: "completed", endedAt }, + ]); + }); + + test("a failed turn persists failed", async () => { + const calls: string[] = []; + + await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async (_runId, status) => { + calls.push(status); + return { status }; + }, + }, + "run_fire1@acme.workbench.test", + { status: "failed", hadReply: false }, + new Date("2026-01-01T00:02:00.000Z"), + ); + + expect(calls).toEqual(["failed"]); + }); + + test("unknown address is a no-op", async () => { + let marked = false; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => undefined, + isRoutineFire: async () => true, + markTerminal: async () => { + marked = true; + return {}; + }, + }, + "run_missing@acme.workbench.test", + { status: "completed", hadReply: true }, + ); + + expect(won).toBe(false); + expect(marked).toBe(false); + }); + + test("skips an intermediate completed turn that had no reply", async () => { + let marked = false; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async () => { + marked = true; + return {}; + }, + }, + "run_fire1@acme.workbench.test", + { status: "completed", hadReply: false }, + ); + + expect(won).toBe(false); + expect(marked).toBe(false); + }); + + test("persists completed when the turn had a reply", async () => { + const calls: string[] = []; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async (_runId, status) => { + calls.push(status); + return { status }; + }, + }, + "run_fire1@acme.workbench.test", + { status: "completed", hadReply: true }, + ); + + expect(won).toBe(true); + expect(calls).toEqual(["completed"]); + }); + + test("persists failed after a skipped intermediate tool step", async () => { + const calls: string[] = []; + const port = { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async ( + _runId: string, + status: "completed" | "failed" | "cancelled", + ) => { + if (calls.length > 0) return null; + calls.push(status); + return { status }; + }, + }; + + const skipped = await settleRoutineFireFromTurn( + port, + "run_fire1@acme.workbench.test", + { status: "completed", hadReply: false }, + ); + const won = await settleRoutineFireFromTurn( + port, + "run_fire1@acme.workbench.test", + { status: "failed", hadReply: false }, + ); + + expect(skipped).toBe(false); + expect(won).toBe(true); + expect(calls).toEqual(["failed"]); + }); +}); diff --git a/packages/routines/src/settle-fire.ts b/packages/routines/src/settle-fire.ts new file mode 100644 index 000000000..d5b74d6e5 --- /dev/null +++ b/packages/routines/src/settle-fire.ts @@ -0,0 +1,74 @@ +// Persist a routine fire's terminal status onto `workflow_run` once the +// fire has actually finished. Warm-keep (CL-6681) leaves a folded delivery +// agent deployed, so the platform never settles `workflow_run.status` on +// its own; Insights and Mission Control then read a finished fire as +// "Running now" forever. `markTerminal` is the one write that stamps +// `completed`/`failed`/`cancelled` and `endedAt` in a single shot — the +// same store method the hub already uses for native workflow runs. +// +// Only a row in `routine_run` is a fire. A workbench host or invited +// agent is also a folded run; marking those terminal would lie about +// their lifecycle. The port's `isRoutineFire` is that gate. + +export type RoutineFireSettleStatus = "completed" | "failed" | "cancelled"; + +export type RoutineFireSettlePort = { + isRoutineFire(runId: string): Promise; + markTerminal( + runId: string, + status: RoutineFireSettleStatus, + endedAt: Date, + ): Promise; +}; + +export type RoutineFireTurnSettlePort = RoutineFireSettlePort & { + lookupRunByAddress(address: string): Promise<{ id: string } | undefined>; +}; + +/** + * Stamp a routine fire terminal. Returns whether this caller won the + * flip (`markTerminal` is single-shot: a later call against an already + * terminal row returns null). A run that is not a routine fire is a + * no-op, never a persist. + */ +export async function settleRoutineFire( + port: RoutineFireSettlePort, + input: { + runId: string; + status: RoutineFireSettleStatus; + endedAt?: Date; + }, +): Promise { + if (!(await port.isRoutineFire(input.runId))) return false; + const won = await port.markTerminal( + input.runId, + input.status, + input.endedAt ?? new Date(), + ); + return won !== null; +} + +/** + * `settleRoutineFire` from a finalized turn: look the run up by the + * turn's agent address, then persist the turn's own terminal status. + * + * Intermediate tool-use steps finalize as `completed` with `hadReply: + * false`. Those must not stamp the fire — `markTerminal` is single-shot, + * so an early completed would hide a later failure. A real reply + * (`hadReply: true`) or a failed turn settles immediately. + */ +export async function settleRoutineFireFromTurn( + port: RoutineFireTurnSettlePort, + address: string, + turn: { readonly status: "completed" | "failed"; readonly hadReply: boolean }, + endedAt: Date = new Date(), +): Promise { + if (turn.status === "completed" && !turn.hadReply) return false; + const run = await port.lookupRunByAddress(address); + if (run === undefined) return false; + return settleRoutineFire(port, { + runId: run.id, + status: turn.status, + endedAt, + }); +} diff --git a/packages/routines/test/routes.test.ts b/packages/routines/test/routes.test.ts index ee5822747..6e8d9c13d 100644 --- a/packages/routines/test/routes.test.ts +++ b/packages/routines/test/routes.test.ts @@ -710,6 +710,27 @@ describe("createRoutineRoutes", () => { expect(listBody.items).toHaveLength(0); }); + test("GET /routines/:id/runs reports completed and endedAt after a finished fire", async () => { + const endedAt = "2026-01-01T00:01:00.000Z"; + const deps = buildDeps({ + runSummaryResolver: { + async resolveRunSummary(_tenantId, _runId) { + return { status: "completed", endedAt }; + }, + }, + }); + const app = mountAs(createRoutineRoutes(deps), "user_1"); + const { body: created } = await createRoutine(app, VALID_BODY); + await app.request(`/routines/${created["id"]}/run`, { method: "POST" }); + + const response = await app.request(`/routines/${created["id"]}/runs`); + const body = (await response.json()) as { + items: { run?: { status: string; endedAt: string } }[]; + }; + expect(body.items[0]?.run?.status).toBe("completed"); + expect(body.items[0]?.run?.endedAt).toBe(endedAt); + }); + test("run summaries enrich when a resolver is wired, and are omitted without one", async () => { const deps = buildDeps({ runSummaryResolver: { diff --git a/packages/run-scope/src/in-flight-turns.ts b/packages/run-scope/src/in-flight-turns.ts new file mode 100644 index 000000000..d25c80647 --- /dev/null +++ b/packages/run-scope/src/in-flight-turns.ts @@ -0,0 +1,43 @@ +// In-flight inference turns for a batch of workflow runs. Listing +// surfaces (Insights `feed=fires`, routine run summaries) attach these +// so a live tool-loop older than the abandoned-fire window still reads +// as running, without adding a field to Interchange's WorkflowRunResponse. +import { and, eq, inArray } from "drizzle-orm"; +import type { DB } from "@intx/db"; +import { inferenceTurn } from "@intx/db/schema"; + +export type InFlightListingTurn = { + readonly status: "running"; + readonly endedAt: null; +}; + +/** + * Run ids in `runIds` that currently have at least one `inference_turn` + * still `running`. Every requested id is present after the query — an + * empty array means queried, none in flight, not "turns omitted". Empty + * input is an empty map — no round trip. + */ +export async function listingTurnsByRunId( + db: DB["db"], + runIds: readonly string[], +): Promise> { + if (runIds.length === 0) return new Map(); + const byRun = new Map(); + for (const id of runIds) byRun.set(id, []); + const rows = await db + .select({ runId: inferenceTurn.runId }) + .from(inferenceTurn) + .where( + and( + inArray(inferenceTurn.runId, [...runIds]), + eq(inferenceTurn.status, "running"), + ), + ); + const turn: InFlightListingTurn = { status: "running", endedAt: null }; + for (const row of rows) { + const existing = byRun.get(row.runId); + if (existing === undefined) byRun.set(row.runId, [turn]); + else existing.push(turn); + } + return byRun; +} diff --git a/packages/run-scope/src/index.ts b/packages/run-scope/src/index.ts index 7ded94637..64db5d0fb 100644 --- a/packages/run-scope/src/index.ts +++ b/packages/run-scope/src/index.ts @@ -6,3 +6,5 @@ export { type ResolveRoutineFires, type RoutineFireInfo, } from "./scope-routes"; +export { listingTurnsByRunId } from "./in-flight-turns"; +export type { InFlightListingTurn } from "./in-flight-turns"; diff --git a/packages/run-scope/src/scope-routes.ts b/packages/run-scope/src/scope-routes.ts index 204a57a16..a0708132f 100644 --- a/packages/run-scope/src/scope-routes.ts +++ b/packages/run-scope/src/scope-routes.ts @@ -34,6 +34,7 @@ import { workflowDefinition, workflowRun } from "@intx/db/schema"; import type { WorkflowRunStatus } from "@intx/types"; import { foldedRun } from "@corbits/folded-runs"; import { makeErrorEnvelope } from "@corbits/error-sink"; +import { listingTurnsByRunId } from "./in-flight-turns"; /** A routine fire's parent, resolved by `resolveRoutineFires` below. */ export type RoutineFireInfo = { @@ -255,28 +256,37 @@ export async function listTopLevelRunFires( ? await resolveRoutineFires(foldedIds) : new Map(); - return rows - .filter((row) => row.foldedMarkerId === null || routineFires.has(row.id)) - .map((row) => { - const fire = routineFires.get(row.id); - return { - id: row.id, - definitionId: row.definitionId, - definitionName: row.definitionName, - tenantId: row.tenantId, - // Guarded by `isNotNull(workflowRun.address)` above. - address: row.address as string, - status: toViewStatus(row.status), - publicKey: row.publicKey, - kernelId: row.kernelId, - sidecarId: row.sidecarId, - createdAt: toTimestamp(row.createdAt), - updatedAt: toTimestamp(row.endedAt ?? row.createdAt), - endedAt: row.endedAt ? toTimestamp(row.endedAt) : null, - routineId: fire?.routineId ?? null, - routineName: fire?.routineName ?? null, - }; - }); + const kept = rows.filter( + (row) => row.foldedMarkerId === null || routineFires.has(row.id), + ); + const turnsByRun = await listingTurnsByRunId( + db, + kept.map((row) => row.id), + ); + + return kept.map((row) => { + const fire = routineFires.get(row.id); + const turns = turnsByRun.get(row.id) ?? []; + return { + id: row.id, + definitionId: row.definitionId, + definitionName: row.definitionName, + tenantId: row.tenantId, + // Guarded by `isNotNull(workflowRun.address)` above. + address: row.address as string, + status: toViewStatus(row.status), + publicKey: row.publicKey, + kernelId: row.kernelId, + sidecarId: row.sidecarId, + createdAt: toTimestamp(row.createdAt), + updatedAt: toTimestamp(row.endedAt ?? row.createdAt), + endedAt: row.endedAt ? toTimestamp(row.endedAt) : null, + routineId: fire?.routineId ?? null, + routineName: fire?.routineName ?? null, + hasInFlightTurn: turns.length > 0, + turns, + }; + }); } /** diff --git a/packages/run-scope/test/scope-routes.drizzle.test.ts b/packages/run-scope/test/scope-routes.drizzle.test.ts index 32007aa25..2eef71782 100644 --- a/packages/run-scope/test/scope-routes.drizzle.test.ts +++ b/packages/run-scope/test/scope-routes.drizzle.test.ts @@ -333,6 +333,8 @@ describeIfDb("listTopLevelRunFires", () => { definitionName: "workbench-digest", routineId: "rtn_pulse_check", routineName: "Pulse check", + hasInFlightTurn: false, + turns: [], }); const direct = rows.find((row) => row.id === "run_direct_deployment1"); @@ -340,6 +342,8 @@ describeIfDb("listTopLevelRunFires", () => { definitionName: "researcher", routineId: null, routineName: null, + hasInFlightTurn: false, + turns: [], }); } finally { await close();