From 436b65867c942df934e36dd355664ed0049c1948 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 01:43:43 -0700 Subject: [PATCH 1/6] Stop showing finished routine fires as still running (CL-6778) Warm-keep leaves a fire's delivery agent deployed, so workflow_run.status stays running after the reply. Insights, Mission Control, and the shell activity feed now read status through runOutcomeStatus: a running fire past the fire window is finished, not in-flight forever. --- apps/web/src/insights-stats.test.ts | 21 +++++++ apps/web/src/insights-stats.ts | 6 +- .../src/pages/insights-page-render.test.tsx | 16 +++-- apps/web/src/pages/insights-page.test.ts | 58 +++++++++++++++++++ apps/web/src/pages/insights-page.tsx | 56 ++++++++++++++---- apps/web/src/pages/mission-control-page.tsx | 29 +++++++--- apps/web/src/shell/routine-activity.ts | 11 +++- apps/web/test/insights-page.test.tsx | 2 +- apps/web/test/mission-control-page.test.tsx | 35 ++++++++++- apps/web/test/routine-activity.test.ts | 26 ++++++++- packages/routines/src/client.ts | 1 + packages/routines/src/health.test.ts | 38 ++++++++++++ packages/routines/src/health.ts | 25 +++++++- packages/routines/src/run-language.test.ts | 2 + packages/routines/src/run-language.ts | 2 + 15 files changed, 286 insertions(+), 42 deletions(-) diff --git a/apps/web/src/insights-stats.test.ts b/apps/web/src/insights-stats.test.ts index 6d8818a20..f5ba9b04f 100644 --- a/apps/web/src/insights-stats.test.ts +++ b/apps/web/src/insights-stats.test.ts @@ -1,10 +1,13 @@ import { describe, expect, test } from "bun:test"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; + import { computeInsightsStats, computeTraceStats, filterRunsByCreatedAt, groupRunsByDefinition, + INSIGHTS_RECENT_LIMIT, purposeRunsForInsights, runDisplayName, } from "./insights-stats"; @@ -94,6 +97,8 @@ describe("computeInsightsStats", () => { routine({ id: "r1", enabled: true }), routine({ id: "r2", enabled: false }), ], + INSIGHTS_RECENT_LIMIT, + Date.parse("2026-01-03T00:01:00.000Z"), ); expect(stats.totalRuns).toBe(3); @@ -105,6 +110,22 @@ describe("computeInsightsStats", () => { expect(stats.recentRuns.map((r) => r.id)).toEqual(["1", "2", "3"]); }); + test("warm-keep: a running run past the fire window is not counted as running", () => { + const stats = computeInsightsStats( + [ + run({ + id: "stale", + status: "running", + createdAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + }), + ], + [], + ); + expect(stats.running).toBe(0); + }); + test("limits recent runs", () => { const runs = Array.from({ length: 5 }, (_, i) => run({ diff --git a/apps/web/src/insights-stats.ts b/apps/web/src/insights-stats.ts index 71ce56340..78bce82ff 100644 --- a/apps/web/src/insights-stats.ts +++ b/apps/web/src/insights-stats.ts @@ -3,6 +3,7 @@ // existing endpoints. import { isWorkbenchHostDefinitionName } from "@corbits/chat/workbench-host-naming"; +import { runOutcomeStatus } from "@corbits/routines/client"; import type { InsightsRun, RunTraceSpan } from "./insights-api"; import type { Routine } from "./routines-api"; @@ -158,6 +159,7 @@ export function computeInsightsStats( runs: readonly InsightsRun[], routines: readonly Routine[], recentLimit: number = INSIGHTS_RECENT_LIMIT, + now: number = Date.now(), ): InsightsStats { const purposeful = purposeRunsForInsights(runs); let running = 0; @@ -165,7 +167,8 @@ export function computeInsightsStats( let stopped = 0; let deployed = 0; for (const run of purposeful) { - switch (run.status) { + const outcome = runOutcomeStatus(run, now) ?? run.status; + switch (outcome) { case "running": case "updating": running += 1; @@ -174,6 +177,7 @@ export function computeInsightsStats( errored += 1; break; case "stopped": + case "completed": stopped += 1; break; case "deployed": diff --git a/apps/web/src/pages/insights-page-render.test.tsx b/apps/web/src/pages/insights-page-render.test.tsx index 4be3a9fa3..7b7bdbaa0 100644 --- a/apps/web/src/pages/insights-page-render.test.tsx +++ b/apps/web/src/pages/insights-page-render.test.tsx @@ -135,11 +135,10 @@ describe("InsightsPage 'Running now' strip", () => { expect(el.textContent).toContain("Weekly digest"); }); - // Liveness is not a windowed property: a run that started long before the - // 7-day window and is still running must not disappear from the strip or - // read 0 in the "Running now" KPI just because its start time falls - // outside `range`. - test("a run started 8 days ago that is still running stays in the strip and the KPI", () => { + // Warm-keep (CL-6681 / CL-6778): a lingering `running` status from 8 + // days ago is not an in-flight job. "Running now" still reads the full + // run set (not the 7-day range), but `runOutcomeStatus` is the reading. + test("warm-keep: a running run started 8 days ago is not in the strip or the KPI", () => { const eightDaysAgo = new Date( Date.now() - 8 * 24 * 60 * 60 * 1000, ).toISOString(); @@ -160,10 +159,9 @@ describe("InsightsPage 'Running now' strip", () => { ], nextCursor: null, }); - expect(el.textContent).toContain("Running now"); - expect(el.textContent).toContain("1 in progress"); - expect(el.textContent).toContain("Long haul"); - expect(el.textContent).toContain("in flight"); + expect(el.textContent).not.toContain("Running now"); + expect(el.textContent).not.toContain("1 in progress"); + expect(el.textContent).not.toContain("in flight"); }); test("the elapsed label ticks forward while a run is live, not frozen at first render", async () => { diff --git a/apps/web/src/pages/insights-page.test.ts b/apps/web/src/pages/insights-page.test.ts index b0927ea12..4f3ba8ed5 100644 --- a/apps/web/src/pages/insights-page.test.ts +++ b/apps/web/src/pages/insights-page.test.ts @@ -9,10 +9,12 @@ // so this landing scope never takes a workbench override anymore. import { describe, expect, test } from "bun:test"; import type { DayActivity } from "@corbits/insights/client"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; import { costPerDay, elapsedLabel, + isRunningNow, resolveInsightsScope, runsPerDay, } from "./insights-page"; @@ -178,3 +180,59 @@ describe("elapsedLabel", () => { expect(elapsedLabel("not-a-date", Date.now())).toBe("—"); }); }); + +describe("isRunningNow", () => { + test("a running run still inside the fire window is in flight", () => { + expect( + isRunningNow( + run({ + id: "fresh", + createdAt: new Date().toISOString(), + status: "running", + }), + ), + ).toBe(true); + }); + + test("updating is in flight", () => { + expect( + isRunningNow( + run({ + id: "updating", + createdAt: new Date().toISOString(), + status: "updating", + }), + ), + ).toBe(true); + }); + + test("a stopped run is not in flight", () => { + expect( + isRunningNow( + run({ + id: "done", + createdAt: new Date().toISOString(), + status: "stopped", + }), + ), + ).toBe(false); + }); + + // Warm-keep (CL-6681 / CL-6778): a routine's delivery agent stays deployed + // after it replies, so workflow_run.status lingers on `running`. Past the + // fire window that is not an in-flight job — Insights must not keep it in + // "Running now" forever. + test("warm-keep: a running run past the fire window is not in flight", () => { + expect( + isRunningNow( + run({ + id: "stale", + createdAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + status: "running", + }), + ), + ).toBe(false); + }); +}); diff --git a/apps/web/src/pages/insights-page.tsx b/apps/web/src/pages/insights-page.tsx index b646ea638..1118ad312 100644 --- a/apps/web/src/pages/insights-page.tsx +++ b/apps/web/src/pages/insights-page.tsx @@ -13,7 +13,6 @@ import { PageShell, RichEmptyState, RUN_STATUS_DOT_TONE, - RUN_STATUS_LABEL, RUN_STATUS_TONE, Skeleton, StatGrid, @@ -33,6 +32,7 @@ import { type TraceSpan, } from "@corbits/react-ui"; import { ChartBar } from "@corbits/icons"; +import { runOutcomeStatus, runStatusLabel } from "@corbits/routines/client"; import type * as React from "react"; import { useEffect, useMemo, useState } from "react"; @@ -54,7 +54,7 @@ import { type OverallUsage, } from "@corbits/insights/client"; -import type { WorkflowRunStatus } from "@intx/types"; +import { workflowRunStatuses, type WorkflowRunStatus } from "@intx/types"; import { SignedOutNotice, type APIQuery } from "@corbits/api-query"; import { workbenchesQueryKey, @@ -145,6 +145,18 @@ export function statusTone(status: WorkflowRunStatus): BadgeTone { return RUN_STATUS_TONE[WORKFLOW_RUN_STATUS_ALIAS[status]]; } +function isWorkflowRunStatus(status: string): status is WorkflowRunStatus { + return workflowRunStatuses.some((value) => value === status); +} + +function insightsStatusTone(status: string): BadgeTone { + if (status === "completed") return RUN_STATUS_TONE.completed; + if (status === "failed") return RUN_STATUS_TONE.failed; + if (status === "cancelled") return RUN_STATUS_TONE.stopped; + if (isWorkflowRunStatus(status)) return statusTone(status); + return "neutral"; +} + function tileValue(value: string | number | null, loading: boolean): string { if (loading) return ""; if (value === null) return "—"; @@ -455,9 +467,20 @@ function useTickingNow(enabled: boolean): number { /** A run actually in flight right now (`status: running | updating`) — * liveness is not a windowed property, so this filters the full run set, - * never the range-filtered one. */ -function isRunningNow(run: InsightsRun): boolean { - return run.status === "running" || run.status === "updating"; + * never the range-filtered one. Warm-keep (CL-6681) leaves a finished + * fire's delivery agent deployed, so a lingering `running` past the fire + * window is not in flight — `runOutcomeStatus` is the one reading. + */ +export function isRunningNow( + run: InsightsRun, + now: number = Date.now(), +): boolean { + const outcome = runOutcomeStatus(run, now); + return outcome === "running" || outcome === "updating"; +} + +function insightsRunStatus(run: InsightsRun, now: number = Date.now()): string { + return runOutcomeStatus(run, now) ?? run.status; } /** @@ -475,8 +498,11 @@ function RunningNowStrip({ readonly runs: readonly InsightsRun[]; readonly onOpenRun: (id: string) => void; }) { - const running = runs.filter(isRunningNow); - const now = useTickingNow(running.length > 0); + const maybeLive = runs.some( + (run) => run.status === "running" || run.status === "updating", + ); + const now = useTickingNow(maybeLive); + const running = runs.filter((run) => isRunningNow(run, now)); if (running.length === 0) return null; return ( @@ -496,7 +522,7 @@ function RunningNowStrip({ onClick={() => onOpenRun(run.id)} > @@ -507,7 +533,7 @@ function RunningNowStrip({ {elapsedLabel(run.createdAt, now)} - {RUN_STATUS_LABEL.running} + {runStatusLabel("running")} @@ -612,7 +638,9 @@ function RecentRunRows({ - {row.status} + + {runStatusLabel(insightsRunStatus(row))} + ))} @@ -680,7 +708,9 @@ function InsightsLanding({ // `range.from` and is still going is running right now regardless of when // it started, so "Running now" reads off every fetched run, never the // range-filtered subset above. - const runningNow = purposeRunsForInsights(runs).filter(isRunningNow); + const runningNow = purposeRunsForInsights(runs).filter((run) => + isRunningNow(run), + ); // Absent usage → zeros at the client boundary (never demo peaks / em-dash // for "no spend"). Real fetched summary is preserved when present. @@ -999,7 +1029,9 @@ function DefinitionRunTable({ {...onRowActivate(() => onOpenRun(row.id))} > - {row.status} + + {runStatusLabel(insightsRunStatus(row))} + {formatWhen(row.createdAt)} {runDurationLabel(row)} diff --git a/apps/web/src/pages/mission-control-page.tsx b/apps/web/src/pages/mission-control-page.tsx index 5c7909040..cecdb1a50 100644 --- a/apps/web/src/pages/mission-control-page.tsx +++ b/apps/web/src/pages/mission-control-page.tsx @@ -32,6 +32,7 @@ import { useQueryClient } from "@tanstack/react-query"; import { formatUsd } from "@corbits/insights/client"; import { CHAT_STRINGS, type Workbench } from "@corbits/chat-ui"; +import { runOutcomeStatus, runStatusLabel } from "@corbits/routines/client"; import { approveApproval, rejectApproval, useAPIQuery } from "../api"; import { useBench } from "../bench-context"; @@ -63,13 +64,21 @@ type InFlightRow = { readonly steps: string; }; -function routineInFlightRow(routine: RoutineActivityItem): InFlightRow { +function routineInFlightRow( + routine: RoutineActivityItem, + now: number, +): InFlightRow { + const status = + runOutcomeStatus( + { createdAt: routine.startedAt, status: routine.status }, + now, + ) ?? routine.status; return { key: `routine:${routine.id}`, label: routine.name, context: "routine", createdAt: routine.startedAt, - statusLabel: "Running", + statusLabel: runStatusLabel(status), statusTone: RUN_STATUS_TONE.running, // The routine feed carries no step count — an honest dash, not a guess. steps: "—", @@ -79,10 +88,17 @@ function routineInFlightRow(routine: RoutineActivityItem): InFlightRow { /** Every routine this bench is actively running right now, newest first. */ export function computeInFlightRows( routines: readonly RoutineActivityItem[], + now: number = Date.now(), ): readonly InFlightRow[] { const rows = routines - .filter((routine) => routine.status === "running") - .map(routineInFlightRow); + .filter( + (routine) => + runOutcomeStatus( + { createdAt: routine.startedAt, status: routine.status }, + now, + ) === "running", + ) + .map((routine) => routineInFlightRow(routine, now)); return rows.sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); } @@ -227,10 +243,7 @@ export function MissionControlRoute({ const inFlightRows = activity.kind === "ready" ? computeInFlightRows(activity.routines) : []; const activeRunsCount = - activity.kind === "ready" - ? activity.routines.filter((routine) => routine.status === "running") - .length - : null; + activity.kind === "ready" ? inFlightRows.length : null; const jumpBackRows = activity.kind === "ready" diff --git a/apps/web/src/shell/routine-activity.ts b/apps/web/src/shell/routine-activity.ts index 9b7646289..c17143382 100644 --- a/apps/web/src/shell/routine-activity.ts +++ b/apps/web/src/shell/routine-activity.ts @@ -10,6 +10,8 @@ // toward Mission Control's "Active runs" — exactly CL-6595's desync // between the Routines page's own "Running now" pill and Mission Control's // "0 / nothing running". +import { runOutcomeStatus } from "@corbits/routines/client"; + import { listRoutineRunFires } from "../agents-api"; import type { RunFire } from "../agents-api"; @@ -20,19 +22,22 @@ export type RoutineActivityItem = { readonly startedAt: string; }; -function toRoutineActivityItem(run: RunFire): RoutineActivityItem { +function toRoutineActivityItem(run: RunFire, now: number): RoutineActivityItem { return { id: run.id, name: run.routineName ?? run.definitionName, - status: run.status, + status: runOutcomeStatus(run, now) ?? run.status, startedAt: run.createdAt, }; } export function listRoutineActivity( tenantId: string, + now: number = Date.now(), ): Promise { return listRoutineRunFires(tenantId).then((runs) => - runs.filter((run) => run.routineId !== null).map(toRoutineActivityItem), + runs + .filter((run) => run.routineId !== null) + .map((run) => toRoutineActivityItem(run, now)), ); } diff --git a/apps/web/test/insights-page.test.tsx b/apps/web/test/insights-page.test.tsx index 02a205194..ed59876ff 100644 --- a/apps/web/test/insights-page.test.tsx +++ b/apps/web/test/insights-page.test.tsx @@ -563,7 +563,7 @@ describe("InsightsRunsHistory definition grouping", () => { expect(groups[0]?.getAttribute("data-definition-group")).toBe("wfd_a"); const firstGroupRows = groups[0]?.querySelectorAll("tbody tr") ?? []; expect(firstGroupRows.length).toBe(2); - expect(firstGroupRows[0]?.textContent).toContain("error"); + expect(firstGroupRows[0]?.textContent).toContain("Failed"); expect(el.textContent).not.toContain("Showing the 100 most recent runs."); }); diff --git a/apps/web/test/mission-control-page.test.tsx b/apps/web/test/mission-control-page.test.tsx index 8b95c66e1..89a77d4c2 100644 --- a/apps/web/test/mission-control-page.test.tsx +++ b/apps/web/test/mission-control-page.test.tsx @@ -4,6 +4,7 @@ import { createRoot } from "react-dom/client"; import type { Root } from "react-dom/client"; import type { Workbench } from "@corbits/chat-ui"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; import { BenchContext, type BenchState } from "../src/bench-context"; import { NavigationProvider } from "../src/navigation"; @@ -26,7 +27,7 @@ function routine(overrides: Partial): RoutineActivityItem { id: "rtn_1", name: "Weekly digest", status: "running", - startedAt: "2026-08-19T09:00:00.000Z", + startedAt: new Date().toISOString(), ...overrides, }; } @@ -57,16 +58,44 @@ describe("computeInFlightRows", () => { routine({ id: "old", status: "running", - startedAt: "2026-08-19T08:00:00.000Z", + startedAt: new Date(Date.now() - 60_000).toISOString(), }), routine({ id: "new", status: "running", - startedAt: "2026-08-19T11:00:00.000Z", + startedAt: new Date().toISOString(), }), ]); expect(rows.map((row) => row.key)).toEqual(["routine:new", "routine:old"]); }); + + test("a running routine still inside the fire window stays in-flight", () => { + const rows = computeInFlightRows([ + routine({ + id: "fresh", + status: "running", + startedAt: new Date().toISOString(), + }), + ]); + expect(rows.map((row) => row.key)).toEqual(["routine:fresh"]); + expect(rows[0]?.statusLabel).toBe("Running now"); + }); + + // Warm-keep (CL-6681 / CL-6778): Mission Control's active-run count and + // in-flight table must not keep a finished fire as "Running" forever just + // because the delivery agent is still deployed. + test("warm-keep: a running routine past the fire window is not in-flight", () => { + const rows = computeInFlightRows([ + routine({ + id: "stale", + status: "running", + startedAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + }), + ]); + expect(rows).toEqual([]); + }); }); describe("computeJumpBackRows", () => { diff --git a/apps/web/test/routine-activity.test.ts b/apps/web/test/routine-activity.test.ts index 86a0cc293..15b5507c9 100644 --- a/apps/web/test/routine-activity.test.ts +++ b/apps/web/test/routine-activity.test.ts @@ -9,6 +9,8 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client"; + import { listRoutineActivity } from "../src/shell/routine-activity"; const realFetch = globalThis.fetch; @@ -38,8 +40,8 @@ const runningFire = { tenantId: "tnt_1", address: "run_1@tnt1.example", status: "running", - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), routineId: "rtn_1", routineName: "Weekly digest", }; @@ -81,7 +83,7 @@ describe("listRoutineActivity", () => { id: "run_1", name: "Weekly digest", status: "running", - startedAt: "2026-01-01T00:00:00.000Z", + startedAt: runningFire.createdAt, }, ]); }); @@ -98,6 +100,24 @@ describe("listRoutineActivity", () => { expect(item?.status).not.toBe("running"); }); + // Warm-keep (CL-6681 / CL-6778): the fires feed still reports `running` + // because the delivery agent stays deployed. Past the fire window that + // row must not count as routine activity in flight. + test("warm-keep: a running fire past the window no longer reads as running", async () => { + stubTopLevelRunsFetch([ + { + ...runningFire, + createdAt: new Date( + Date.now() - FIRE_RUNNING_WINDOW_MS - 1, + ).toISOString(), + }, + ]); + + const [item] = await listRoutineActivity("tnt_1"); + + expect(item?.status).not.toBe("running"); + }); + // A directly-triggered deployment run is also a `feed=fires` row (it is // not a folded run at all), but it has no routine parent -- it must not // be counted as routine activity. diff --git a/packages/routines/src/client.ts b/packages/routines/src/client.ts index b3534c098..d84ce7556 100644 --- a/packages/routines/src/client.ts +++ b/packages/routines/src/client.ts @@ -31,6 +31,7 @@ export { lastFailedFire, medianFireDurationMs, routineHealth, + runOutcomeStatus, } from "./health"; export type { RoutineFire, diff --git a/packages/routines/src/health.test.ts b/packages/routines/src/health.test.ts index 48c518448..66a81e6c6 100644 --- a/packages/routines/src/health.test.ts +++ b/packages/routines/src/health.test.ts @@ -7,6 +7,7 @@ import { lastFailedFire, medianFireDurationMs, routineHealth, + runOutcomeStatus, } from "./health"; import type { RoutineFire, RoutineHealthSubject } from "./health"; @@ -47,6 +48,13 @@ describe("fireOutcomeStatus", () => { ).toBe("completed"); }); + test("a running fire one millisecond inside its window still reads as running", () => { + const stillInside = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS; + expect( + fireOutcomeStatus(fire("r1", {}, { status: "running" }), stillInside), + ).toBe("running"); + }); + test("every terminal status passes through unchanged, however old", () => { const longAfter = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS * 10; expect( @@ -62,6 +70,36 @@ describe("fireOutcomeStatus", () => { }); }); +describe("runOutcomeStatus", () => { + test("a top-level running status still inside its window reads as running", () => { + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "running" }, NOW), + ).toBe("running"); + }); + + test("warm-keep: a top-level running status past its window reads as completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running" }, + staleNow, + ), + ).toBe("completed"); + }); + + test("updating and terminal platform statuses pass through", () => { + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "updating" }, NOW), + ).toBe("updating"); + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "error" }, NOW), + ).toBe("error"); + expect( + runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "stopped" }, NOW), + ).toBe("stopped"); + }); +}); + describe("cleanFireStreak", () => { test("counts successes from the newest fire and stops at the first failure", () => { expect( diff --git a/packages/routines/src/health.ts b/packages/routines/src/health.ts index 0fa8691a1..c5fc25b1f 100644 --- a/packages/routines/src/health.ts +++ b/packages/routines/src/health.ts @@ -65,7 +65,9 @@ export type RoutineHealth = { readonly medianDurationMs: number | null; }; -function statusOf(fire: RoutineFire): string | null { +function statusOf(fire: { + readonly run?: Record; +}): string | null { const status = fire.run?.status; return typeof status === "string" ? status : null; } @@ -91,7 +93,9 @@ export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; * `fire.run?.status` directly. */ export function fireOutcomeStatus( - fire: RoutineFire, + fire: Pick & { + readonly run?: Record; + }, now: number, ): string | null { const status = statusOf(fire); @@ -101,6 +105,23 @@ export function fireOutcomeStatus( return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running"; } +/** + * `fireOutcomeStatus` for a platform run whose status lives at the top + * level (`workflow_run.status`), not nested on a routine fire's `run`. + * Insights, Mission Control, and the shell activity feed all see that + * shape; they must not read the raw column while warm-keep (CL-6681) + * leaves a finished fire's delivery agent deployed. + */ +export function runOutcomeStatus( + run: { readonly createdAt: string; readonly status: string }, + now: number, +): string | null { + return fireOutcomeStatus( + { createdAt: run.createdAt, run: { status: run.status } }, + now, + ); +} + /** A fire failed when it recorded a launch error (the synthetic * `schedule-failed` row) or its run settled as failed. */ export function fireFailed(fire: RoutineFire): boolean { diff --git a/packages/routines/src/run-language.test.ts b/packages/routines/src/run-language.test.ts index 1c70c0dea..aeeb372ff 100644 --- a/packages/routines/src/run-language.test.ts +++ b/packages/routines/src/run-language.test.ts @@ -9,8 +9,10 @@ import { describe("runStatusLabel", () => { test("every status a routine surface can show reads as words", () => { expect(runStatusLabel("running")).toBe("Running now"); + expect(runStatusLabel("updating")).toBe("Running now"); expect(runStatusLabel("completed")).toBe("Finished"); expect(runStatusLabel("failed")).toBe("Failed"); + expect(runStatusLabel("error")).toBe("Failed"); expect(runStatusLabel("cancelled")).toBe("Cancelled"); expect(runStatusLabel("queued")).toBe("Waiting to start"); }); diff --git a/packages/routines/src/run-language.ts b/packages/routines/src/run-language.ts index 9157b773f..97e7d4851 100644 --- a/packages/routines/src/run-language.ts +++ b/packages/routines/src/run-language.ts @@ -11,8 +11,10 @@ const RUN_STATUS_WORDS: Readonly> = { running: "Running now", + updating: "Running now", completed: "Finished", failed: "Failed", + error: "Failed", cancelled: "Cancelled", queued: "Waiting to start", pending: "Waiting to start", From 6d53ef429cbc844ad10ce40bcab3bc7ecdcab898 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 02:29:16 -0700 Subject: [PATCH 2/6] Persist finished routine fires and honor endedAt immediately (CL-6778) Warm-keep leaves a fire delivery agent deployed, so workflow_run.status stays running after the reply. Turn finalize now stamps markTerminal plus endedAt for routine fires only. Insights, Mission Control, and health treat that stamp as finished at once. The running window is last-resort for abandoned fires. Failed fires stay failed. --- apps/hub/src/index.ts | 40 +++++- apps/web/src/insights-stats.test.ts | 15 ++ apps/web/src/pages/insights-page.test.ts | 13 ++ apps/web/src/pages/insights-page.tsx | 6 +- apps/web/src/pages/mission-control-page.tsx | 14 +- apps/web/src/shell/routine-activity.ts | 2 + apps/web/test/mission-control-page.test.tsx | 12 ++ apps/web/test/routine-activity.test.ts | 13 ++ packages/routines/src/health.test.ts | 73 ++++++++++ packages/routines/src/health.ts | 46 +++--- packages/routines/src/index.ts | 7 + packages/routines/src/settle-fire.test.ts | 151 ++++++++++++++++++++ packages/routines/src/settle-fire.ts | 68 +++++++++ packages/routines/test/routes.test.ts | 21 +++ 14 files changed, 457 insertions(+), 24 deletions(-) create mode 100644 packages/routines/src/settle-fire.test.ts create mode 100644 packages/routines/src/settle-fire.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 8a467fd92..3b460f0bd 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -14,6 +14,7 @@ import { createSidecarAllocationStore, createSignalCorrelationStore, createWorkflowRunDispatchStore, + createWorkflowRunStore, listVisibleOfferings, resolveCredentialByName, resolveCredentialRequirement, @@ -118,6 +119,7 @@ import type { FinalizedTurnToolCall } from "@corbits/turn-artifacts"; import { decodedOrNull } from "@corbits/url-path"; import { createCryptoProviderCache, + findFoldedRunByAddress, lookupFoldedRunReconnectKey, } from "@corbits/folded-runs"; import { createTopLevelRunRoutes } from "@corbits/run-scope"; @@ -209,6 +211,7 @@ import { resolveLaunchableDefinition, routine as routineTable, routineRun as routineRunTable, + settleRoutineFireFromTurn, } from "@corbits/routines"; import { createSidecarProvisioner as createE2BSidecarProvisioner, @@ -817,6 +820,31 @@ export async function createHub(config: HubConfig) { store: insightsUsage.store, generateId: () => generateId("inferenceTurn"), }); + // CL-6778: a routine fire's folded delivery agent stays deployed after + // it replies, so `workflow_run.status` would linger on `running` unless + // something stamps it terminal. `onTurnFinalized` is that stamp — + // `markTerminal` + `endedAt`, gated on a `routine_run` row so a + // workbench host or invited agent is never settled by accident. + const workflowRunStore = createWorkflowRunStore(db); + const routineFireSettlePort = { + async lookupRunByAddress(address: string) { + const row = await findFoldedRunByAddress(db, address); + return row === undefined ? undefined : { id: row.id }; + }, + async isRoutineFire(runId: string) { + const rows = await db + .select({ runId: routineRunTable.runId }) + .from(routineRunTable) + .where(eq(routineRunTable.runId, runId)) + .limit(1); + return rows.length > 0; + }, + markTerminal: ( + runId: string, + status: "completed" | "failed" | "cancelled", + endedAt: Date, + ) => workflowRunStore.markTerminal(runId, status, endedAt), + }; // CL-6257: per-message-run stage latency (message-received → // reactor.start → inference.start → first-token → reply-posted). The // vendored event collector never persists the events this reads (see @@ -835,8 +863,16 @@ export async function createHub(config: HubConfig) { // surviving loss loud (error-level cause, counted) instead of a // swallowed WRN. db: withTurnPartPersistGuard(withTurnPartWriteDefaults(db)), - onTurnFinalized: (agentAddress, turn) => - artifactDeliveryHandlerRef.current?.(agentAddress, turn), + onTurnFinalized: (agentAddress, turn) => { + artifactDeliveryHandlerRef.current?.(agentAddress, turn); + void settleRoutineFireFromTurn( + routineFireSettlePort, + agentAddress, + turn, + ).catch((err: unknown) => { + log.warn`Failed to settle routine fire for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`; + }); + }, // Per-turn usage, emitted once when the collector finalizes a turn. onUsage: (_agentAddress, usage) => { void usageSink diff --git a/apps/web/src/insights-stats.test.ts b/apps/web/src/insights-stats.test.ts index f5ba9b04f..d0f4273de 100644 --- a/apps/web/src/insights-stats.test.ts +++ b/apps/web/src/insights-stats.test.ts @@ -126,6 +126,21 @@ describe("computeInsightsStats", () => { expect(stats.running).toBe(0); }); + test("endedAt drops a just-finished running run from the running count immediately", () => { + const stats = computeInsightsStats( + [ + run({ + id: "just-finished", + status: "running", + createdAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + }), + ], + [], + ); + expect(stats.running).toBe(0); + }); + test("limits recent runs", () => { const runs = Array.from({ length: 5 }, (_, i) => run({ diff --git a/apps/web/src/pages/insights-page.test.ts b/apps/web/src/pages/insights-page.test.ts index 4f3ba8ed5..f346d0fb5 100644 --- a/apps/web/src/pages/insights-page.test.ts +++ b/apps/web/src/pages/insights-page.test.ts @@ -222,6 +222,19 @@ describe("isRunningNow", () => { // after it replies, so workflow_run.status lingers on `running`. Past the // fire window that is not an in-flight job — Insights must not keep it in // "Running now" forever. + test("endedAt drops in-flight immediately, even while status is still running inside the window", () => { + expect( + isRunningNow( + run({ + id: "just-finished", + createdAt: new Date().toISOString(), + status: "running", + endedAt: new Date().toISOString(), + }), + ), + ).toBe(false); + }); + test("warm-keep: a running run past the fire window is not in flight", () => { expect( isRunningNow( diff --git a/apps/web/src/pages/insights-page.tsx b/apps/web/src/pages/insights-page.tsx index 1118ad312..1b1ec06d5 100644 --- a/apps/web/src/pages/insights-page.tsx +++ b/apps/web/src/pages/insights-page.tsx @@ -467,9 +467,9 @@ function useTickingNow(enabled: boolean): number { /** A run actually in flight right now (`status: running | updating`) — * liveness is not a windowed property, so this filters the full run set, - * never the range-filtered one. Warm-keep (CL-6681) leaves a finished - * fire's delivery agent deployed, so a lingering `running` past the fire - * window is not in flight — `runOutcomeStatus` is the one reading. + * never the range-filtered one. A persisted `endedAt` means the fire + * already finished, even if `status` still reads `running`. The fire + * window is last-resort for abandoned fires that never got that stamp. */ export function isRunningNow( run: InsightsRun, diff --git a/apps/web/src/pages/mission-control-page.tsx b/apps/web/src/pages/mission-control-page.tsx index cecdb1a50..601325ba4 100644 --- a/apps/web/src/pages/mission-control-page.tsx +++ b/apps/web/src/pages/mission-control-page.tsx @@ -70,7 +70,11 @@ function routineInFlightRow( ): InFlightRow { const status = runOutcomeStatus( - { createdAt: routine.startedAt, status: routine.status }, + { + createdAt: routine.startedAt, + status: routine.status, + ...(routine.endedAt !== undefined ? { endedAt: routine.endedAt } : {}), + }, now, ) ?? routine.status; return { @@ -94,7 +98,13 @@ export function computeInFlightRows( .filter( (routine) => runOutcomeStatus( - { createdAt: routine.startedAt, status: routine.status }, + { + createdAt: routine.startedAt, + status: routine.status, + ...(routine.endedAt !== undefined + ? { endedAt: routine.endedAt } + : {}), + }, now, ) === "running", ) diff --git a/apps/web/src/shell/routine-activity.ts b/apps/web/src/shell/routine-activity.ts index c17143382..6756b9042 100644 --- a/apps/web/src/shell/routine-activity.ts +++ b/apps/web/src/shell/routine-activity.ts @@ -20,6 +20,7 @@ export type RoutineActivityItem = { readonly name: string; readonly status: string; readonly startedAt: string; + readonly endedAt?: string | null; }; function toRoutineActivityItem(run: RunFire, now: number): RoutineActivityItem { @@ -28,6 +29,7 @@ function toRoutineActivityItem(run: RunFire, now: number): RoutineActivityItem { name: run.routineName ?? run.definitionName, status: runOutcomeStatus(run, now) ?? run.status, startedAt: run.createdAt, + ...(run.endedAt !== undefined ? { endedAt: run.endedAt } : {}), }; } diff --git a/apps/web/test/mission-control-page.test.tsx b/apps/web/test/mission-control-page.test.tsx index 89a77d4c2..3699d8ddc 100644 --- a/apps/web/test/mission-control-page.test.tsx +++ b/apps/web/test/mission-control-page.test.tsx @@ -84,6 +84,18 @@ describe("computeInFlightRows", () => { // Warm-keep (CL-6681 / CL-6778): Mission Control's active-run count and // in-flight table must not keep a finished fire as "Running" forever just // because the delivery agent is still deployed. + test("endedAt drops a just-finished running routine from in-flight immediately", () => { + const rows = computeInFlightRows([ + routine({ + id: "just-finished", + status: "running", + startedAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + }), + ]); + expect(rows).toEqual([]); + }); + test("warm-keep: a running routine past the fire window is not in-flight", () => { const rows = computeInFlightRows([ routine({ diff --git a/apps/web/test/routine-activity.test.ts b/apps/web/test/routine-activity.test.ts index 15b5507c9..56cea1374 100644 --- a/apps/web/test/routine-activity.test.ts +++ b/apps/web/test/routine-activity.test.ts @@ -100,6 +100,19 @@ describe("listRoutineActivity", () => { expect(item?.status).not.toBe("running"); }); + test("endedAt drops a just-finished running fire from running immediately", async () => { + stubTopLevelRunsFetch([ + { + ...runningFire, + endedAt: new Date().toISOString(), + }, + ]); + + const [item] = await listRoutineActivity("tnt_1"); + + expect(item?.status).not.toBe("running"); + }); + // Warm-keep (CL-6681 / CL-6778): the fires feed still reports `running` // because the delivery agent stays deployed. Past the fire window that // row must not count as routine activity in flight. diff --git a/packages/routines/src/health.test.ts b/packages/routines/src/health.test.ts index 66a81e6c6..40d2e44b9 100644 --- a/packages/routines/src/health.test.ts +++ b/packages/routines/src/health.test.ts @@ -68,6 +68,33 @@ describe("fireOutcomeStatus", () => { test("null when the platform has no run to report", () => { expect(fireOutcomeStatus(fire("r1", {}, {}), NOW)).toBeNull(); }); + + test("endedAt on a still-running column is finished immediately, not at T+window", () => { + expect( + fireOutcomeStatus( + fire( + "r1", + {}, + { status: "running", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + NOW, + ), + ).toBe("completed"); + }); + + test("a persisted failed fire stays failed past the window — never remapped to completed", () => { + const longAfter = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS * 10; + expect( + fireOutcomeStatus( + fire( + "r1", + {}, + { status: "failed", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + longAfter, + ), + ).toBe("failed"); + }); }); describe("runOutcomeStatus", () => { @@ -87,6 +114,19 @@ describe("runOutcomeStatus", () => { ).toBe("completed"); }); + test("endedAt drops a top-level running status immediately", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + endedAt: "2026-01-01T00:00:30.000Z", + }, + NOW, + ), + ).toBe("completed"); + }); + test("updating and terminal platform statuses pass through", () => { expect( runOutcomeStatus({ createdAt: FIRE_CREATED_AT, status: "updating" }, NOW), @@ -225,6 +265,22 @@ describe("routineHealth", () => { ).toBe("running"); }); + test("endedAt on the latest fire is not Running now, even inside the window", () => { + expect( + routineHealth( + healthy, + [ + fire( + "r1", + {}, + { status: "running", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + ], + NOW, + ).state, + ).not.toBe("running"); + }); + test("warm-keep (CL-6681): a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; const health = routineHealth( @@ -236,6 +292,23 @@ describe("routineHealth", () => { expect(health.label).not.toBe("Running now"); }); + test("a persisted failed latest fire is Last run failed, even past the window", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const health = routineHealth( + healthy, + [ + fire( + "r1", + {}, + { status: "failed", endedAt: "2026-01-01T00:00:30.000Z" }, + ), + ], + staleNow, + ); + expect(health.state).toBe("failing"); + expect(health.label).toBe("Last run failed"); + }); + test("consecutive failures are stated in the caption, not just the pill", () => { const health = routineHealth({ ...healthy, consecutiveFailures: 2 }, [ fire("r1", { error: "boom" }, {}), diff --git a/packages/routines/src/health.ts b/packages/routines/src/health.ts index c5fc25b1f..f54e9e51f 100644 --- a/packages/routines/src/health.ts +++ b/packages/routines/src/health.ts @@ -73,24 +73,26 @@ function statusOf(fire: { } /** - * How long a fire may credibly still be doing work before its lingering - * `running` status is read as warm-keep (CL-6681) — a routine's delivery - * agent deliberately stays deployed after it replies, so - * `workflow_run.status` never settles out of `running` on its own. Past - * this window a `running` fire is presumed to have already delivered its - * reply, so every surface badging its status reads it through + * How long a fire may credibly still be doing work before a lingering + * `running` status with no `endedAt` is read as an abandoned fire + * (warm-keep CL-6681 / CL-6778). A routine fire that finished is + * supposed to land `completed`/`failed`/`cancelled` plus `endedAt` via + * `markTerminal`; this window is the last-resort reading for a fire + * that never got that write — never the happy path. Past it, a + * still-`running` fire with no end stamp is presumed to have already + * delivered, so every surface badging its status reads it through * `fireOutcomeStatus` rather than the raw column. */ export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; /** - * A fire's status the way this build should show it: the raw `run.status` - * for every terminal value, but a `running` status older than - * `FIRE_RUNNING_WINDOW_MS` is read as `completed` instead of taken - * literally — see that constant's own comment. This is the one place that - * tells a fire still doing work apart from one merely staying warm; every - * caller that needs a fire's displayed status goes through here, never - * `fire.run?.status` directly. + * A fire's status the way this build should show it: the raw + * `run.status` for every non-running value; a `running` row that + * already carries `endedAt` is finished immediately (the persist + * path); a `running` row with no `endedAt` older than + * `FIRE_RUNNING_WINDOW_MS` is an abandoned fire read as `completed`. + * Every caller that needs a fire's displayed status goes through here, + * never `fire.run?.status` directly. */ export function fireOutcomeStatus( fire: Pick & { @@ -100,6 +102,8 @@ export function fireOutcomeStatus( ): string | null { const status = statusOf(fire); if (status !== "running") return status; + const endedAt = fire.run?.endedAt; + if (typeof endedAt === "string" && endedAt !== "") return "completed"; const startedAt = Date.parse(fire.createdAt); if (Number.isNaN(startedAt)) return status; return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running"; @@ -109,15 +113,23 @@ export function fireOutcomeStatus( * `fireOutcomeStatus` for a platform run whose status lives at the top * level (`workflow_run.status`), not nested on a routine fire's `run`. * Insights, Mission Control, and the shell activity feed all see that - * shape; they must not read the raw column while warm-keep (CL-6681) - * leaves a finished fire's delivery agent deployed. + * shape. `endedAt` is the persist-path signal that the fire already + * finished; the running-window is only for abandoned fires that never + * got that stamp. */ export function runOutcomeStatus( - run: { readonly createdAt: string; readonly status: string }, + run: { + readonly createdAt: string; + readonly status: string; + readonly endedAt?: string | null; + }, now: number, ): string | null { return fireOutcomeStatus( - { createdAt: run.createdAt, run: { status: run.status } }, + { + createdAt: run.createdAt, + run: { status: run.status, endedAt: run.endedAt }, + }, now, ); } diff --git a/packages/routines/src/index.ts b/packages/routines/src/index.ts index 0d0ab56ef..565009004 100644 --- a/packages/routines/src/index.ts +++ b/packages/routines/src/index.ts @@ -69,6 +69,13 @@ export type { RunSummaryResolver, } from "./routes"; +export { settleRoutineFire, settleRoutineFireFromTurn } from "./settle-fire"; +export type { + RoutineFireSettlePort, + RoutineFireSettleStatus, + RoutineFireTurnSettlePort, +} from "./settle-fire"; + export { createWorkflowRoutineRoutes } from "./workflow-routine-routes"; export type { CreateWorkflowRoutineRoutesDeps, diff --git a/packages/routines/src/settle-fire.test.ts b/packages/routines/src/settle-fire.test.ts new file mode 100644 index 000000000..840543eec --- /dev/null +++ b/packages/routines/src/settle-fire.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, test } from "bun:test"; + +import { settleRoutineFire, settleRoutineFireFromTurn } from "./settle-fire"; + +describe("settleRoutineFire", () => { + test("persists completed + endedAt via markTerminal after a finished fire", async () => { + const endedAt = new Date("2026-01-01T00:01:00.000Z"); + const calls: { + runId: string; + status: string; + endedAt: Date; + }[] = []; + + const won = await settleRoutineFire( + { + isRoutineFire: async () => true, + markTerminal: async (runId, status, at) => { + calls.push({ runId, status, endedAt: at }); + return { id: runId, status, endedAt: at }; + }, + }, + { runId: "run_fire1", status: "completed", endedAt }, + ); + + expect(won).toBe(true); + expect(calls).toEqual([ + { runId: "run_fire1", status: "completed", endedAt }, + ]); + }); + + test("a failed fire persists failed, never completed", async () => { + const endedAt = new Date("2026-01-01T00:01:00.000Z"); + const calls: string[] = []; + + await settleRoutineFire( + { + isRoutineFire: async () => true, + markTerminal: async (_runId, status, _at) => { + calls.push(status); + return { status }; + }, + }, + { runId: "run_fire1", status: "failed", endedAt }, + ); + + expect(calls).toEqual(["failed"]); + }); + + test("does not markTerminal a run that is not a routine fire", async () => { + let marked = false; + + const won = await settleRoutineFire( + { + isRoutineFire: async () => false, + markTerminal: async () => { + marked = true; + return {}; + }, + }, + { runId: "run_chat1", status: "completed" }, + ); + + expect(won).toBe(false); + expect(marked).toBe(false); + }); + + test("a losing markTerminal (already terminal) is not a second persist", async () => { + const won = await settleRoutineFire( + { + isRoutineFire: async () => true, + markTerminal: async () => null, + }, + { runId: "run_fire1", status: "completed" }, + ); + + expect(won).toBe(false); + }); +}); + +describe("settleRoutineFireFromTurn", () => { + test("looks up the run by address and persists the turn's terminal status", async () => { + const endedAt = new Date("2026-01-01T00:02:00.000Z"); + const calls: { + runId: string; + status: string; + endedAt: Date; + }[] = []; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async (address) => + address === "run_fire1@acme.workbench.test" + ? { id: "run_fire1" } + : undefined, + isRoutineFire: async (runId) => runId === "run_fire1", + markTerminal: async (runId, status, at) => { + calls.push({ runId, status, endedAt: at }); + return { id: runId, status, endedAt: at }; + }, + }, + "run_fire1@acme.workbench.test", + { status: "completed" }, + endedAt, + ); + + expect(won).toBe(true); + expect(calls).toEqual([ + { runId: "run_fire1", status: "completed", endedAt }, + ]); + }); + + test("a failed turn persists failed", async () => { + const calls: string[] = []; + + await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async (_runId, status) => { + calls.push(status); + return { status }; + }, + }, + "run_fire1@acme.workbench.test", + { status: "failed" }, + new Date("2026-01-01T00:02:00.000Z"), + ); + + expect(calls).toEqual(["failed"]); + }); + + test("unknown address is a no-op", async () => { + let marked = false; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => undefined, + isRoutineFire: async () => true, + markTerminal: async () => { + marked = true; + return {}; + }, + }, + "run_missing@acme.workbench.test", + { status: "completed" }, + ); + + expect(won).toBe(false); + expect(marked).toBe(false); + }); +}); diff --git a/packages/routines/src/settle-fire.ts b/packages/routines/src/settle-fire.ts new file mode 100644 index 000000000..ef826b532 --- /dev/null +++ b/packages/routines/src/settle-fire.ts @@ -0,0 +1,68 @@ +// Persist a routine fire's terminal status onto `workflow_run` once the +// fire has actually finished. Warm-keep (CL-6681) leaves a folded delivery +// agent deployed, so the platform never settles `workflow_run.status` on +// its own; Insights and Mission Control then read a finished fire as +// "Running now" forever. `markTerminal` is the one write that stamps +// `completed`/`failed`/`cancelled` and `endedAt` in a single shot — the +// same store method the hub already uses for native workflow runs. +// +// Only a row in `routine_run` is a fire. A workbench host or invited +// agent is also a folded run; marking those terminal would lie about +// their lifecycle. The port's `isRoutineFire` is that gate. + +export type RoutineFireSettleStatus = "completed" | "failed" | "cancelled"; + +export type RoutineFireSettlePort = { + isRoutineFire(runId: string): Promise; + markTerminal( + runId: string, + status: RoutineFireSettleStatus, + endedAt: Date, + ): Promise; +}; + +export type RoutineFireTurnSettlePort = RoutineFireSettlePort & { + lookupRunByAddress(address: string): Promise<{ id: string } | undefined>; +}; + +/** + * Stamp a routine fire terminal. Returns whether this caller won the + * flip (`markTerminal` is single-shot: a later call against an already + * terminal row returns null). A run that is not a routine fire is a + * no-op, never a persist. + */ +export async function settleRoutineFire( + port: RoutineFireSettlePort, + input: { + runId: string; + status: RoutineFireSettleStatus; + endedAt?: Date; + }, +): Promise { + if (!(await port.isRoutineFire(input.runId))) return false; + const won = await port.markTerminal( + input.runId, + input.status, + input.endedAt ?? new Date(), + ); + return won !== null; +} + +/** + * `settleRoutineFire` from a finalized turn: look the run up by the + * turn's agent address, then persist the turn's own terminal status. + */ +export async function settleRoutineFireFromTurn( + port: RoutineFireTurnSettlePort, + address: string, + turn: { readonly status: "completed" | "failed" }, + endedAt: Date = new Date(), +): Promise { + const run = await port.lookupRunByAddress(address); + if (run === undefined) return false; + return settleRoutineFire(port, { + runId: run.id, + status: turn.status, + endedAt, + }); +} diff --git a/packages/routines/test/routes.test.ts b/packages/routines/test/routes.test.ts index ee5822747..6e8d9c13d 100644 --- a/packages/routines/test/routes.test.ts +++ b/packages/routines/test/routes.test.ts @@ -710,6 +710,27 @@ describe("createRoutineRoutes", () => { expect(listBody.items).toHaveLength(0); }); + test("GET /routines/:id/runs reports completed and endedAt after a finished fire", async () => { + const endedAt = "2026-01-01T00:01:00.000Z"; + const deps = buildDeps({ + runSummaryResolver: { + async resolveRunSummary(_tenantId, _runId) { + return { status: "completed", endedAt }; + }, + }, + }); + const app = mountAs(createRoutineRoutes(deps), "user_1"); + const { body: created } = await createRoutine(app, VALID_BODY); + await app.request(`/routines/${created["id"]}/run`, { method: "POST" }); + + const response = await app.request(`/routines/${created["id"]}/runs`); + const body = (await response.json()) as { + items: { run?: { status: string; endedAt: string } }[]; + }; + expect(body.items[0]?.run?.status).toBe("completed"); + expect(body.items[0]?.run?.endedAt).toBe(endedAt); + }); + test("run summaries enrich when a resolver is wired, and are omitted without one", async () => { const deps = buildDeps({ runSummaryResolver: { From 4d3fc3f9843378807e513939e38618f756c9f38c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 14:55:32 -0700 Subject: [PATCH 3/6] Skip intermediate tool steps when settling routine fires (CL-6778) onTurnFinalized also fires on tool-use steps as completed with hadReply false. Stamping those terminal is single-shot, so a later failed turn cannot persist. Skip that pair; settle on a real reply or a failed turn. --- apps/hub/src/index.ts | 9 ++- packages/routines/src/settle-fire.test.ts | 77 ++++++++++++++++++++++- packages/routines/src/settle-fire.ts | 8 ++- 3 files changed, 85 insertions(+), 9 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 3b460f0bd..31a04010a 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -865,11 +865,10 @@ export async function createHub(config: HubConfig) { db: withTurnPartPersistGuard(withTurnPartWriteDefaults(db)), onTurnFinalized: (agentAddress, turn) => { artifactDeliveryHandlerRef.current?.(agentAddress, turn); - void settleRoutineFireFromTurn( - routineFireSettlePort, - agentAddress, - turn, - ).catch((err: unknown) => { + void settleRoutineFireFromTurn(routineFireSettlePort, agentAddress, { + status: turn.status, + hadReply: turn.hadReply, + }).catch((err: unknown) => { log.warn`Failed to settle routine fire for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`; }); }, diff --git a/packages/routines/src/settle-fire.test.ts b/packages/routines/src/settle-fire.test.ts index 840543eec..894509e8d 100644 --- a/packages/routines/src/settle-fire.test.ts +++ b/packages/routines/src/settle-fire.test.ts @@ -99,7 +99,7 @@ describe("settleRoutineFireFromTurn", () => { }, }, "run_fire1@acme.workbench.test", - { status: "completed" }, + { status: "completed", hadReply: true }, endedAt, ); @@ -122,7 +122,7 @@ describe("settleRoutineFireFromTurn", () => { }, }, "run_fire1@acme.workbench.test", - { status: "failed" }, + { status: "failed", hadReply: false }, new Date("2026-01-01T00:02:00.000Z"), ); @@ -142,10 +142,81 @@ describe("settleRoutineFireFromTurn", () => { }, }, "run_missing@acme.workbench.test", - { status: "completed" }, + { status: "completed", hadReply: true }, ); expect(won).toBe(false); expect(marked).toBe(false); }); + + test("skips an intermediate completed turn that had no reply", async () => { + let marked = false; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async () => { + marked = true; + return {}; + }, + }, + "run_fire1@acme.workbench.test", + { status: "completed", hadReply: false }, + ); + + expect(won).toBe(false); + expect(marked).toBe(false); + }); + + test("persists completed when the turn had a reply", async () => { + const calls: string[] = []; + + const won = await settleRoutineFireFromTurn( + { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async (_runId, status) => { + calls.push(status); + return { status }; + }, + }, + "run_fire1@acme.workbench.test", + { status: "completed", hadReply: true }, + ); + + expect(won).toBe(true); + expect(calls).toEqual(["completed"]); + }); + + test("persists failed after a skipped intermediate tool step", async () => { + const calls: string[] = []; + const port = { + lookupRunByAddress: async () => ({ id: "run_fire1" }), + isRoutineFire: async () => true, + markTerminal: async ( + _runId: string, + status: "completed" | "failed" | "cancelled", + ) => { + if (calls.length > 0) return null; + calls.push(status); + return { status }; + }, + }; + + const skipped = await settleRoutineFireFromTurn( + port, + "run_fire1@acme.workbench.test", + { status: "completed", hadReply: false }, + ); + const won = await settleRoutineFireFromTurn( + port, + "run_fire1@acme.workbench.test", + { status: "failed", hadReply: false }, + ); + + expect(skipped).toBe(false); + expect(won).toBe(true); + expect(calls).toEqual(["failed"]); + }); }); diff --git a/packages/routines/src/settle-fire.ts b/packages/routines/src/settle-fire.ts index ef826b532..d5b74d6e5 100644 --- a/packages/routines/src/settle-fire.ts +++ b/packages/routines/src/settle-fire.ts @@ -51,13 +51,19 @@ export async function settleRoutineFire( /** * `settleRoutineFire` from a finalized turn: look the run up by the * turn's agent address, then persist the turn's own terminal status. + * + * Intermediate tool-use steps finalize as `completed` with `hadReply: + * false`. Those must not stamp the fire — `markTerminal` is single-shot, + * so an early completed would hide a later failure. A real reply + * (`hadReply: true`) or a failed turn settles immediately. */ export async function settleRoutineFireFromTurn( port: RoutineFireTurnSettlePort, address: string, - turn: { readonly status: "completed" | "failed" }, + turn: { readonly status: "completed" | "failed"; readonly hadReply: boolean }, endedAt: Date = new Date(), ): Promise { + if (turn.status === "completed" && !turn.hadReply) return false; const run = await port.lookupRunByAddress(address); if (run === undefined) return false; return settleRoutineFire(port, { From abc26ab851b9262c4460901e6452d996747d4749 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 16:51:16 -0700 Subject: [PATCH 4/6] Keep live routine fires running until persist or abandonment (CL-6778) The 10-minute window remapped any still-running fire without endedAt to completed, so a live tool loop older than ten minutes showed Finished. Apply that window only when the fire is already known abandoned. Persist via endedAt still finishes immediately. --- apps/web/src/insights-stats.test.ts | 4 +- .../src/pages/insights-page-render.test.tsx | 17 ++--- apps/web/src/pages/insights-page.test.ts | 4 +- apps/web/src/pages/insights-page.tsx | 5 +- apps/web/test/mission-control-page.test.tsx | 4 +- apps/web/test/routine-activity.test.ts | 9 ++- apps/web/test/routines-page.test.tsx | 6 +- packages/routines/src/health.test.ts | 63 +++++++++++++++++-- packages/routines/src/health.ts | 36 +++++++---- 9 files changed, 106 insertions(+), 42 deletions(-) diff --git a/apps/web/src/insights-stats.test.ts b/apps/web/src/insights-stats.test.ts index d0f4273de..420d378cf 100644 --- a/apps/web/src/insights-stats.test.ts +++ b/apps/web/src/insights-stats.test.ts @@ -110,7 +110,7 @@ describe("computeInsightsStats", () => { expect(stats.recentRuns.map((r) => r.id)).toEqual(["1", "2", "3"]); }); - test("warm-keep: a running run past the fire window is not counted as running", () => { + test("a live running run past the fire window is still counted as running", () => { const stats = computeInsightsStats( [ run({ @@ -123,7 +123,7 @@ describe("computeInsightsStats", () => { ], [], ); - expect(stats.running).toBe(0); + expect(stats.running).toBe(1); }); test("endedAt drops a just-finished running run from the running count immediately", () => { diff --git a/apps/web/src/pages/insights-page-render.test.tsx b/apps/web/src/pages/insights-page-render.test.tsx index 7b7bdbaa0..ab33245e6 100644 --- a/apps/web/src/pages/insights-page-render.test.tsx +++ b/apps/web/src/pages/insights-page-render.test.tsx @@ -135,10 +135,12 @@ describe("InsightsPage 'Running now' strip", () => { expect(el.textContent).toContain("Weekly digest"); }); - // Warm-keep (CL-6681 / CL-6778): a lingering `running` status from 8 - // days ago is not an in-flight job. "Running now" still reads the full - // run set (not the 7-day range), but `runOutcomeStatus` is the reading. - test("warm-keep: a running run started 8 days ago is not in the strip or the KPI", () => { + // Liveness is not a windowed property: a run that started long before the + // 7-day window and is still running must not disappear from the strip or + // read 0 in the "Running now" KPI just because its start time falls + // outside `range`. Persist has not settled (`endedAt` absent), so the + // fire is live — not remapped to completed by the abandoned-fire window. + test("a run started 8 days ago that is still running stays in the strip and the KPI", () => { const eightDaysAgo = new Date( Date.now() - 8 * 24 * 60 * 60 * 1000, ).toISOString(); @@ -159,9 +161,10 @@ describe("InsightsPage 'Running now' strip", () => { ], nextCursor: null, }); - expect(el.textContent).not.toContain("Running now"); - expect(el.textContent).not.toContain("1 in progress"); - expect(el.textContent).not.toContain("in flight"); + expect(el.textContent).toContain("Running now"); + expect(el.textContent).toContain("1 in progress"); + expect(el.textContent).toContain("Long haul"); + expect(el.textContent).toContain("in flight"); }); test("the elapsed label ticks forward while a run is live, not frozen at first render", async () => { diff --git a/apps/web/src/pages/insights-page.test.ts b/apps/web/src/pages/insights-page.test.ts index f346d0fb5..f0fb0168a 100644 --- a/apps/web/src/pages/insights-page.test.ts +++ b/apps/web/src/pages/insights-page.test.ts @@ -235,7 +235,7 @@ describe("isRunningNow", () => { ).toBe(false); }); - test("warm-keep: a running run past the fire window is not in flight", () => { + test("a live running run past the fire window is still in flight", () => { expect( isRunningNow( run({ @@ -246,6 +246,6 @@ describe("isRunningNow", () => { status: "running", }), ), - ).toBe(false); + ).toBe(true); }); }); diff --git a/apps/web/src/pages/insights-page.tsx b/apps/web/src/pages/insights-page.tsx index 1b1ec06d5..84ed8b977 100644 --- a/apps/web/src/pages/insights-page.tsx +++ b/apps/web/src/pages/insights-page.tsx @@ -468,8 +468,9 @@ function useTickingNow(enabled: boolean): number { /** A run actually in flight right now (`status: running | updating`) — * liveness is not a windowed property, so this filters the full run set, * never the range-filtered one. A persisted `endedAt` means the fire - * already finished, even if `status` still reads `running`. The fire - * window is last-resort for abandoned fires that never got that stamp. + * already finished, even if `status` still reads `running`. A live fire + * with no `endedAt` stays in flight however old it is — the running + * window is only for fires already known abandoned. */ export function isRunningNow( run: InsightsRun, diff --git a/apps/web/test/mission-control-page.test.tsx b/apps/web/test/mission-control-page.test.tsx index 3699d8ddc..3384e2fb6 100644 --- a/apps/web/test/mission-control-page.test.tsx +++ b/apps/web/test/mission-control-page.test.tsx @@ -96,7 +96,7 @@ describe("computeInFlightRows", () => { expect(rows).toEqual([]); }); - test("warm-keep: a running routine past the fire window is not in-flight", () => { + test("a live running routine past the fire window stays in-flight", () => { const rows = computeInFlightRows([ routine({ id: "stale", @@ -106,7 +106,7 @@ describe("computeInFlightRows", () => { ).toISOString(), }), ]); - expect(rows).toEqual([]); + expect(rows.map((row) => row.key)).toEqual(["routine:stale"]); }); }); diff --git a/apps/web/test/routine-activity.test.ts b/apps/web/test/routine-activity.test.ts index 56cea1374..eee605d1a 100644 --- a/apps/web/test/routine-activity.test.ts +++ b/apps/web/test/routine-activity.test.ts @@ -113,10 +113,9 @@ describe("listRoutineActivity", () => { expect(item?.status).not.toBe("running"); }); - // Warm-keep (CL-6681 / CL-6778): the fires feed still reports `running` - // because the delivery agent stays deployed. Past the fire window that - // row must not count as routine activity in flight. - test("warm-keep: a running fire past the window no longer reads as running", async () => { + // A live fire still in a tool loop can outlast the abandoned-fire + // window. Persist has not settled, so the row stays running. + test("a live running fire past the window still reads as running", async () => { stubTopLevelRunsFetch([ { ...runningFire, @@ -128,7 +127,7 @@ describe("listRoutineActivity", () => { const [item] = await listRoutineActivity("tnt_1"); - expect(item?.status).not.toBe("running"); + expect(item?.status).toBe("running"); }); // A directly-triggered deployment run is also a `feed=fires` row (it is diff --git a/apps/web/test/routines-page.test.tsx b/apps/web/test/routines-page.test.tsx index 62e306bff..989acb70f 100644 --- a/apps/web/test/routines-page.test.tsx +++ b/apps/web/test/routines-page.test.tsx @@ -223,7 +223,7 @@ describe("GlobalRoutinesList", () => { expect(markup).not.toContain(">running<"); }); - test("warm-keep (CL-6681): a fire whose 'running' status is stale reads as finished, not stuck Running now forever", () => { + test("a live fire whose 'running' status is older than the window still reads as Running now", () => { const markup = renderList([ row({ runs: [ @@ -236,8 +236,8 @@ describe("GlobalRoutinesList", () => { ], }), ]); - expect(markup).not.toContain("Running now"); - expect(markup).toContain("Finished"); + expect(markup).toContain("Running now"); + expect(markup).not.toContain("Finished"); }); test("a failing routine states its failure count in words, not only in colour", () => { diff --git a/packages/routines/src/health.test.ts b/packages/routines/src/health.test.ts index 40d2e44b9..120b9d205 100644 --- a/packages/routines/src/health.test.ts +++ b/packages/routines/src/health.test.ts @@ -41,17 +41,37 @@ describe("fireOutcomeStatus", () => { ); }); - test("warm-keep (CL-6681): a running fire past its window reads as completed", () => { + test("a live fire past the window still reads as running until persist settles", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; expect( fireOutcomeStatus(fire("r1", {}, { status: "running" }), staleNow), + ).toBe("running"); + }); + + test("an 8-day still-running live fire stays running", () => { + const eightDays = Date.parse(FIRE_CREATED_AT) + 8 * 24 * 60 * 60 * 1000; + expect( + fireOutcomeStatus(fire("r1", {}, { status: "running" }), eightDays), + ).toBe("running"); + }); + + test("abandoned: a running fire past its window reads as completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus( + fire("r1", { abandoned: true }, { status: "running" }), + staleNow, + ), ).toBe("completed"); }); - test("a running fire one millisecond inside its window still reads as running", () => { + test("abandoned: a running fire one millisecond inside its window still reads as running", () => { const stillInside = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS; expect( - fireOutcomeStatus(fire("r1", {}, { status: "running" }), stillInside), + fireOutcomeStatus( + fire("r1", { abandoned: true }, { status: "running" }), + stillInside, + ), ).toBe("running"); }); @@ -104,13 +124,23 @@ describe("runOutcomeStatus", () => { ).toBe("running"); }); - test("warm-keep: a top-level running status past its window reads as completed", () => { + test("a live top-level running status past the window still reads as running", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; expect( runOutcomeStatus( { createdAt: FIRE_CREATED_AT, status: "running" }, staleNow, ), + ).toBe("running"); + }); + + test("abandoned: a top-level running status past its window reads as completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running", abandoned: true }, + staleNow, + ), ).toBe("completed"); }); @@ -161,13 +191,23 @@ describe("cleanFireStreak", () => { ).toBe(1); }); - test("a stale running fire (warm-keep) counts as a success, not in-flight forever", () => { + test("a live running fire past the window stays in-flight and does not extend the streak", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; expect( cleanFireStreak( [fire("r2", {}, { status: "running" }), fire("r1")], staleNow, ), + ).toBe(1); + }); + + test("an abandoned stale running fire counts as a success, not in-flight forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + cleanFireStreak( + [fire("r2", { abandoned: true }, { status: "running" }), fire("r1")], + staleNow, + ), ).toBe(2); }); @@ -281,13 +321,24 @@ describe("routineHealth", () => { ).not.toBe("running"); }); - test("warm-keep (CL-6681): a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => { + test("a live latest run past the window still reports Running now", () => { const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; const health = routineHealth( healthy, [fire("r1", {}, { status: "running" })], staleNow, ); + expect(health.state).toBe("running"); + expect(health.label).toBe("Running now"); + }); + + test("abandoned: a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const health = routineHealth( + healthy, + [fire("r1", { abandoned: true }, { status: "running" })], + staleNow, + ); expect(health.state).toBe("ok"); expect(health.label).not.toBe("Running now"); }); diff --git a/packages/routines/src/health.ts b/packages/routines/src/health.ts index f54e9e51f..a122851ca 100644 --- a/packages/routines/src/health.ts +++ b/packages/routines/src/health.ts @@ -19,6 +19,12 @@ export type RoutineFire = { readonly createdAt: string; readonly error?: string | null; readonly run?: Record; + /** + * True when this fire is already known abandoned (sidecar dead, no + * in-flight turn). The running-window is only applied then — a live + * fire still doing work stays `running` however old it is. + */ + readonly abandoned?: boolean; }; /** The routine fields health depends on — every one of them already @@ -73,15 +79,13 @@ function statusOf(fire: { } /** - * How long a fire may credibly still be doing work before a lingering - * `running` status with no `endedAt` is read as an abandoned fire - * (warm-keep CL-6681 / CL-6778). A routine fire that finished is - * supposed to land `completed`/`failed`/`cancelled` plus `endedAt` via - * `markTerminal`; this window is the last-resort reading for a fire - * that never got that write — never the happy path. Past it, a - * still-`running` fire with no end stamp is presumed to have already - * delivered, so every surface badging its status reads it through - * `fireOutcomeStatus` rather than the raw column. + * How long an *abandoned* fire may linger as `running` with no `endedAt` + * before it is read as completed (warm-keep CL-6681 / CL-6778). A finished + * fire is supposed to land `completed`/`failed`/`cancelled` plus `endedAt` + * via `markTerminal`; this window is last-resort for a fire already known + * abandoned that never got that write. It is never applied to a live + * in-flight fire — a tool loop can outlast ten minutes, and persist has + * not settled yet. */ export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; @@ -89,14 +93,16 @@ export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; * A fire's status the way this build should show it: the raw * `run.status` for every non-running value; a `running` row that * already carries `endedAt` is finished immediately (the persist - * path); a `running` row with no `endedAt` older than - * `FIRE_RUNNING_WINDOW_MS` is an abandoned fire read as `completed`. + * path); a `running` row with no `endedAt` stays `running` until + * persist settles, unless the fire is already known abandoned — + * only then does `FIRE_RUNNING_WINDOW_MS` remap it to `completed`. * Every caller that needs a fire's displayed status goes through here, * never `fire.run?.status` directly. */ export function fireOutcomeStatus( fire: Pick & { readonly run?: Record; + readonly abandoned?: boolean; }, now: number, ): string | null { @@ -104,6 +110,7 @@ export function fireOutcomeStatus( if (status !== "running") return status; const endedAt = fire.run?.endedAt; if (typeof endedAt === "string" && endedAt !== "") return "completed"; + if (fire.abandoned !== true) return status; const startedAt = Date.parse(fire.createdAt); if (Number.isNaN(startedAt)) return status; return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running"; @@ -114,20 +121,23 @@ export function fireOutcomeStatus( * level (`workflow_run.status`), not nested on a routine fire's `run`. * Insights, Mission Control, and the shell activity feed all see that * shape. `endedAt` is the persist-path signal that the fire already - * finished; the running-window is only for abandoned fires that never - * got that stamp. + * finished. The running-window is only for fires already known + * abandoned (`abandoned: true`) that never got that stamp — a live + * in-flight run stays `running` however old it is. */ export function runOutcomeStatus( run: { readonly createdAt: string; readonly status: string; readonly endedAt?: string | null; + readonly abandoned?: boolean; }, now: number, ): string | null { return fireOutcomeStatus( { createdAt: run.createdAt, + ...(run.abandoned === true ? { abandoned: true } : {}), run: { status: run.status, endedAt: run.endedAt }, }, now, From 274f4ac7a994ca52e493772c32a9f66c0c0d1ece Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 17:21:17 -0700 Subject: [PATCH 5/6] Settle abandoned fires only from an explicit listing signal (CL-6778) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Omitting turns is not "no in-flight turn" — treating it that way remapped a live tool loop to completed at ten minutes. Abandoned only when the producer attached an empty turns array or hasInFlightTurn: false. The fires feed always attaches both after the query. --- apps/hub/src/routine-run-summary.ts | 4 + apps/web/src/agents-api.ts | 10 +- apps/web/src/insights-api.ts | 11 ++ apps/web/src/insights-stats.ts | 8 +- apps/web/src/pages/insights-page.tsx | 14 +- apps/web/src/pages/mission-control-page.tsx | 41 ++--- apps/web/src/pages/routine-detail-page.tsx | 7 +- apps/web/src/pages/routines-page.tsx | 9 +- apps/web/src/shell/routine-activity.ts | 26 +++- packages/routines/src/client.ts | 5 + packages/routines/src/health.test.ts | 143 ++++++++++++++++++ packages/routines/src/health.ts | 138 +++++++++++++++-- packages/run-scope/src/in-flight-turns.ts | 43 ++++++ packages/run-scope/src/index.ts | 2 + packages/run-scope/src/scope-routes.ts | 54 ++++--- .../test/scope-routes.drizzle.test.ts | 4 + 16 files changed, 455 insertions(+), 64 deletions(-) create mode 100644 packages/run-scope/src/in-flight-turns.ts diff --git a/apps/hub/src/routine-run-summary.ts b/apps/hub/src/routine-run-summary.ts index c064611e9..98f89e624 100644 --- a/apps/hub/src/routine-run-summary.ts +++ b/apps/hub/src/routine-run-summary.ts @@ -8,6 +8,7 @@ import { and, eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { workflowRun } from "@intx/db/schema"; import type { RunSummaryResolver } from "@corbits/routines"; +import { listingTurnsByRunId } from "@corbits/run-scope"; export function createHubRunSummaryResolver(db: DB["db"]): RunSummaryResolver { return { @@ -19,10 +20,13 @@ export function createHubRunSummaryResolver(db: DB["db"]): RunSummaryResolver { ), }); if (row === undefined) return undefined; + const turns = (await listingTurnsByRunId(db, [runId])).get(runId) ?? []; return { status: row.status, createdAt: row.createdAt.toISOString(), endedAt: row.endedAt?.toISOString() ?? null, + hasInFlightTurn: turns.length > 0, + turns, }; }, }; diff --git a/apps/web/src/agents-api.ts b/apps/web/src/agents-api.ts index 5cc0d76e9..4bcff2657 100644 --- a/apps/web/src/agents-api.ts +++ b/apps/web/src/agents-api.ts @@ -31,7 +31,15 @@ export type AgentInstance = typeof WorkflowRunResponse.infer; export type CatalogModel = typeof ModelResponse.infer; const RunFireResponse = WorkflowRunResponse.and( - type({ routineId: "string | null", routineName: "string | null" }), + type({ + routineId: "string | null", + routineName: "string | null", + "hasInFlightTurn?": "boolean", + "turns?": type({ + status: "string", + "endedAt?": "string | null", + }).array(), + }), ); /** A `feed=fires` row: every `AgentInstance` field plus the routine that * fired it (both `null` for a directly-triggered deployment with no diff --git a/apps/web/src/insights-api.ts b/apps/web/src/insights-api.ts index 35a954009..63cd69268 100644 --- a/apps/web/src/insights-api.ts +++ b/apps/web/src/insights-api.ts @@ -224,17 +224,28 @@ export function insightsScopePath(tenantId: string): string { return `/api/tenants/${tenantId}/insights/scope`; } +export const ListingTurnSchema = type({ + status: "string", + "endedAt?": "string | null", +}); + /** * `WorkflowRunResponse` plus the two fields only the `feed=fires` mode of * `/top-level-runs` reports (CL-6249): the routine that fired this run, * when it has one. Both are `null` for a run with no routine/task * parent — a directly launched workflow — so a caller falls back to * `definitionName` honestly instead of inventing a routine. + * `turns` / `hasInFlightTurn` are this build's listing of in-flight + * inference turns for the run (not Interchange fields) so a live + * tool-loop can stay running past the abandoned-fire window. Omitting + * them is not "no in-flight turn". */ export const InsightsRunSchema = WorkflowRunResponse.and( type({ routineId: "string | null", routineName: "string | null", + "hasInFlightTurn?": "boolean", + "turns?": ListingTurnSchema.array(), }), ); export type InsightsRun = typeof InsightsRunSchema.infer; diff --git a/apps/web/src/insights-stats.ts b/apps/web/src/insights-stats.ts index 78bce82ff..761ed5948 100644 --- a/apps/web/src/insights-stats.ts +++ b/apps/web/src/insights-stats.ts @@ -3,7 +3,10 @@ // existing endpoints. import { isWorkbenchHostDefinitionName } from "@corbits/chat/workbench-host-naming"; -import { runOutcomeStatus } from "@corbits/routines/client"; +import { + runOutcomeStatus, + withListingAbandoned, +} from "@corbits/routines/client"; import type { InsightsRun, RunTraceSpan } from "./insights-api"; import type { Routine } from "./routines-api"; @@ -167,7 +170,8 @@ export function computeInsightsStats( let stopped = 0; let deployed = 0; for (const run of purposeful) { - const outcome = runOutcomeStatus(run, now) ?? run.status; + const outcome = + runOutcomeStatus(withListingAbandoned(run, now), now) ?? run.status; switch (outcome) { case "running": case "updating": diff --git a/apps/web/src/pages/insights-page.tsx b/apps/web/src/pages/insights-page.tsx index 84ed8b977..6bc61583f 100644 --- a/apps/web/src/pages/insights-page.tsx +++ b/apps/web/src/pages/insights-page.tsx @@ -32,7 +32,11 @@ import { type TraceSpan, } from "@corbits/react-ui"; import { ChartBar } from "@corbits/icons"; -import { runOutcomeStatus, runStatusLabel } from "@corbits/routines/client"; +import { + runOutcomeStatus, + runStatusLabel, + withListingAbandoned, +} from "@corbits/routines/client"; import type * as React from "react"; import { useEffect, useMemo, useState } from "react"; @@ -469,19 +473,19 @@ function useTickingNow(enabled: boolean): number { * liveness is not a windowed property, so this filters the full run set, * never the range-filtered one. A persisted `endedAt` means the fire * already finished, even if `status` still reads `running`. A live fire - * with no `endedAt` stays in flight however old it is — the running - * window is only for fires already known abandoned. + * with an in-flight turn stays in flight however old it is; without an + * explicit no-in-flight signal, missing `turns` is not abandonment. */ export function isRunningNow( run: InsightsRun, now: number = Date.now(), ): boolean { - const outcome = runOutcomeStatus(run, now); + const outcome = runOutcomeStatus(withListingAbandoned(run, now), now); return outcome === "running" || outcome === "updating"; } function insightsRunStatus(run: InsightsRun, now: number = Date.now()): string { - return runOutcomeStatus(run, now) ?? run.status; + return runOutcomeStatus(withListingAbandoned(run, now), now) ?? run.status; } /** diff --git a/apps/web/src/pages/mission-control-page.tsx b/apps/web/src/pages/mission-control-page.tsx index 601325ba4..ef4a6f4c8 100644 --- a/apps/web/src/pages/mission-control-page.tsx +++ b/apps/web/src/pages/mission-control-page.tsx @@ -32,7 +32,11 @@ import { useQueryClient } from "@tanstack/react-query"; import { formatUsd } from "@corbits/insights/client"; import { CHAT_STRINGS, type Workbench } from "@corbits/chat-ui"; -import { runOutcomeStatus, runStatusLabel } from "@corbits/routines/client"; +import { + runOutcomeStatus, + runStatusLabel, + withListingAbandoned, +} from "@corbits/routines/client"; import { approveApproval, rejectApproval, useAPIQuery } from "../api"; import { useBench } from "../bench-context"; @@ -64,19 +68,27 @@ type InFlightRow = { readonly steps: string; }; +function listingFromRoutine(routine: RoutineActivityItem, now: number) { + return withListingAbandoned( + { + createdAt: routine.startedAt, + status: routine.status, + ...(routine.endedAt !== undefined ? { endedAt: routine.endedAt } : {}), + ...(routine.hasInFlightTurn !== undefined + ? { hasInFlightTurn: routine.hasInFlightTurn } + : {}), + ...(routine.turns !== undefined ? { turns: routine.turns } : {}), + }, + now, + ); +} + function routineInFlightRow( routine: RoutineActivityItem, now: number, ): InFlightRow { const status = - runOutcomeStatus( - { - createdAt: routine.startedAt, - status: routine.status, - ...(routine.endedAt !== undefined ? { endedAt: routine.endedAt } : {}), - }, - now, - ) ?? routine.status; + runOutcomeStatus(listingFromRoutine(routine, now), now) ?? routine.status; return { key: `routine:${routine.id}`, label: routine.name, @@ -97,16 +109,7 @@ export function computeInFlightRows( const rows = routines .filter( (routine) => - runOutcomeStatus( - { - createdAt: routine.startedAt, - status: routine.status, - ...(routine.endedAt !== undefined - ? { endedAt: routine.endedAt } - : {}), - }, - now, - ) === "running", + runOutcomeStatus(listingFromRoutine(routine, now), now) === "running", ) .map((routine) => routineInFlightRow(routine, now)); return rows.sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); diff --git a/apps/web/src/pages/routine-detail-page.tsx b/apps/web/src/pages/routine-detail-page.tsx index 161f6e5b0..3a9ea3895 100644 --- a/apps/web/src/pages/routine-detail-page.tsx +++ b/apps/web/src/pages/routine-detail-page.tsx @@ -46,6 +46,7 @@ import { routineHealth, routineScheduleSentence, timezoneForTrigger, + withListingAbandoned, } from "@corbits/routines/client"; import type { RoutineHealth } from "@corbits/routines/client"; @@ -352,7 +353,11 @@ export function RoutineDetailPage({ readonly onSaveSchedule: (expression: string) => Promise; readonly onEdit: () => void; }) { - const health = routineHealth(row.routine, row.runs, now); + const health = routineHealth( + row.routine, + row.runs.map((run) => withListingAbandoned(run, now)), + now, + ); const latestRunId = row.runs.find((run) => !fireNeverStarted(run.triggeredBy))?.runId ?? null; return ( diff --git a/apps/web/src/pages/routines-page.tsx b/apps/web/src/pages/routines-page.tsx index 8fd0d2f60..4b6e35edf 100644 --- a/apps/web/src/pages/routines-page.tsx +++ b/apps/web/src/pages/routines-page.tsx @@ -41,6 +41,7 @@ import { routineScheduleSentence, runStatusLabel, triggeredByLabel, + withListingAbandoned, } from "@corbits/routines/client"; import type { RoutineHealth } from "@corbits/routines/client"; @@ -189,7 +190,7 @@ export function RunStatusCell({ readonly run: RoutineRun; readonly now: number; }) { - const status = fireOutcomeStatus(run, now); + const status = fireOutcomeStatus(withListingAbandoned(run, now), now); if (status === null) { return —; } @@ -203,7 +204,11 @@ export function routineRowHealth( row: GlobalRoutineRow, now: number, ): RoutineHealth { - return routineHealth(row.routine, row.runs, now); + return routineHealth( + row.routine, + row.runs.map((run) => withListingAbandoned(run, now)), + now, + ); } /** "At 09:00, Monday through Friday (UTC)" — the schedule as a sentence, diff --git a/apps/web/src/shell/routine-activity.ts b/apps/web/src/shell/routine-activity.ts index 6756b9042..518706f84 100644 --- a/apps/web/src/shell/routine-activity.ts +++ b/apps/web/src/shell/routine-activity.ts @@ -10,7 +10,11 @@ // toward Mission Control's "Active runs" — exactly CL-6595's desync // between the Routines page's own "Running now" pill and Mission Control's // "0 / nothing running". -import { runOutcomeStatus } from "@corbits/routines/client"; +import { + runOutcomeStatus, + withListingAbandoned, +} from "@corbits/routines/client"; +import type { ListingTurn } from "@corbits/routines/client"; import { listRoutineRunFires } from "../agents-api"; import type { RunFire } from "../agents-api"; @@ -21,15 +25,33 @@ export type RoutineActivityItem = { readonly status: string; readonly startedAt: string; readonly endedAt?: string | null; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; }; function toRoutineActivityItem(run: RunFire, now: number): RoutineActivityItem { + const listing = withListingAbandoned( + { + createdAt: run.createdAt, + status: run.status, + ...(run.endedAt !== undefined ? { endedAt: run.endedAt } : {}), + ...(run.hasInFlightTurn !== undefined + ? { hasInFlightTurn: run.hasInFlightTurn } + : {}), + ...(run.turns !== undefined ? { turns: run.turns } : {}), + }, + now, + ); return { id: run.id, name: run.routineName ?? run.definitionName, - status: runOutcomeStatus(run, now) ?? run.status, + status: runOutcomeStatus(listing, now) ?? run.status, startedAt: run.createdAt, ...(run.endedAt !== undefined ? { endedAt: run.endedAt } : {}), + ...(run.hasInFlightTurn !== undefined + ? { hasInFlightTurn: run.hasInFlightTurn } + : {}), + ...(run.turns !== undefined ? { turns: run.turns } : {}), }; } diff --git a/packages/routines/src/client.ts b/packages/routines/src/client.ts index d84ce7556..20d7071c7 100644 --- a/packages/routines/src/client.ts +++ b/packages/routines/src/client.ts @@ -29,11 +29,16 @@ export { fireFailed, fireOutcomeStatus, lastFailedFire, + listingAbandoned, + listingHasInFlightTurn, medianFireDurationMs, routineHealth, runOutcomeStatus, + withListingAbandoned, } from "./health"; export type { + ListingRun, + ListingTurn, RoutineFire, RoutineHealth, RoutineHealthState, diff --git a/packages/routines/src/health.test.ts b/packages/routines/src/health.test.ts index 120b9d205..3fdf9bff3 100644 --- a/packages/routines/src/health.test.ts +++ b/packages/routines/src/health.test.ts @@ -5,6 +5,8 @@ import { FIRE_RUNNING_WINDOW_MS, fireOutcomeStatus, lastFailedFire, + listingAbandoned, + listingHasInFlightTurn, medianFireDurationMs, routineHealth, runOutcomeStatus, @@ -115,6 +117,26 @@ describe("fireOutcomeStatus", () => { ), ).toBe("failed"); }); + + test("listing-shaped empty turns past the window settles to completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus( + fire("r1", { turns: [] }, { status: "running" }), + staleNow, + ), + ).toBe("completed"); + }); + + test("listing-shaped hasInFlightTurn true past the window stays running", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus( + fire("r1", { hasInFlightTurn: true }, { status: "running" }), + staleNow, + ), + ).toBe("running"); + }); }); describe("runOutcomeStatus", () => { @@ -170,6 +192,127 @@ describe("runOutcomeStatus", () => { }); }); +describe("listingAbandoned", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const listing = { createdAt: FIRE_CREATED_AT }; + + test("omitted turns and hasInFlightTurn is not abandoned past the window", () => { + expect(listingAbandoned(listing, staleNow)).toBe(false); + expect(listingHasInFlightTurn(listing)).toBe(false); + }); + + test("empty turns after a real query past the window is abandoned", () => { + expect(listingAbandoned({ ...listing, turns: [] }, staleNow)).toBe(true); + }); + + test("hasInFlightTurn false past the window is abandoned", () => { + expect( + listingAbandoned({ ...listing, hasInFlightTurn: false }, staleNow), + ).toBe(true); + }); + + test("hasInFlightTurn true past the window is not abandoned", () => { + expect( + listingAbandoned({ ...listing, hasInFlightTurn: true }, staleNow), + ).toBe(false); + expect(listingHasInFlightTurn({ ...listing, hasInFlightTurn: true })).toBe( + true, + ); + }); + + test("a running turn past the window is not abandoned", () => { + expect( + listingAbandoned( + { ...listing, turns: [{ status: "running", endedAt: null }] }, + staleNow, + ), + ).toBe(false); + }); + + test("finished turns only past the window is abandoned", () => { + expect( + listingAbandoned( + { + ...listing, + turns: [{ status: "completed", endedAt: "2026-01-01T00:01:00.000Z" }], + }, + staleNow, + ), + ).toBe(true); + }); + + test("empty nested run.turns past the window is abandoned", () => { + expect(listingAbandoned({ ...listing, run: { turns: [] } }, staleNow)).toBe( + true, + ); + }); + + test("empty turns inside the window is not abandoned", () => { + expect(listingAbandoned({ ...listing, turns: [] }, NOW)).toBe(false); + }); +}); + +describe("runOutcomeStatus listing-shaped", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + + test("missing turns past the window stays running", () => { + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running" }, + staleNow, + ), + ).toBe("running"); + }); + + test("empty turns past the window settles to completed", () => { + expect( + runOutcomeStatus( + { createdAt: FIRE_CREATED_AT, status: "running", turns: [] }, + staleNow, + ), + ).toBe("completed"); + }); + + test("hasInFlightTurn false past the window settles to completed", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + hasInFlightTurn: false, + }, + staleNow, + ), + ).toBe("completed"); + }); + + test("hasInFlightTurn true past the window stays running", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + hasInFlightTurn: true, + }, + staleNow, + ), + ).toBe("running"); + }); + + test("a running listing turn past the window stays running", () => { + expect( + runOutcomeStatus( + { + createdAt: FIRE_CREATED_AT, + status: "running", + turns: [{ status: "running", endedAt: null }], + }, + staleNow, + ), + ).toBe("running"); + }); +}); + describe("cleanFireStreak", () => { test("counts successes from the newest fire and stops at the first failure", () => { expect( diff --git a/packages/routines/src/health.ts b/packages/routines/src/health.ts index a122851ca..ec0e53ae8 100644 --- a/packages/routines/src/health.ts +++ b/packages/routines/src/health.ts @@ -9,6 +9,12 @@ // routine's condition have to agree, and they can only agree if there is // one function that decides. +/** One turn on a listing-shaped payload — enough to tell in-flight from settled. */ +export type ListingTurn = { + readonly status: string; + readonly endedAt?: string | null; +}; + /** The per-fire history row this module reads — structurally the subset of * `./client`'s `RoutineRun` that says whether a fire worked and how long * it took. Declared here rather than imported so `./client` can re-export @@ -25,6 +31,25 @@ export type RoutineFire = { * fire still doing work stays `running` however old it is. */ readonly abandoned?: boolean; + /** + * True when the listing already resolved an in-flight turn for this + * fire. Equivalent to `turns` containing a running turn; either form + * keeps a live tool-loop `running` past `FIRE_RUNNING_WINDOW_MS`. + */ + readonly hasInFlightTurn?: boolean; + /** Listing-shaped turns for this fire, when the producer attached them. */ + readonly turns?: readonly ListingTurn[]; +}; + +/** A listing row (Insights fire, shell activity, reconstructed Mission + * Control row, or a routine fire) from which abandonment can be derived + * without a dedicated Interchange field. */ +export type ListingRun = { + readonly createdAt: string; + readonly abandoned?: boolean; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; + readonly run?: Record; }; /** The routine fields health depends on — every one of them already @@ -78,6 +103,21 @@ function statusOf(fire: { return typeof status === "string" ? status : null; } +function isInFlightTurn(turn: unknown): boolean { + if (typeof turn !== "object" || turn === null) return false; + const status = "status" in turn ? turn.status : undefined; + if (status !== "running") return false; + const endedAt = "endedAt" in turn ? turn.endedAt : undefined; + return endedAt === undefined || endedAt === null || endedAt === ""; +} + +function nestedTurns( + run: Record | undefined, +): readonly unknown[] | undefined { + const turns = run?.turns; + return Array.isArray(turns) ? turns : undefined; +} + /** * How long an *abandoned* fire may linger as `running` with no `endedAt` * before it is read as completed (warm-keep CL-6681 / CL-6778). A finished @@ -89,13 +129,79 @@ function statusOf(fire: { */ export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; +type InFlightSignal = "yes" | "no" | "unknown"; + +function runHasInFlightTurn(run: Record | undefined): boolean { + return run?.hasInFlightTurn === true; +} + +function runHasNoInFlightTurn( + run: Record | undefined, +): boolean { + return run?.hasInFlightTurn === false; +} + +/** + * Absent `turns` / `hasInFlightTurn` is unknown, not "no in-flight turn". + * Treating omit as empty reverts the live-tool-loop 10-minute false-complete. + */ +function listingInFlightSignal(listing: ListingRun): InFlightSignal { + if (listing.hasInFlightTurn === true || runHasInFlightTurn(listing.run)) { + return "yes"; + } + if (listing.turns?.some(isInFlightTurn) === true) return "yes"; + const nested = nestedTurns(listing.run); + if (nested?.some(isInFlightTurn) === true) return "yes"; + if (listing.turns !== undefined) return "no"; + if (nested !== undefined) return "no"; + if (listing.hasInFlightTurn === false || runHasNoInFlightTurn(listing.run)) { + return "no"; + } + return "unknown"; +} + +/** + * A listing row has an in-flight turn when the producer said so + * (`hasInFlightTurn`) or attached a running turn (top-level `turns` or + * nested on `run`). A finished turn (`endedAt` set, or status other than + * `running`) does not count. Omitted fields are unknown, not false. + */ +export function listingHasInFlightTurn(listing: ListingRun): boolean { + return listingInFlightSignal(listing) === "yes"; +} + +/** + * Abandoned only when the producer explicitly said there is no in-flight + * turn — an empty `turns` array after a real query, or `hasInFlightTurn: + * false` — and the fire is older than `FIRE_RUNNING_WINDOW_MS`. Omitting + * those fields is not a no; a live tool-loop listing that has not attached + * them must stay running. + */ +export function listingAbandoned(listing: ListingRun, now: number): boolean { + if (listingInFlightSignal(listing) !== "no") return false; + const startedAt = Date.parse(listing.createdAt); + if (Number.isNaN(startedAt)) return false; + return now - startedAt > FIRE_RUNNING_WINDOW_MS; +} + +/** Pass `abandoned: true` into outcome helpers when the listing is abandoned. */ +export function withListingAbandoned( + listing: T, + now: number, +): T { + if (!listingAbandoned(listing, now)) return listing; + return { ...listing, abandoned: true }; +} + /** * A fire's status the way this build should show it: the raw * `run.status` for every non-running value; a `running` row that * already carries `endedAt` is finished immediately (the persist - * path); a `running` row with no `endedAt` stays `running` until - * persist settles, unless the fire is already known abandoned — - * only then does `FIRE_RUNNING_WINDOW_MS` remap it to `completed`. + * path); a `running` row with an in-flight turn stays `running` + * however old it is; a `running` row whose listing explicitly says + * there is no in-flight turn is abandoned past `FIRE_RUNNING_WINDOW_MS` + * and remapped to `completed`. Omitting `turns`/`hasInFlightTurn` is + * not that signal — the fire stays `running`. * Every caller that needs a fire's displayed status goes through here, * never `fire.run?.status` directly. */ @@ -103,6 +209,8 @@ export function fireOutcomeStatus( fire: Pick & { readonly run?: Record; readonly abandoned?: boolean; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; }, now: number, ): string | null { @@ -110,7 +218,9 @@ export function fireOutcomeStatus( if (status !== "running") return status; const endedAt = fire.run?.endedAt; if (typeof endedAt === "string" && endedAt !== "") return "completed"; - if (fire.abandoned !== true) return status; + if (listingHasInFlightTurn(fire)) return status; + const abandoned = fire.abandoned === true || listingAbandoned(fire, now); + if (!abandoned) return status; const startedAt = Date.parse(fire.createdAt); if (Number.isNaN(startedAt)) return status; return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running"; @@ -121,9 +231,10 @@ export function fireOutcomeStatus( * level (`workflow_run.status`), not nested on a routine fire's `run`. * Insights, Mission Control, and the shell activity feed all see that * shape. `endedAt` is the persist-path signal that the fire already - * finished. The running-window is only for fires already known - * abandoned (`abandoned: true`) that never got that stamp — a live - * in-flight run stays `running` however old it is. + * finished. A listing-shaped payload with an in-flight turn stays + * `running` however old it is; one that explicitly says there is no + * in-flight turn is abandoned past `FIRE_RUNNING_WINDOW_MS` and + * remapped to `completed`. Omitting those fields is not a no. */ export function runOutcomeStatus( run: { @@ -131,14 +242,21 @@ export function runOutcomeStatus( readonly status: string; readonly endedAt?: string | null; readonly abandoned?: boolean; + readonly hasInFlightTurn?: boolean; + readonly turns?: readonly ListingTurn[]; }, now: number, ): string | null { + const listing = withListingAbandoned(run, now); return fireOutcomeStatus( { - createdAt: run.createdAt, - ...(run.abandoned === true ? { abandoned: true } : {}), - run: { status: run.status, endedAt: run.endedAt }, + createdAt: listing.createdAt, + ...(listing.abandoned === true ? { abandoned: true } : {}), + ...(listing.hasInFlightTurn !== undefined + ? { hasInFlightTurn: listing.hasInFlightTurn } + : {}), + ...(listing.turns !== undefined ? { turns: listing.turns } : {}), + run: { status: listing.status, endedAt: listing.endedAt }, }, now, ); diff --git a/packages/run-scope/src/in-flight-turns.ts b/packages/run-scope/src/in-flight-turns.ts new file mode 100644 index 000000000..d25c80647 --- /dev/null +++ b/packages/run-scope/src/in-flight-turns.ts @@ -0,0 +1,43 @@ +// In-flight inference turns for a batch of workflow runs. Listing +// surfaces (Insights `feed=fires`, routine run summaries) attach these +// so a live tool-loop older than the abandoned-fire window still reads +// as running, without adding a field to Interchange's WorkflowRunResponse. +import { and, eq, inArray } from "drizzle-orm"; +import type { DB } from "@intx/db"; +import { inferenceTurn } from "@intx/db/schema"; + +export type InFlightListingTurn = { + readonly status: "running"; + readonly endedAt: null; +}; + +/** + * Run ids in `runIds` that currently have at least one `inference_turn` + * still `running`. Every requested id is present after the query — an + * empty array means queried, none in flight, not "turns omitted". Empty + * input is an empty map — no round trip. + */ +export async function listingTurnsByRunId( + db: DB["db"], + runIds: readonly string[], +): Promise> { + if (runIds.length === 0) return new Map(); + const byRun = new Map(); + for (const id of runIds) byRun.set(id, []); + const rows = await db + .select({ runId: inferenceTurn.runId }) + .from(inferenceTurn) + .where( + and( + inArray(inferenceTurn.runId, [...runIds]), + eq(inferenceTurn.status, "running"), + ), + ); + const turn: InFlightListingTurn = { status: "running", endedAt: null }; + for (const row of rows) { + const existing = byRun.get(row.runId); + if (existing === undefined) byRun.set(row.runId, [turn]); + else existing.push(turn); + } + return byRun; +} diff --git a/packages/run-scope/src/index.ts b/packages/run-scope/src/index.ts index 7ded94637..64db5d0fb 100644 --- a/packages/run-scope/src/index.ts +++ b/packages/run-scope/src/index.ts @@ -6,3 +6,5 @@ export { type ResolveRoutineFires, type RoutineFireInfo, } from "./scope-routes"; +export { listingTurnsByRunId } from "./in-flight-turns"; +export type { InFlightListingTurn } from "./in-flight-turns"; diff --git a/packages/run-scope/src/scope-routes.ts b/packages/run-scope/src/scope-routes.ts index 204a57a16..a0708132f 100644 --- a/packages/run-scope/src/scope-routes.ts +++ b/packages/run-scope/src/scope-routes.ts @@ -34,6 +34,7 @@ import { workflowDefinition, workflowRun } from "@intx/db/schema"; import type { WorkflowRunStatus } from "@intx/types"; import { foldedRun } from "@corbits/folded-runs"; import { makeErrorEnvelope } from "@corbits/error-sink"; +import { listingTurnsByRunId } from "./in-flight-turns"; /** A routine fire's parent, resolved by `resolveRoutineFires` below. */ export type RoutineFireInfo = { @@ -255,28 +256,37 @@ export async function listTopLevelRunFires( ? await resolveRoutineFires(foldedIds) : new Map(); - return rows - .filter((row) => row.foldedMarkerId === null || routineFires.has(row.id)) - .map((row) => { - const fire = routineFires.get(row.id); - return { - id: row.id, - definitionId: row.definitionId, - definitionName: row.definitionName, - tenantId: row.tenantId, - // Guarded by `isNotNull(workflowRun.address)` above. - address: row.address as string, - status: toViewStatus(row.status), - publicKey: row.publicKey, - kernelId: row.kernelId, - sidecarId: row.sidecarId, - createdAt: toTimestamp(row.createdAt), - updatedAt: toTimestamp(row.endedAt ?? row.createdAt), - endedAt: row.endedAt ? toTimestamp(row.endedAt) : null, - routineId: fire?.routineId ?? null, - routineName: fire?.routineName ?? null, - }; - }); + const kept = rows.filter( + (row) => row.foldedMarkerId === null || routineFires.has(row.id), + ); + const turnsByRun = await listingTurnsByRunId( + db, + kept.map((row) => row.id), + ); + + return kept.map((row) => { + const fire = routineFires.get(row.id); + const turns = turnsByRun.get(row.id) ?? []; + return { + id: row.id, + definitionId: row.definitionId, + definitionName: row.definitionName, + tenantId: row.tenantId, + // Guarded by `isNotNull(workflowRun.address)` above. + address: row.address as string, + status: toViewStatus(row.status), + publicKey: row.publicKey, + kernelId: row.kernelId, + sidecarId: row.sidecarId, + createdAt: toTimestamp(row.createdAt), + updatedAt: toTimestamp(row.endedAt ?? row.createdAt), + endedAt: row.endedAt ? toTimestamp(row.endedAt) : null, + routineId: fire?.routineId ?? null, + routineName: fire?.routineName ?? null, + hasInFlightTurn: turns.length > 0, + turns, + }; + }); } /** diff --git a/packages/run-scope/test/scope-routes.drizzle.test.ts b/packages/run-scope/test/scope-routes.drizzle.test.ts index 32007aa25..2eef71782 100644 --- a/packages/run-scope/test/scope-routes.drizzle.test.ts +++ b/packages/run-scope/test/scope-routes.drizzle.test.ts @@ -333,6 +333,8 @@ describeIfDb("listTopLevelRunFires", () => { definitionName: "workbench-digest", routineId: "rtn_pulse_check", routineName: "Pulse check", + hasInFlightTurn: false, + turns: [], }); const direct = rows.find((row) => row.id === "run_direct_deployment1"); @@ -340,6 +342,8 @@ describeIfDb("listTopLevelRunFires", () => { definitionName: "researcher", routineId: null, routineName: null, + hasInFlightTurn: false, + turns: [], }); } finally { await close(); From a5e774715725e43f4a32cc6b51a1de174ee841c3 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 22:00:56 -0700 Subject: [PATCH 6/6] hub: report routine fire settle failures through reportError (CL-6778) --- apps/hub/src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 31a04010a..23dd81e89 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -870,6 +870,10 @@ export async function createHub(config: HubConfig) { hadReply: turn.hadReply, }).catch((err: unknown) => { log.warn`Failed to settle routine fire for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`; + reportError(err, { + operation: "routine_fire_settle_from_turn", + agentId: agentAddress, + }); }); }, // Per-turn usage, emitted once when the collector finalizes a turn.