Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 41 additions & 2 deletions apps/hub/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
createSidecarAllocationStore,
createSignalCorrelationStore,
createWorkflowRunDispatchStore,
createWorkflowRunStore,
listVisibleOfferings,
resolveCredentialByName,
resolveCredentialRequirement,
Expand Down Expand Up @@ -118,6 +119,7 @@ import type { FinalizedTurnToolCall } from "@corbits/turn-artifacts";
import { decodedOrNull } from "@corbits/url-path";
import {
createCryptoProviderCache,
findFoldedRunByAddress,
lookupFoldedRunReconnectKey,
} from "@corbits/folded-runs";
import { createTopLevelRunRoutes } from "@corbits/run-scope";
Expand Down Expand Up @@ -209,6 +211,7 @@ import {
resolveLaunchableDefinition,
routine as routineTable,
routineRun as routineRunTable,
settleRoutineFireFromTurn,
} from "@corbits/routines";
import {
createSidecarProvisioner as createE2BSidecarProvisioner,
Expand Down Expand Up @@ -817,6 +820,31 @@ export async function createHub(config: HubConfig) {
store: insightsUsage.store,
generateId: () => generateId("inferenceTurn"),
});
// CL-6778: a routine fire's folded delivery agent stays deployed after
// it replies, so `workflow_run.status` would linger on `running` unless
// something stamps it terminal. `onTurnFinalized` is that stamp —
// `markTerminal` + `endedAt`, gated on a `routine_run` row so a
// workbench host or invited agent is never settled by accident.
const workflowRunStore = createWorkflowRunStore(db);
const routineFireSettlePort = {
async lookupRunByAddress(address: string) {
const row = await findFoldedRunByAddress(db, address);
return row === undefined ? undefined : { id: row.id };
},
async isRoutineFire(runId: string) {
const rows = await db
.select({ runId: routineRunTable.runId })
.from(routineRunTable)
.where(eq(routineRunTable.runId, runId))
.limit(1);
return rows.length > 0;
},
markTerminal: (
runId: string,
status: "completed" | "failed" | "cancelled",
endedAt: Date,
) => workflowRunStore.markTerminal(runId, status, endedAt),
};
// CL-6257: per-message-run stage latency (message-received →
// reactor.start → inference.start → first-token → reply-posted). The
// vendored event collector never persists the events this reads (see
Expand All @@ -835,8 +863,19 @@ export async function createHub(config: HubConfig) {
// surviving loss loud (error-level cause, counted) instead of a
// swallowed WRN.
db: withTurnPartPersistGuard(withTurnPartWriteDefaults(db)),
onTurnFinalized: (agentAddress, turn) =>
artifactDeliveryHandlerRef.current?.(agentAddress, turn),
onTurnFinalized: (agentAddress, turn) => {
artifactDeliveryHandlerRef.current?.(agentAddress, turn);
void settleRoutineFireFromTurn(routineFireSettlePort, agentAddress, {
status: turn.status,
hadReply: turn.hadReply,
}).catch((err: unknown) => {
log.warn`Failed to settle routine fire for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
reportError(err, {
operation: "routine_fire_settle_from_turn",
agentId: agentAddress,
});
});
},
// Per-turn usage, emitted once when the collector finalizes a turn.
onUsage: (_agentAddress, usage) => {
void usageSink
Expand Down
4 changes: 4 additions & 0 deletions apps/hub/src/routine-run-summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { and, eq } from "drizzle-orm";
import type { DB } from "@intx/db";
import { workflowRun } from "@intx/db/schema";
import type { RunSummaryResolver } from "@corbits/routines";
import { listingTurnsByRunId } from "@corbits/run-scope";

export function createHubRunSummaryResolver(db: DB["db"]): RunSummaryResolver {
return {
Expand All @@ -19,10 +20,13 @@ export function createHubRunSummaryResolver(db: DB["db"]): RunSummaryResolver {
),
});
if (row === undefined) return undefined;
const turns = (await listingTurnsByRunId(db, [runId])).get(runId) ?? [];
return {
status: row.status,
createdAt: row.createdAt.toISOString(),
endedAt: row.endedAt?.toISOString() ?? null,
hasInFlightTurn: turns.length > 0,
turns,
};
},
};
Expand Down
10 changes: 9 additions & 1 deletion apps/web/src/agents-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,15 @@ export type AgentInstance = typeof WorkflowRunResponse.infer;
export type CatalogModel = typeof ModelResponse.infer;

const RunFireResponse = WorkflowRunResponse.and(
type({ routineId: "string | null", routineName: "string | null" }),
type({
routineId: "string | null",
routineName: "string | null",
"hasInFlightTurn?": "boolean",
"turns?": type({
status: "string",
"endedAt?": "string | null",
}).array(),
}),
);
/** A `feed=fires` row: every `AgentInstance` field plus the routine that
* fired it (both `null` for a directly-triggered deployment with no
Expand Down
11 changes: 11 additions & 0 deletions apps/web/src/insights-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,17 +224,28 @@ export function insightsScopePath(tenantId: string): string {
return `/api/tenants/${tenantId}/insights/scope`;
}

export const ListingTurnSchema = type({
status: "string",
"endedAt?": "string | null",
});

/**
* `WorkflowRunResponse` plus the two fields only the `feed=fires` mode of
* `/top-level-runs` reports (CL-6249): the routine that fired this run,
* when it has one. Both are `null` for a run with no routine/task
* parent — a directly launched workflow — so a caller falls back to
* `definitionName` honestly instead of inventing a routine.
* `turns` / `hasInFlightTurn` are this build's listing of in-flight
* inference turns for the run (not Interchange fields) so a live
* tool-loop can stay running past the abandoned-fire window. Omitting
* them is not "no in-flight turn".
*/
export const InsightsRunSchema = WorkflowRunResponse.and(
type({
routineId: "string | null",
routineName: "string | null",
"hasInFlightTurn?": "boolean",
"turns?": ListingTurnSchema.array(),
}),
);
export type InsightsRun = typeof InsightsRunSchema.infer;
Expand Down
36 changes: 36 additions & 0 deletions apps/web/src/insights-stats.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import { describe, expect, test } from "bun:test";

import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client";

import {
computeInsightsStats,
computeTraceStats,
filterRunsByCreatedAt,
groupRunsByDefinition,
INSIGHTS_RECENT_LIMIT,
purposeRunsForInsights,
runDisplayName,
} from "./insights-stats";
Expand Down Expand Up @@ -94,6 +97,8 @@ describe("computeInsightsStats", () => {
routine({ id: "r1", enabled: true }),
routine({ id: "r2", enabled: false }),
],
INSIGHTS_RECENT_LIMIT,
Date.parse("2026-01-03T00:01:00.000Z"),
);

expect(stats.totalRuns).toBe(3);
Expand All @@ -105,6 +110,37 @@ describe("computeInsightsStats", () => {
expect(stats.recentRuns.map((r) => r.id)).toEqual(["1", "2", "3"]);
});

test("a live running run past the fire window is still counted as running", () => {
const stats = computeInsightsStats(
[
run({
id: "stale",
status: "running",
createdAt: new Date(
Date.now() - FIRE_RUNNING_WINDOW_MS - 1,
).toISOString(),
}),
],
[],
);
expect(stats.running).toBe(1);
});

test("endedAt drops a just-finished running run from the running count immediately", () => {
const stats = computeInsightsStats(
[
run({
id: "just-finished",
status: "running",
createdAt: new Date().toISOString(),
endedAt: new Date().toISOString(),
}),
],
[],
);
expect(stats.running).toBe(0);
});

test("limits recent runs", () => {
const runs = Array.from({ length: 5 }, (_, i) =>
run({
Expand Down
10 changes: 9 additions & 1 deletion apps/web/src/insights-stats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
// existing endpoints.

import { isWorkbenchHostDefinitionName } from "@corbits/chat/workbench-host-naming";
import {
runOutcomeStatus,
withListingAbandoned,
} from "@corbits/routines/client";

import type { InsightsRun, RunTraceSpan } from "./insights-api";
import type { Routine } from "./routines-api";
Expand Down Expand Up @@ -158,14 +162,17 @@ export function computeInsightsStats(
runs: readonly InsightsRun[],
routines: readonly Routine[],
recentLimit: number = INSIGHTS_RECENT_LIMIT,
now: number = Date.now(),
): InsightsStats {
const purposeful = purposeRunsForInsights(runs);
let running = 0;
let errored = 0;
let stopped = 0;
let deployed = 0;
for (const run of purposeful) {
switch (run.status) {
const outcome =
runOutcomeStatus(withListingAbandoned(run, now), now) ?? run.status;
switch (outcome) {
case "running":
case "updating":
running += 1;
Expand All @@ -174,6 +181,7 @@ export function computeInsightsStats(
errored += 1;
break;
case "stopped":
case "completed":
stopped += 1;
break;
case "deployed":
Expand Down
3 changes: 2 additions & 1 deletion apps/web/src/pages/insights-page-render.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,8 @@ describe("InsightsPage 'Running now' strip", () => {
// Liveness is not a windowed property: a run that started long before the
// 7-day window and is still running must not disappear from the strip or
// read 0 in the "Running now" KPI just because its start time falls
// outside `range`.
// outside `range`. Persist has not settled (`endedAt` absent), so the
// fire is live — not remapped to completed by the abandoned-fire window.
test("a run started 8 days ago that is still running stays in the strip and the KPI", () => {
const eightDaysAgo = new Date(
Date.now() - 8 * 24 * 60 * 60 * 1000,
Expand Down
71 changes: 71 additions & 0 deletions apps/web/src/pages/insights-page.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
// so this landing scope never takes a workbench override anymore.
import { describe, expect, test } from "bun:test";
import type { DayActivity } from "@corbits/insights/client";
import { FIRE_RUNNING_WINDOW_MS } from "@corbits/routines/client";

import {
costPerDay,
elapsedLabel,
isRunningNow,
resolveInsightsScope,
runsPerDay,
} from "./insights-page";
Expand Down Expand Up @@ -178,3 +180,72 @@ describe("elapsedLabel", () => {
expect(elapsedLabel("not-a-date", Date.now())).toBe("—");
});
});

describe("isRunningNow", () => {
test("a running run still inside the fire window is in flight", () => {
expect(
isRunningNow(
run({
id: "fresh",
createdAt: new Date().toISOString(),
status: "running",
}),
),
).toBe(true);
});

test("updating is in flight", () => {
expect(
isRunningNow(
run({
id: "updating",
createdAt: new Date().toISOString(),
status: "updating",
}),
),
).toBe(true);
});

test("a stopped run is not in flight", () => {
expect(
isRunningNow(
run({
id: "done",
createdAt: new Date().toISOString(),
status: "stopped",
}),
),
).toBe(false);
});

// Warm-keep (CL-6681 / CL-6778): a routine's delivery agent stays deployed
// after it replies, so workflow_run.status lingers on `running`. Past the
// fire window that is not an in-flight job — Insights must not keep it in
// "Running now" forever.
test("endedAt drops in-flight immediately, even while status is still running inside the window", () => {
expect(
isRunningNow(
run({
id: "just-finished",
createdAt: new Date().toISOString(),
status: "running",
endedAt: new Date().toISOString(),
}),
),
).toBe(false);
});

test("a live running run past the fire window is still in flight", () => {
expect(
isRunningNow(
run({
id: "stale",
createdAt: new Date(
Date.now() - FIRE_RUNNING_WINDOW_MS - 1,
).toISOString(),
status: "running",
}),
),
).toBe(true);
});
});
Loading
Loading