diff --git a/apps/web/src/pages/insights-page-render.test.tsx b/apps/web/src/pages/insights-page-render.test.tsx index 4be3a9fa3..f04881ab7 100644 --- a/apps/web/src/pages/insights-page-render.test.tsx +++ b/apps/web/src/pages/insights-page-render.test.tsx @@ -11,7 +11,10 @@ import { createRoot } from "react-dom/client"; import type { Root } from "react-dom/client"; import type { APIQuery } from "@corbits/api-query"; -import { EMPTY_OVERALL_USAGE } from "@corbits/insights/client"; +import { + EMPTY_OVERALL_USAGE, + EMPTY_TOKEN_TOTALS, +} from "@corbits/insights/client"; import { InsightsPage, useInsightsWindow } from "./insights-page"; import { BenchContext } from "../bench-context"; @@ -192,3 +195,71 @@ describe("InsightsPage 'Running now' strip", () => { expect(el.textContent).not.toBe(before); }); }); + +describe("CL-6659 unreported local-model tokens", () => { + test("turns with no token counts label the exclusion instead of Cost $0.00 / 0/0", () => { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + const range = { + from: "2026-01-08T18:00:00.000Z", + to: "2026-01-15T18:00:00.000Z", + }; + act(() => { + root?.render( + {}}> + + null} + scopeLabel="All workbenches" + /> + + , + ); + }); + expect(container?.textContent).toContain("Token counts were not reported"); + expect(container?.textContent).toContain("qwen3:latest"); + expect(container?.textContent).not.toMatch(/Cost\s*\$0\.00/); + }); +}); diff --git a/apps/web/src/pages/insights-page.tsx b/apps/web/src/pages/insights-page.tsx index b646ea638..369074c43 100644 --- a/apps/web/src/pages/insights-page.tsx +++ b/apps/web/src/pages/insights-page.tsx @@ -46,6 +46,7 @@ import { formatUsd, INSIGHTS_WINDOW_DAYS, modelsWithMissingRates, + modelsWithUnreportedTokens, tokensLabel, topModelsByCost, type DayActivity, @@ -388,13 +389,17 @@ function runOutcomeData(stats: { ]; } -/** Tokens-by-model series for the tokens-over-time chart — token volume - * (unlike cost) is always a known number for a recorded turn, so this - * stays honest for every model without a null-rate caveat. Capped to the - * top models by cost (the models that matter most to the spend story), - * same as `TimeSeriesChart`'s own "≤5 series" rule. */ +/** Tokens-by-model series for the tokens-over-time chart. Token volume is + * a known number only when the adapter reported it; unreported turns are + * excluded from this series rather than plotted as a silent zero. Capped + * to the top models by cost (the models that matter most to the spend + * story), same as `TimeSeriesChart`'s own "≤5 series" rule. */ function tokensOverTimeSeries(days: readonly DayActivity[]) { - const models = topModelsByCost(days); + const models = topModelsByCost(days).filter((model) => + days.some( + (d) => (d.byModel.find((m) => m.model === model)?.tokens ?? 0) > 0, + ), + ); return models.map((model) => ({ label: model, values: days.map( @@ -541,13 +546,25 @@ function ModelCostTable({ {m.model} - {m.costUsd === null && m.tokens.total > 0 + {m.costUsd === null || (m.tokens.total === 0 && m.turns > 0) ? "—" : formatUsd(m.costUsd)} - {formatCount(m.tokens.input)} - {formatCount(m.tokens.cacheRead)} - {formatCount(m.tokens.output)} + + {m.tokens.total === 0 && m.turns > 0 + ? "—" + : formatCount(m.tokens.input)} + + + {m.tokens.total === 0 && m.turns > 0 + ? "—" + : formatCount(m.tokens.cacheRead)} + + + {m.tokens.total === 0 && m.turns > 0 + ? "—" + : formatCount(m.tokens.output)} + ))} @@ -688,6 +705,8 @@ function InsightsLanding({ const mosaicParts = tokenParts(usage); const hitRate = cacheHitRate(usage); const missingRates = modelsWithMissingRates(usage); + const unreportedTokens = modelsWithUnreportedTokens(usage); + const tokensUnreported = usage.turns > 0 && usage.tokens.total === 0; const activityDays = activitySeriesForWindow(activity ?? [], range); const activityWindowEmpty = recentActivityDays(activityDays).every( (d) => d.turns === 0, @@ -704,7 +723,9 @@ function InsightsLanding({ const tokensSparkline = activityDays.map((d) => d.tokens); const runsSparkline = runsPerDay(purposeRuns, activityDays); const costSparkline = - missingRates.length === 0 ? costPerDay(activityDays) : undefined; + missingRates.length === 0 && unreportedTokens.length === 0 + ? costPerDay(activityDays) + : undefined; return (
@@ -712,8 +733,15 @@ function InsightsLanding({ {noUsageInWindow ? null : ( )} @@ -811,6 +841,13 @@ function InsightsLanding({

No usage recorded yet in this window.

) : null} + {unreportedTokens.length > 0 ? ( +

+ Token counts were not reported for: {unreportedTokens.join(", ")}. + Those turns do not contribute a fabricated cost or token total. +

+ ) : null} + {missingRates.length > 0 ? (

Rates unknown for: {missingRates.join(", ")}. Those turns do not diff --git a/packages/insights/src/client.test.ts b/packages/insights/src/client.test.ts index 425e91a62..c0671bca8 100644 --- a/packages/insights/src/client.test.ts +++ b/packages/insights/src/client.test.ts @@ -7,6 +7,8 @@ import { formatCount, formatUsd, INSIGHTS_WINDOW_DAYS, + modelsWithMissingRates, + modelsWithUnreportedTokens, tokensLabel, topModelsByCost, usageChromeLabel, @@ -165,3 +167,55 @@ describe("CL-6877 usage chrome labels", () => { expect(usageChromeLabel(EMPTY_OVERALL_USAGE)).not.toContain("0 tok"); }); }); + +describe("CL-6659 unreported tokens vs missing rates", () => { + const zeroTokens = { + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + thinking: 0, + total: 0, + }; + + test("modelsWithUnreportedTokens names models that recorded turns but no token counts", () => { + expect( + modelsWithUnreportedTokens({ + turns: 2, + tokens: zeroTokens, + costUsd: null, + byModel: [ + { + model: "qwen3:latest", + turns: 2, + tokens: zeroTokens, + costUsd: null, + }, + ], + }), + ).toEqual(["qwen3:latest"]); + expect(modelsWithUnreportedTokens(EMPTY_OVERALL_USAGE)).toEqual([]); + }); + + test("modelsWithMissingRates still requires tokens so unreported turns are not labeled as unknown rates", () => { + expect( + modelsWithMissingRates({ + turns: 1, + tokens: zeroTokens, + costUsd: null, + byModel: [ + { + model: "qwen3:latest", + turns: 1, + tokens: zeroTokens, + costUsd: null, + }, + ], + }), + ).toEqual([]); + }); + + test("usageChromeLabel is an em-dash when cost is unknown and tokens were not reported", () => { + expect(usageChromeLabel({ costUsd: null, tokens: zeroTokens })).toBe("—"); + }); +}); diff --git a/packages/insights/src/client.ts b/packages/insights/src/client.ts index d830c10c2..1da3eeacb 100644 --- a/packages/insights/src/client.ts +++ b/packages/insights/src/client.ts @@ -221,6 +221,7 @@ export function usageChromeLabel(usage: { }): string { const cost = formatUsd(usage.costUsd); const tok = tokensLabel(usage.tokens); + if (usage.costUsd === null && tok === undefined) return "—"; return tok === undefined ? cost : `${cost} · ${tok}`; } @@ -230,3 +231,12 @@ export function modelsWithMissingRates(usage: OverallUsage): readonly string[] { .filter((m) => m.costUsd === null && m.tokens.total > 0) .map((m) => m.model); } + +/** Models that recorded turns but no token counts — not the same as $0 spend. */ +export function modelsWithUnreportedTokens( + usage: OverallUsage, +): readonly string[] { + return usage.byModel + .filter((m) => m.turns > 0 && m.tokens.total === 0) + .map((m) => m.model); +} diff --git a/packages/insights/src/index.ts b/packages/insights/src/index.ts index 852dee5d1..c136cb708 100644 --- a/packages/insights/src/index.ts +++ b/packages/insights/src/index.ts @@ -54,6 +54,7 @@ export { summarizeLatency, summarizeUsage, summarizeUsageByTenant, + teamSpaceWorkbenchRows, type DayActivity, type LatencyStageStat, type LatencySummary, diff --git a/packages/insights/src/queries.test.ts b/packages/insights/src/queries.test.ts index e8016aa42..4ee8ed719 100644 --- a/packages/insights/src/queries.test.ts +++ b/packages/insights/src/queries.test.ts @@ -6,6 +6,7 @@ import { emptyOverallUsageSummary, summarizeUsage, summarizeUsageByTenant, + teamSpaceWorkbenchRows, } from "./queries"; describe("summarizeUsage", () => { @@ -432,3 +433,128 @@ describe("summarizeUsageByTenant", () => { expect(summedCost).toBe(aggregate.costUsd ?? 0); }); }); + +describe("CL-6659 unreported tokens and team-space breakdowns", () => { + const zeroTokens = { + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + thinking: 0, + }; + + test("a recorded turn with no token counts is null cost, not $0.00", async () => { + const store = createMemoryUsageStore(); + await store.insertUsage({ + id: "u1", + tenantId: "tenant-acme", + sessionId: "s1", + turnId: "t1", + provider: "ollama", + model: "qwen3:latest", + tokens: zeroTokens, + }); + + const summary = await summarizeUsage(store, ["tenant-acme"]); + expect(summary.turns).toBe(1); + expect(summary.tokens.total).toBe(0); + expect(summary.costUsd).toBeNull(); + expect(summary.byModel[0]?.costUsd).toBeNull(); + }); + + test("ollama turns with real tokens and a FREE catalog row cost $0, not null", async () => { + const store = createMemoryUsageStore(); + await store.insertUsage({ + id: "u1", + tenantId: "tenant-acme", + sessionId: "s1", + turnId: "t1", + provider: "ollama", + model: "qwen3.8:27b", + tokens: { + input: 100, + cacheRead: 0, + cacheWrite: 0, + output: 50, + thinking: 0, + }, + }); + + const summary = await summarizeUsage(store, ["tenant-acme"]); + expect(summary.turns).toBe(1); + expect(summary.tokens.total).toBe(150); + expect(summary.costUsd).toBe(0); + }); + + test("team-space parent with recorded turns stays in the workbench breakdown", () => { + const rows = [ + { + tenantId: "team-root", + turns: 3, + tokens: { + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + thinking: 0, + total: 0, + }, + costUsd: null, + }, + { + tenantId: "workbench-a", + turns: 0, + tokens: { + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + thinking: 0, + total: 0, + }, + costUsd: 0, + }, + ]; + const kept = teamSpaceWorkbenchRows(rows, { + tenantId: "team-root", + isTeamSpace: true, + }); + expect(kept.map((r) => r.tenantId)).toEqual(["team-root", "workbench-a"]); + }); + + test("empty team-space parent is still dropped as a duplicate of All workbenches", () => { + const rows = [ + { + tenantId: "team-root", + turns: 0, + tokens: { + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + thinking: 0, + total: 0, + }, + costUsd: 0, + }, + { + tenantId: "workbench-a", + turns: 2, + tokens: { + input: 10, + cacheRead: 0, + cacheWrite: 0, + output: 5, + thinking: 0, + total: 15, + }, + costUsd: null, + }, + ]; + const kept = teamSpaceWorkbenchRows(rows, { + tenantId: "team-root", + isTeamSpace: true, + }); + expect(kept.map((r) => r.tenantId)).toEqual(["workbench-a"]); + }); +}); diff --git a/packages/insights/src/queries.ts b/packages/insights/src/queries.ts index 13bf1cd0b..fc0a7fcbe 100644 --- a/packages/insights/src/queries.ts +++ b/packages/insights/src/queries.ts @@ -93,6 +93,10 @@ export function emptyOverallUsageSummary(): OverallUsageSummary { function turnCostUsd(row: UsageTurnRecord): number | null { if (row.reportedCostUsd !== null) return row.reportedCostUsd; + // A recorded turn with no token counts is not $0 spend — we did not + // observe usage, so cost is unknown (CL-6659). Catalog FREE rates still + // apply once tokens are present. + if (totalTokens(row.tokens) === 0) return null; return costUsd({ provider: row.provider ?? "", model: row.model, @@ -209,10 +213,27 @@ export async function summarizeUsageByTenant( }); } +/** + * Team-space `/workbenches` rows: drop an empty parent (it is a duplicate + * of the "All workbenches" landing, CL-6368). Keep the parent when it + * recorded turns of its own — otherwise those turns vanish from the + * breakdown (CL-6659). + */ +export function teamSpaceWorkbenchRows< + T extends { readonly tenantId: string; readonly turns: number }, +>( + rows: readonly T[], + opts: { readonly tenantId: string; readonly isTeamSpace: boolean }, +): readonly T[] { + if (!opts.isTeamSpace) return rows; + return rows.filter((row) => row.tenantId !== opts.tenantId || row.turns > 0); +} + /** * Activity histogram by UTC day across a tenant scope (see summarizeUsage - * for what `tenantIds` means). Token totals are always known for recorded - * turns; pre-sink history simply does not appear. + * for what `tenantIds` means). Token totals are known when the adapter + * reported them; a recorded turn with every class at zero is an honest + * absence, not $0.00. Pre-sink history simply does not appear. */ export async function activityByDay( store: UsageStore, diff --git a/packages/insights/src/routes.ts b/packages/insights/src/routes.ts index a5b86c265..08cd71efe 100644 --- a/packages/insights/src/routes.ts +++ b/packages/insights/src/routes.ts @@ -23,6 +23,7 @@ import { summarizeLatency, summarizeUsage, summarizeUsageByTenant, + teamSpaceWorkbenchRows, type RunTraceReader, type ToolCallReader, } from "./queries"; @@ -210,12 +211,11 @@ export function createInsightsRoutes( * seeing the scope's sum. Calling it for a leaf workbench (no * descendants) returns that one workbench's own row. * - * A `parentId === null` requested tenant is never itself a workbench — - * it is the account root, the container real workbenches (each its own - * child tenant, CL-6089) live under. Its own row would just be a - * zero-usage rollup-of-itself duplicate of the "All workbenches" - * landing this chart already sits on, so it is dropped rather than - * listed alongside its children (CL-6368). + * A `parentId === null` requested tenant is the account root, the + * container real workbenches live under (CL-6089). An empty parent row + * is a duplicate of the "All workbenches" landing and is dropped + * (CL-6368). A parent that recorded turns of its own stays in the + * breakdown so those turns are not silently excluded (CL-6659). */ app.get( "/workbenches", @@ -229,9 +229,10 @@ export function createInsightsRoutes( summarizeUsageByTenant(deps.store, scope, range), tenantNames(deps.db, tenant.id, tenant.name, scope), ]); - const isTeamSpace = tenant.parentId === null; - const items = rows - .filter((row) => !isTeamSpace || row.tenantId !== tenant.id) + const items = teamSpaceWorkbenchRows(rows, { + tenantId: tenant.id, + isTeamSpace: tenant.parentId === null, + }) .map((row) => ({ tenantId: row.tenantId, name: names.get(row.tenantId) ?? row.tenantId, diff --git a/packages/ollama-adapter/src/adapter.test.ts b/packages/ollama-adapter/src/adapter.test.ts index 45ca88d53..b25dc73c5 100644 --- a/packages/ollama-adapter/src/adapter.test.ts +++ b/packages/ollama-adapter/src/adapter.test.ts @@ -74,7 +74,9 @@ describe("createOllamaAdapter", () => { const inner = createOpenAIAdapter(source); const wrappedBuilt = wrapped.buildRequest(messages, "gpt-oss:20b", options); const innerBuilt = inner.buildRequest(messages, "gpt-oss:20b", options); - expect(bodyOf(wrappedBuilt)).toEqual(bodyOf(innerBuilt)); + const wrappedBody = bodyOf(wrappedBuilt); + const { stream_options: _streamOptions, ...rest } = wrappedBody; + expect(rest).toEqual(bodyOf(innerBuilt)); expect(wrappedBuilt.url).toBe(innerBuilt.url); expect(wrappedBuilt.headers).toEqual(innerBuilt.headers); }); @@ -180,4 +182,57 @@ describe("createOllamaAdapter", () => { query: "this person", }); }); + + test("every request asks the OpenAI-compat endpoint for a terminal usage chunk", () => { + const wrapped = createOllamaAdapter(source, undefined); + const body = bodyOf(wrapped.buildRequest(messages, "gpt-oss:20b", options)); + expect(body["stream_options"]).toEqual({ include_usage: true }); + }); + + test("parseResponse maps native prompt_eval_count/eval_count onto inference.usage", () => { + const wrapped = createOllamaAdapter(source, undefined); + const events = wrapped.parseResponse( + JSON.stringify({ + choices: [{ delta: {}, finish_reason: "stop" }], + prompt_eval_count: 42, + eval_count: 17, + }), + ); + const usage = events.filter((event) => event.type === "inference.usage"); + expect(usage).toHaveLength(1); + expect(usage[0]?.data).toEqual({ + usage: { + input: 42, + output: 17, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }, + source, + }); + }); + + test("parseResponse does not double-emit usage when the OpenAI-shaped object is already present", () => { + const wrapped = createOllamaAdapter(source, undefined); + const events = wrapped.parseResponse( + JSON.stringify({ + choices: [{ delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 4 }, + prompt_eval_count: 99, + eval_count: 99, + }), + ); + const usage = events.filter((event) => event.type === "inference.usage"); + expect(usage).toHaveLength(1); + expect(usage[0]?.data).toEqual({ + usage: { + input: 10, + output: 4, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }, + source, + }); + }); }); diff --git a/packages/ollama-adapter/src/adapter.ts b/packages/ollama-adapter/src/adapter.ts index 4eb2520dc..bead8d88d 100644 --- a/packages/ollama-adapter/src/adapter.ts +++ b/packages/ollama-adapter/src/adapter.ts @@ -18,7 +18,11 @@ import type { BuiltRequest, ProviderAdapter, } from "@intx/inference"; -import type { LastCycleSource } from "@intx/types/runtime"; +import type { + InferenceEvent, + LastCycleSource, + TokenUsage, +} from "@intx/types/runtime"; import { parseOllamaAdapterConfig, @@ -38,20 +42,18 @@ type OllamaChatBody = { max_tokens?: number; max_completion_tokens?: number; reasoning_effort?: string; + stream_options?: { include_usage: boolean }; }; function applyOverride( built: BuiltRequest, override: OllamaAdapterOverride, ): BuiltRequest { - if ( - override.numCtx === undefined && - override.maxOutputTokens === undefined && - override.reasoningEffort === undefined - ) { - return built; - } const body = JSON.parse(built.body) as OllamaChatBody; + // Without include_usage, Ollama's OpenAI-compat stream often ends with no + // usage object; the harness then synthesizes zero token counts that Insights + // used to display as Cost $0.00 / 0/0 (CL-6659). + body.stream_options = { include_usage: true }; if (override.numCtx !== undefined) { body.options = { ...body.options, num_ctx: override.numCtx }; } @@ -71,6 +73,81 @@ function applyOverride( return { ...built, body: JSON.stringify(body) }; } +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function asNonNegativeInt(value: unknown): number | null { + return typeof value === "number" && Number.isInteger(value) && value >= 0 + ? value + : null; +} + +function openaiShapedUsage(value: unknown): TokenUsage | null { + if (!isPlainObject(value)) return null; + const input = asNonNegativeInt(value["prompt_tokens"]); + const output = asNonNegativeInt(value["completion_tokens"]); + if (input === null && output === null) return null; + const details = value["prompt_tokens_details"]; + const completionDetails = value["completion_tokens_details"]; + const cacheRead = isPlainObject(details) + ? asNonNegativeInt(details["cached_tokens"]) + : null; + const thinking = isPlainObject(completionDetails) + ? asNonNegativeInt(completionDetails["reasoning_tokens"]) + : null; + return { + input: input ?? 0, + output: output ?? 0, + cacheRead: cacheRead ?? 0, + cacheWrite: 0, + thinking: thinking ?? 0, + }; +} + +function ollamaTokenUsageFromChunk(raw: string): TokenUsage | null { + try { + const parsed: unknown = JSON.parse(raw); + if (!isPlainObject(parsed)) return null; + const fromUsage = openaiShapedUsage(parsed["usage"]); + if (fromUsage !== null) return fromUsage; + const input = asNonNegativeInt(parsed["prompt_eval_count"]); + const output = asNonNegativeInt(parsed["eval_count"]); + if (input === null && output === null) return null; + return { + input: input ?? 0, + output: output ?? 0, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }; + } catch { + // report-error-ignore: a non-JSON SSE chunk is the common case, not a + // failure — this function is "usage if present", never an error path. + return null; + } +} + +function withOllamaUsage( + events: readonly InferenceEvent[], + raw: string, + source: LastCycleSource, +): InferenceEvent[] { + if (events.some((event) => event.type === "inference.usage")) { + return [...events]; + } + const usage = ollamaTokenUsageFromChunk(raw); + if (usage === null) return [...events]; + return [ + ...events, + { + type: "inference.usage", + seq: 0, + data: { usage, source }, + }, + ]; +} + /** * `AdapterFactory` for the `ollama` provider key, the named export a * `SIDECAR_ADAPTER_MANIFEST` entry points at. `quirks` is this package's @@ -104,19 +181,30 @@ export const createOllamaAdapter: AdapterFactory = ( ); }, parseResponse: (sseData) => - reclassifyInlineToolJsonEvents( - reclassifyThinkingEvents( - inner.parseResponse(sseData), - streamThinkState, + withOllamaUsage( + reclassifyInlineToolJsonEvents( + reclassifyThinkingEvents( + inner.parseResponse(sseData), + streamThinkState, + ), + streamInlineState, + { flush: responseChunkIsTerminal(sseData) }, ), - streamInlineState, - { flush: responseChunkIsTerminal(sseData) }, + sseData, + source, ), parseJSONResponse: (body) => - reclassifyInlineToolJsonEvents( - reclassifyThinkingEvents(inner.parseJSONResponse(body), jsonThinkState), - jsonInlineState, - { flush: true }, + withOllamaUsage( + reclassifyInlineToolJsonEvents( + reclassifyThinkingEvents( + inner.parseJSONResponse(body), + jsonThinkState, + ), + jsonInlineState, + { flush: true }, + ), + body, + source, ), }; };