Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 72 additions & 1 deletion apps/web/src/pages/insights-page-render.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ import { createRoot } from "react-dom/client";
import type { Root } from "react-dom/client";

import type { APIQuery } from "@corbits/api-query";
import { EMPTY_OVERALL_USAGE } from "@corbits/insights/client";
import {
EMPTY_OVERALL_USAGE,
EMPTY_TOKEN_TOTALS,
} from "@corbits/insights/client";

import { InsightsPage, useInsightsWindow } from "./insights-page";
import { BenchContext } from "../bench-context";
Expand Down Expand Up @@ -192,3 +195,71 @@ describe("InsightsPage 'Running now' strip", () => {
expect(el.textContent).not.toBe(before);
});
});

describe("CL-6659 unreported local-model tokens", () => {
test("turns with no token counts label the exclusion instead of Cost $0.00 / 0/0", () => {
container = document.createElement("div");
document.body.appendChild(container);
root = createRoot(container);
const range = {
from: "2026-01-08T18:00:00.000Z",
to: "2026-01-15T18:00:00.000Z",
};
act(() => {
root?.render(
<NavigationProvider navigate={() => {}}>
<BenchContext.Provider value={benchState}>
<InsightsPage
path="/insights"
summary={readyEmpty({
turns: 2,
tokens: { ...EMPTY_TOKEN_TOTALS },
costUsd: null,
byModel: [
{
model: "qwen3:latest",
turns: 2,
tokens: { ...EMPTY_TOKEN_TOTALS },
costUsd: null,
},
],
})}
activity={readyEmpty([
{
day: "2026-01-15",
turns: 2,
tokens: 0,
byModel: [
{ model: "qwen3:latest", tokens: 0, costUsd: null },
],
},
])}
byTool={readyEmpty([])}
runs={readyEmpty({ data: [], nextCursor: null })}
routines={readyEmpty([])}
workbenches={readyEmpty({
items: [
{
tenantId: "tnt_bench_a",
name: "Support bench",
turns: 2,
tokens: { ...EMPTY_TOKEN_TOTALS },
costUsd: null,
},
],
})}
latency={{ kind: "loading" }}
range={range}
scope={null}
resolveWorkbenchIdForTenant={() => null}
scopeLabel="All workbenches"
/>
</BenchContext.Provider>
</NavigationProvider>,
);
});
expect(container?.textContent).toContain("Token counts were not reported");
expect(container?.textContent).toContain("qwen3:latest");
expect(container?.textContent).not.toMatch(/Cost\s*\$0\.00/);
});
});
69 changes: 53 additions & 16 deletions apps/web/src/pages/insights-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import {
formatUsd,
INSIGHTS_WINDOW_DAYS,
modelsWithMissingRates,
modelsWithUnreportedTokens,
tokensLabel,
topModelsByCost,
type DayActivity,
Expand Down Expand Up @@ -388,13 +389,17 @@ function runOutcomeData(stats: {
];
}

/** Tokens-by-model series for the tokens-over-time chart — token volume
* (unlike cost) is always a known number for a recorded turn, so this
* stays honest for every model without a null-rate caveat. Capped to the
* top models by cost (the models that matter most to the spend story),
* same as `TimeSeriesChart`'s own "≤5 series" rule. */
/** Tokens-by-model series for the tokens-over-time chart. Token volume is
* a known number only when the adapter reported it; unreported turns are
* excluded from this series rather than plotted as a silent zero. Capped
* to the top models by cost (the models that matter most to the spend
* story), same as `TimeSeriesChart`'s own "≤5 series" rule. */
function tokensOverTimeSeries(days: readonly DayActivity[]) {
const models = topModelsByCost(days);
const models = topModelsByCost(days).filter((model) =>
days.some(
(d) => (d.byModel.find((m) => m.model === model)?.tokens ?? 0) > 0,
),
);
return models.map((model) => ({
label: model,
values: days.map(
Expand Down Expand Up @@ -541,13 +546,25 @@ function ModelCostTable({
<TableRow key={m.model}>
<TableCell title={m.model}>{m.model}</TableCell>
<TableCell>
{m.costUsd === null && m.tokens.total > 0
{m.costUsd === null || (m.tokens.total === 0 && m.turns > 0)
? "—"
: formatUsd(m.costUsd)}
</TableCell>
<TableCell>{formatCount(m.tokens.input)}</TableCell>
<TableCell>{formatCount(m.tokens.cacheRead)}</TableCell>
<TableCell>{formatCount(m.tokens.output)}</TableCell>
<TableCell>
{m.tokens.total === 0 && m.turns > 0
? "—"
: formatCount(m.tokens.input)}
</TableCell>
<TableCell>
{m.tokens.total === 0 && m.turns > 0
? "—"
: formatCount(m.tokens.cacheRead)}
</TableCell>
<TableCell>
{m.tokens.total === 0 && m.turns > 0
? "—"
: formatCount(m.tokens.output)}
</TableCell>
</TableRow>
))}
</TableBody>
Expand Down Expand Up @@ -688,6 +705,8 @@ function InsightsLanding({
const mosaicParts = tokenParts(usage);
const hitRate = cacheHitRate(usage);
const missingRates = modelsWithMissingRates(usage);
const unreportedTokens = modelsWithUnreportedTokens(usage);
const tokensUnreported = usage.turns > 0 && usage.tokens.total === 0;
const activityDays = activitySeriesForWindow(activity ?? [], range);
const activityWindowEmpty = recentActivityDays(activityDays).every(
(d) => d.turns === 0,
Expand All @@ -704,16 +723,25 @@ function InsightsLanding({
const tokensSparkline = activityDays.map((d) => d.tokens);
const runsSparkline = runsPerDay(purposeRuns, activityDays);
const costSparkline =
missingRates.length === 0 ? costPerDay(activityDays) : undefined;
missingRates.length === 0 && unreportedTokens.length === 0
? costPerDay(activityDays)
: undefined;

return (
<div className="insights-layout">
<StatGrid columns={4}>
{noUsageInWindow ? null : (
<InsightsStat
label="Cost"
value={tileValue(formatUsd(usage.costUsd), loading)}
detail={`${formatCount(usage.tokens.total)} tokens`}
value={tileValue(
tokensUnreported ? "—" : formatUsd(usage.costUsd),
loading,
)}
detail={
tokensUnreported
? "Token counts were not reported"
: `${formatCount(usage.tokens.total)} tokens`
}
loading={loading}
sparklineLabel="Cost per day this week"
{...(costSparkline === undefined
Expand All @@ -735,12 +763,14 @@ function InsightsLanding({
<InsightsStat
label="Tokens in / out"
value={tileValue(
`${formatCount(usage.tokens.input)} / ${formatCount(usage.tokens.output)}`,
tokensUnreported
? "—"
: `${formatCount(usage.tokens.input)} / ${formatCount(usage.tokens.output)}`,
loading,
)}
detail="input / output"
detail={tokensUnreported ? "not reported" : "input / output"}
loading={loading}
sparklineValues={tokensSparkline}
{...(tokensUnreported ? {} : { sparklineValues: tokensSparkline })}
sparklineLabel="Tokens per day this week"
/>
)}
Expand Down Expand Up @@ -811,6 +841,13 @@ function InsightsLanding({
<p className="insights-note">No usage recorded yet in this window.</p>
) : null}

{unreportedTokens.length > 0 ? (
<p className="insights-note">
Token counts were not reported for: {unreportedTokens.join(", ")}.
Those turns do not contribute a fabricated cost or token total.
</p>
) : null}

{missingRates.length > 0 ? (
<p className="insights-note">
Rates unknown for: {missingRates.join(", ")}. Those turns do not
Expand Down
54 changes: 54 additions & 0 deletions packages/insights/src/client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import {
formatCount,
formatUsd,
INSIGHTS_WINDOW_DAYS,
modelsWithMissingRates,
modelsWithUnreportedTokens,
tokensLabel,
topModelsByCost,
usageChromeLabel,
Expand Down Expand Up @@ -165,3 +167,55 @@ describe("CL-6877 usage chrome labels", () => {
expect(usageChromeLabel(EMPTY_OVERALL_USAGE)).not.toContain("0 tok");
});
});

describe("CL-6659 unreported tokens vs missing rates", () => {
const zeroTokens = {
input: 0,
cacheRead: 0,
cacheWrite: 0,
output: 0,
thinking: 0,
total: 0,
};

test("modelsWithUnreportedTokens names models that recorded turns but no token counts", () => {
expect(
modelsWithUnreportedTokens({
turns: 2,
tokens: zeroTokens,
costUsd: null,
byModel: [
{
model: "qwen3:latest",
turns: 2,
tokens: zeroTokens,
costUsd: null,
},
],
}),
).toEqual(["qwen3:latest"]);
expect(modelsWithUnreportedTokens(EMPTY_OVERALL_USAGE)).toEqual([]);
});

test("modelsWithMissingRates still requires tokens so unreported turns are not labeled as unknown rates", () => {
expect(
modelsWithMissingRates({
turns: 1,
tokens: zeroTokens,
costUsd: null,
byModel: [
{
model: "qwen3:latest",
turns: 1,
tokens: zeroTokens,
costUsd: null,
},
],
}),
).toEqual([]);
});

test("usageChromeLabel is an em-dash when cost is unknown and tokens were not reported", () => {
expect(usageChromeLabel({ costUsd: null, tokens: zeroTokens })).toBe("—");
});
});
10 changes: 10 additions & 0 deletions packages/insights/src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ export function usageChromeLabel(usage: {
}): string {
const cost = formatUsd(usage.costUsd);
const tok = tokensLabel(usage.tokens);
if (usage.costUsd === null && tok === undefined) return "—";
return tok === undefined ? cost : `${cost} · ${tok}`;
}

Expand All @@ -230,3 +231,12 @@ export function modelsWithMissingRates(usage: OverallUsage): readonly string[] {
.filter((m) => m.costUsd === null && m.tokens.total > 0)
.map((m) => m.model);
}

/** Models that recorded turns but no token counts — not the same as $0 spend. */
export function modelsWithUnreportedTokens(
usage: OverallUsage,
): readonly string[] {
return usage.byModel
.filter((m) => m.turns > 0 && m.tokens.total === 0)
.map((m) => m.model);
}
1 change: 1 addition & 0 deletions packages/insights/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ export {
summarizeLatency,
summarizeUsage,
summarizeUsageByTenant,
teamSpaceWorkbenchRows,
type DayActivity,
type LatencyStageStat,
type LatencySummary,
Expand Down
Loading
Loading