From 42c7cb52650917bf5cf92f273910e52741998df7 Mon Sep 17 00:00:00 2001 From: zverev Date: Mon, 5 Oct 2026 23:40:01 +0300 Subject: [PATCH 1/2] feat(pricing): check official display pricing drift --- .agents/skills/refresh-model-catalog/SKILL.md | 10 +- .github/scripts/check-commandcode-pricing.ts | 819 ++++++++++++++++ .github/workflows/model-metadata.yml | 22 + CHANGELOG.md | 1 + README.md | 2 + package.json | 6 +- tests/fixtures/commandcode-pricing-page.html | 7 + tests/test-pricing-check.ts | 877 ++++++++++++++++++ 8 files changed, 1741 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/check-commandcode-pricing.ts create mode 100644 tests/fixtures/commandcode-pricing-page.html create mode 100644 tests/test-pricing-check.ts diff --git a/.agents/skills/refresh-model-catalog/SKILL.md b/.agents/skills/refresh-model-catalog/SKILL.md index b5997a7..233cdc0 100644 --- a/.agents/skills/refresh-model-catalog/SKILL.md +++ b/.agents/skills/refresh-model-catalog/SKILL.md @@ -36,7 +36,15 @@ Never add efforts to the generated file by hand. Manual effort policy for reason ### 3. Update display pricing (manual review) -Fetch and compare against `src/pricing.ts`: +Run the read-only checker before touching prices: + +```sh +npm run check:commandcode-pricing +``` + +It compares the live Provider API catalog and the official pricing page against `MODEL_COSTS` and writes nothing. Investigate every reported drift and promotion by hand; never update `MODEL_COSTS` or `PRICING_LAST_VERIFIED` just to make the checker pass. + +Then fetch and compare against `src/pricing.ts`: - Add entries for new models and remove entries for retired models. Missing models silently display zero cost, so `MODEL_COSTS` must cover the full catalog. - The pricing page's "Cache Read"/"Cache Write" columns map to `cacheRead`/`cacheWrite`; a "—" column means `0`. diff --git a/.github/scripts/check-commandcode-pricing.ts b/.github/scripts/check-commandcode-pricing.ts new file mode 100644 index 0000000..766b217 --- /dev/null +++ b/.github/scripts/check-commandcode-pricing.ts @@ -0,0 +1,819 @@ +import { resolve } from "node:path" +import { pathToFileURL } from "node:url" + +import { commandCodeCostRatesAt } from "../../src/cost.ts" +import { + DEFAULT_MODELS_TIMEOUT_MS, + DEFAULT_MODELS_URL, + fetchCommandCodeModels, +} from "../../src/models.ts" +import { + MODEL_COSTS, + PRICING_LAST_VERIFIED, + PRICING_SOURCE_URL, + type CommandCodeModelCost, + type CommandCodeModelCostRates, + type CommandCodeModelCostTier, +} from "../../src/pricing.ts" + +export interface PricingPageRow { + id: string + cost: CommandCodeModelCost + timeOfDay?: { + offPeak: CommandCodeModelCostRates + peak: CommandCodeModelCostRates + windows: string + effective: string + } + deal?: { + id: string + discountPercent: number + free: boolean + expires?: string + endsWhen?: string + } +} + +export interface PricingCheckIssue { + kind: + | "missing-local-price" + | "missing-page-row" + | "ambiguous-page-row" + | "changed-cost" + | "time-policy" + | "expired-deal" + modelId: string + detail: string +} + +export interface PricingCheckResult { + checkedModelCount: number + issues: readonly PricingCheckIssue[] + unusedPageIds: readonly string[] + // Promotions are visible even when there is no price drift. + deals: readonly { modelId: string; deal: NonNullable }[] +} + +/** + * Explicit page-row aliases keyed by the exact Provider API model id. The page + * mostly uses basenames, so only the models whose page id cannot be derived + * from the API id need an entry here. + */ +const PRICING_PAGE_ID_ALIASES: Readonly> = { + "claude-haiku-4-5-20251001": "claude-haiku-4-5", + "Qwen/Qwen3.6-Max-Preview": "qwen-3.6-max", + "nvidia/nemotron-3-ultra-550b-a55b": "nemotron-3-ultra", +} + +/** DeepSeek V4 documents peak pricing in these two weekday windows. */ +const SUPPORTED_TIME_WINDOWS = "01–04 & 06–10 UTC, Mon–Fri" + +/** First integer hour of the fixed probe week (2026-10-05 is a Monday). */ +const PROBE_WEEK_START_MS = Date.UTC(2026, 9, 5, 0, 0, 0) +const PROBE_HOURS = 168 +const RATE_TOLERANCE = 1e-12 + +const SCRIPT_TAG_RE = /]*>([\s\S]*?)<\/script>/gi +const NEXT_PUSH_RE = /^self\.__next_f\.push\(([\s\S]*)\);?$/ +const FLIGHT_RECORD_RE = /^[0-9a-fA-F]+:([\s\S]*)$/ +const CALENDAR_DATE_RE = /^(\d{4})-(\d{2})-(\d{2})$/ +const ISO_TIMESTAMP_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?Z$/ +const SINGLE_CONTEXT_RE = /^≤\s*(\d+)K$/ +const ABOVE_CONTEXT_RE = /^>\s*(\d+)K$/ + +interface NormalizedCost { + input: number + output: number + cacheRead: number + cacheWrite: number + tiers: readonly CommandCodeModelCostTier[] +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function requireRecord(value: unknown, label: string): Record { + if (!isRecord(value)) throw new Error(`Expected ${label} to be an object`) + return value +} + +function requireNonEmptyString(value: unknown, label: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`Expected ${label} to be a non-empty string`) + } + return value +} + +function requireFiniteNonNegative(value: unknown, label: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Expected ${label} to be a finite non-negative number`) + } + return value +} + +function requirePositiveBand(value: string, label: string): number { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`Expected ${label} to be a positive integer band`) + } + return parsed * 1000 +} + +function requireIsoTimestamp(value: unknown, label: string): string { + const text = requireNonEmptyString(value, label) + const match = ISO_TIMESTAMP_RE.exec(text) + if (!match) throw new Error(`Expected ${label} to be a UTC ISO timestamp with Z`) + const [, year, month, day, hour, minute, second, fraction] = match + const date = new Date( + Date.UTC( + Number(year), + Number(month) - 1, + Number(day), + Number(hour), + Number(minute), + Number(second), + Number((fraction ?? "0").slice(0, 3).padEnd(3, "0")), + ), + ) + const valid = + date.getUTCFullYear() === Number(year) && + date.getUTCMonth() === Number(month) - 1 && + date.getUTCDate() === Number(day) && + date.getUTCHours() === Number(hour) && + date.getUTCMinutes() === Number(minute) && + date.getUTCSeconds() === Number(second) + if (!valid) throw new Error(`Expected ${label} to be a valid UTC ISO timestamp with Z`) + return text +} + +function requireExpires(value: unknown, label: string): string { + const text = requireNonEmptyString(value, label) + const calendar = CALENDAR_DATE_RE.exec(text) + if (calendar) { + const [, year, month, day] = calendar + const date = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day))) + const valid = + date.getUTCFullYear() === Number(year) && + date.getUTCMonth() === Number(month) - 1 && + date.getUTCDate() === Number(day) + if (!valid) throw new Error(`Expected ${label} to be a valid calendar date`) + return text + } + return requireIsoTimestamp(text, label) +} + +function parseRates(value: unknown, label: string): CommandCodeModelCostRates { + const record = requireRecord(value, `${label} rates`) + return { + input: requireFiniteNonNegative(record.input, `${label} input`), + output: requireFiniteNonNegative(record.output, `${label} output`), + cacheRead: + record.cacheRead === undefined + ? 0 + : requireFiniteNonNegative(record.cacheRead, `${label} cacheRead`), + cacheWrite: + record.cacheWrite === undefined + ? 0 + : requireFiniteNonNegative(record.cacheWrite, `${label} cacheWrite`), + } +} + +function ratesEqual(left: CommandCodeModelCostRates, right: CommandCodeModelCostRates): boolean { + return ( + left.input === right.input && + left.output === right.output && + left.cacheRead === right.cacheRead && + left.cacheWrite === right.cacheWrite + ) +} + +function ratesClose(left: CommandCodeModelCostRates, right: CommandCodeModelCostRates): boolean { + return ( + Math.abs(left.input - right.input) <= RATE_TOLERANCE && + Math.abs(left.output - right.output) <= RATE_TOLERANCE && + Math.abs(left.cacheRead - right.cacheRead) <= RATE_TOLERANCE && + Math.abs(left.cacheWrite - right.cacheWrite) <= RATE_TOLERANCE + ) +} + +function parseTiers( + value: unknown, + label: string, +): { base: CommandCodeModelCostRates; tiers: CommandCodeModelCostTier[] } { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`Expected ${label} to have a non-empty tiers array`) + } + const parsed = value.map((tier, index) => { + const record = requireRecord(tier, `${label} tier ${index}`) + return { + context: record.context, + rates: parseRates(record.rates, `${label} tier ${index}`), + } + }) + + if (parsed.length === 1) { + const context = parsed[0]!.context + if (context !== undefined) { + const match = SINGLE_CONTEXT_RE.exec(String(context)) + if (!match) throw new Error(`Unsupported context band for ${label}: ${String(context)}`) + requirePositiveBand(match[1]!, `${label} band ${context}`) + } + return { base: parsed[0]!.rates, tiers: [] } + } + + const upperBounds: number[] = [] + for (let index = 0; index < parsed.length; index += 1) { + const context = parsed[index]!.context + if (typeof context !== "string") { + throw new Error(`Expected ${label} tier ${index} to have a context band`) + } + const isLast = index === parsed.length - 1 + if (isLast) { + const match = ABOVE_CONTEXT_RE.exec(context) + if (!match) throw new Error(`Expected the last ${label} tier to be a '> NK' band: ${context}`) + const above = Number(match[1]) * 1000 + if (!Number.isInteger(above) || above <= 0) { + throw new Error(`Expected ${label} band ${context} to be a positive integer`) + } + if (upperBounds[upperBounds.length - 1] !== above) { + throw new Error(`Expected the last ${label} tier '${context}' to match the previous band`) + } + } else { + const match = SINGLE_CONTEXT_RE.exec(context) + if (!match) { + throw new Error(`Expected ${label} tier ${index} to be a '≤ NK' band: ${context}`) + } + const bound = requirePositiveBand(match[1]!, `${label} band ${context}`) + if (upperBounds.length > 0 && bound <= upperBounds[upperBounds.length - 1]!) { + throw new Error(`Expected ${label} context bands to strictly increase`) + } + upperBounds.push(bound) + } + } + + const base = parsed[0]!.rates + const tiers: CommandCodeModelCostTier[] = [] + let previous = base + for (let index = 1; index < parsed.length; index += 1) { + const rates = parsed[index]!.rates + if (ratesEqual(rates, previous)) continue + tiers.push({ inputTokensAbove: upperBounds[index - 1]!, ...rates }) + previous = rates + } + return { base, tiers } +} + +function parseRow(value: unknown, index: number): PricingPageRow { + const record = requireRecord(value, `pricing row ${index}`) + const id = requireNonEmptyString(record.id, `pricing row ${index} id`) + const label = `pricing row ${id}` + const { base, tiers } = parseTiers(record.tiers, label) + const cost: CommandCodeModelCost = { ...base, ...(tiers.length > 0 ? { tiers } : {}) } + + let timeOfDay: PricingPageRow["timeOfDay"] + if (record.timeOfDay !== undefined) { + const parsed = requireRecord(record.timeOfDay, `${label} timeOfDay`) + const offPeak = parseRates(parsed.offPeak, `${label} offPeak`) + const peak = parseRates(parsed.peak, `${label} peak`) + const windows = requireNonEmptyString(parsed.windows, `${label} windows`) + const effective = requireIsoTimestamp(parsed.effective, `${label} effective`) + if (!ratesEqual(offPeak, base)) { + throw new Error(`Expected ${label} off-peak rates to match the base table rates`) + } + timeOfDay = { offPeak, peak, windows, effective } + } + + let deal: PricingPageRow["deal"] + if (record.deal !== undefined) { + const parsed = requireRecord(record.deal, `${label} deal`) + const dealId = requireNonEmptyString(parsed.id, `${label} deal id`) + const discountPercent = requireFiniteNonNegative( + parsed.discountPercent, + `${label} discountPercent`, + ) + if (discountPercent > 100) { + throw new Error(`Expected ${label} discountPercent to be at most 100`) + } + if (typeof parsed.free !== "boolean") { + throw new Error(`Expected ${label} deal free to be a boolean`) + } + const endsWhen = + parsed.endsWhen === undefined + ? undefined + : requireNonEmptyString(parsed.endsWhen, `${label} endsWhen`) + const expires = + parsed.expires === undefined ? undefined : requireExpires(parsed.expires, `${label} expires`) + deal = { + id: dealId, + discountPercent, + free: parsed.free, + ...(expires !== undefined ? { expires } : {}), + ...(endsWhen !== undefined ? { endsWhen } : {}), + } + } + + return { id, cost, ...(timeOfDay ? { timeOfDay } : {}), ...(deal ? { deal } : {}) } +} + +/** + * Extract the normalized pricing table from Command Code's Next.js Flight + * payload. This depends on the private page format, not on a public pricing + * API, so an unrecognized shape is an error rather than an empty success. + */ +export function parsePricingPage(html: string): readonly PricingPageRow[] { + const chunks: string[] = [] + for (const match of html.matchAll(SCRIPT_TAG_RE)) { + const body = (match[1] ?? "").trim() + const push = NEXT_PUSH_RE.exec(body) + if (!push) continue + let parsed: unknown + try { + parsed = JSON.parse(push[1]!) + } catch { + continue + } + if (Array.isArray(parsed) && parsed[0] === 1 && typeof parsed[1] === "string") { + chunks.push(parsed[1]) + } + } + + const flight = chunks.join("") + const rowsPayloads: unknown[][] = [] + for (const line of flight.split("\n")) { + const record = FLIGHT_RECORD_RE.exec(line) + if (!record) continue + const suffix = record[1]! + if (!suffix.startsWith("[")) continue + let value: unknown + try { + value = JSON.parse(suffix) + } catch { + if (suffix.includes('"rows"')) throw new Error("Malformed pricing page rows record") + continue + } + if ( + Array.isArray(value) && + value.length >= 4 && + isRecord(value[3]) && + Object.prototype.hasOwnProperty.call(value[3], "rows") + ) { + rowsPayloads.push((value[3] as { rows: unknown[] }).rows) + } + } + + if (rowsPayloads.length !== 1) { + throw new Error(`Expected exactly one pricing rows payload, found ${rowsPayloads.length}`) + } + const rows = rowsPayloads[0]! + if (!Array.isArray(rows) || rows.length === 0) { + throw new Error("Expected the pricing rows payload to be a non-empty array") + } + + const parsedRows = rows.map((row, index) => parseRow(row, index)) + const seen = new Set() + for (const row of parsedRows) { + const key = row.id.toLowerCase() + if (seen.has(key)) throw new Error(`Duplicate pricing row id: ${row.id}`) + seen.add(key) + } + return parsedRows +} + +export function pricingPageIdCandidates(modelId: string): readonly string[] { + const candidates = new Set() + candidates.add(modelId.toLowerCase()) + const alias = PRICING_PAGE_ID_ALIASES[modelId] + if (alias !== undefined) { + candidates.add(alias) + } else { + const basename = modelId.slice(modelId.lastIndexOf("/") + 1).toLowerCase() + candidates.add(basename.replace(/^qwen(?=\d)/, "qwen-")) + } + return [...candidates] +} + +function requireLocalRate(value: unknown, modelId: string, field: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Expected ${modelId} ${field} to be a finite non-negative number`) + } + return value +} + +function normalizeLocalCost(cost: CommandCodeModelCost, modelId: string): NormalizedCost { + const rates = { + input: requireLocalRate(cost.input, modelId, "input"), + output: requireLocalRate(cost.output, modelId, "output"), + cacheRead: requireLocalRate(cost.cacheRead, modelId, "cacheRead"), + cacheWrite: requireLocalRate(cost.cacheWrite, modelId, "cacheWrite"), + } + const tiers: CommandCodeModelCostTier[] = [] + let previousThreshold = -1 + let previousRates = rates + for (const tier of cost.tiers ?? []) { + const tierRates = { + input: requireLocalRate(tier.input, modelId, "tier input"), + output: requireLocalRate(tier.output, modelId, "tier output"), + cacheRead: requireLocalRate(tier.cacheRead, modelId, "tier cacheRead"), + cacheWrite: requireLocalRate(tier.cacheWrite, modelId, "tier cacheWrite"), + } + if (!Number.isInteger(tier.inputTokensAbove) || tier.inputTokensAbove <= 0) { + throw new Error(`Expected ${modelId} tier inputTokensAbove to be a positive integer`) + } + if (tier.inputTokensAbove <= previousThreshold) { + throw new Error(`Expected ${modelId} tier thresholds to strictly increase`) + } + previousThreshold = tier.inputTokensAbove + if (ratesEqual(tierRates, previousRates)) continue + tiers.push({ inputTokensAbove: tier.inputTokensAbove, ...tierRates }) + previousRates = tierRates + } + return { ...rates, tiers } +} + +function normalizeCost(cost: CommandCodeModelCost): NormalizedCost { + return { + input: cost.input, + output: cost.output, + cacheRead: cost.cacheRead, + cacheWrite: cost.cacheWrite, + tiers: cost.tiers ?? [], + } +} + +function tiersEqual( + left: readonly CommandCodeModelCostTier[], + right: readonly CommandCodeModelCostTier[], +): boolean { + if (left.length !== right.length) return false + return left.every((tier, index) => { + const other = right[index]! + return tier.inputTokensAbove === other.inputTokensAbove && ratesEqual(tier, other) + }) +} + +function normalizedCostsEqual(left: NormalizedCost, right: NormalizedCost): boolean { + return ratesEqual(left, right) && tiersEqual(left.tiers, right.tiers) +} + +function costJson(cost: NormalizedCost): Record { + return { + input: cost.input, + output: cost.output, + cacheRead: cost.cacheRead, + cacheWrite: cost.cacheWrite, + tiers: cost.tiers.map((tier) => ({ + inputTokensAbove: tier.inputTokensAbove, + input: tier.input, + output: tier.output, + cacheRead: tier.cacheRead, + cacheWrite: tier.cacheWrite, + })), + } +} + +function dealExpiryMs(deal: NonNullable): number | undefined { + if (deal.expires === undefined) return undefined + const calendar = CALENDAR_DATE_RE.exec(deal.expires) + if (calendar) { + const [, year, month, day] = calendar + // The deal covers the whole calendar day, so it expires at the next UTC midnight. + return Date.UTC(Number(year), Number(month) - 1, Number(day) + 1) + } + return Date.parse(deal.expires) +} + +function isPeakProbe(atMs: number): boolean { + const date = new Date(atMs) + const day = date.getUTCDay() + if (day === 0 || day === 6) return false + const hour = date.getUTCHours() + return (hour >= 1 && hour < 4) || (hour >= 6 && hour < 10) +} + +function checkTimePolicy( + modelId: string, + pageRow: PricingPageRow, + atMs: number, +): PricingCheckIssue | undefined { + const baseRates: CommandCodeModelCostRates = { + input: pageRow.cost.input, + output: pageRow.cost.output, + cacheRead: pageRow.cost.cacheRead, + cacheWrite: pageRow.cost.cacheWrite, + } + let expected: (probeMs: number) => CommandCodeModelCostRates = () => baseRates + const timeOfDay = pageRow.timeOfDay + if (timeOfDay) { + if (timeOfDay.windows !== SUPPORTED_TIME_WINDOWS) { + return { + kind: "time-policy", + modelId, + detail: `Unsupported time-pricing window requires review: ${timeOfDay.windows}`, + } + } + if (Date.parse(timeOfDay.effective) > atMs) { + return { + kind: "time-policy", + modelId, + detail: `Future effective date requires review: ${timeOfDay.effective}`, + } + } + expected = (probeMs) => (isPeakProbe(probeMs) ? timeOfDay.peak : timeOfDay.offPeak) + } + + for (let hour = 0; hour < PROBE_HOURS; hour += 1) { + const probeMs = PROBE_WEEK_START_MS + hour * 3_600_000 + const actual = commandCodeCostRatesAt(modelId, pageRow.cost, probeMs) + const want = expected(probeMs) + if (!ratesClose(actual, want)) { + return { + kind: "time-policy", + modelId, + detail: `First mismatch at ${new Date(probeMs).toISOString()}: current=${JSON.stringify( + actual, + )} expected=${JSON.stringify(want)}`, + } + } + } + return undefined +} + +export function checkCommandCodePricing( + modelIds: readonly string[], + pageRows: readonly PricingPageRow[], + localCosts: Readonly>, + atMs: number, +): PricingCheckResult { + if (modelIds.length === 0) throw new Error("Expected at least one model id") + const unique = new Set() + for (const modelId of modelIds) { + if (typeof modelId !== "string" || modelId.length === 0) { + throw new Error("Expected model ids to be non-empty strings") + } + if (unique.has(modelId)) throw new Error(`Duplicate model id: ${modelId}`) + unique.add(modelId) + } + if (typeof atMs !== "number" || !Number.isFinite(atMs)) { + throw new Error("Expected atMs to be a finite timestamp") + } + + const sortedModelIds = [...modelIds].sort() + const matchedByModel = new Map() + const claimCount = new Map() + for (const modelId of sortedModelIds) { + const candidates = new Set(pricingPageIdCandidates(modelId)) + const matches = pageRows.filter((row) => candidates.has(row.id.toLowerCase())) + matchedByModel.set(modelId, matches) + for (const row of matches) claimCount.set(row, (claimCount.get(row) ?? 0) + 1) + } + + const issues: PricingCheckIssue[] = [] + const deals: { modelId: string; deal: NonNullable }[] = [] + + for (const modelId of sortedModelIds) { + const hasLocal = Object.prototype.hasOwnProperty.call(localCosts, modelId) + if (!hasLocal) { + issues.push({ + kind: "missing-local-price", + modelId, + detail: "No MODEL_COSTS entry for this advertised model.", + }) + } + + const matches = matchedByModel.get(modelId) ?? [] + if (matches.length === 0) { + issues.push({ + kind: "missing-page-row", + modelId, + detail: "No pricing page row matches this model id.", + }) + continue + } + if (matches.length > 1) { + issues.push({ + kind: "ambiguous-page-row", + modelId, + detail: `Matched ${matches.length} page rows: ${matches.map((row) => row.id).join(", ")}`, + }) + continue + } + const pageRow = matches[0]! + if ((claimCount.get(pageRow) ?? 0) > 1) { + issues.push({ + kind: "ambiguous-page-row", + modelId, + detail: `Page row ${pageRow.id} is also claimed by another model id.`, + }) + continue + } + + if (pageRow.deal) { + deals.push({ modelId, deal: pageRow.deal }) + const expiry = dealExpiryMs(pageRow.deal) + if (expiry !== undefined && atMs >= expiry) { + issues.push({ + kind: "expired-deal", + modelId, + detail: `Deal ${pageRow.deal.id} expired at ${new Date(expiry).toISOString()}.`, + }) + } + } + + if (hasLocal) { + const local = normalizeLocalCost(localCosts[modelId]!, modelId) + const upstream = normalizeCost(pageRow.cost) + if (!normalizedCostsEqual(local, upstream)) { + issues.push({ + kind: "changed-cost", + modelId, + detail: `current=${JSON.stringify(costJson(local))} upstream=${JSON.stringify( + costJson(upstream), + )}`, + }) + } + } + + const timeIssue = checkTimePolicy(modelId, pageRow, atMs) + if (timeIssue) issues.push(timeIssue) + } + + const unusedPageIds = [ + ...new Set(pageRows.filter((row) => !claimCount.has(row)).map((row) => row.id)), + ].sort() + issues.sort((left, right) => { + if (left.modelId !== right.modelId) return left.modelId < right.modelId ? -1 : 1 + if (left.kind !== right.kind) return left.kind < right.kind ? -1 : 1 + return 0 + }) + + return { checkedModelCount: modelIds.length, issues, unusedPageIds, deals } +} + +function formatId(value: string): string { + return `\`${value.replace(/`/g, "")}\`` +} + +function formatCell(value: string): string { + return value + .replace(/\|/g, "\\|") + .replace(/[\r\n]+/g, " ") + .replace(/`/g, "") +} + +export function renderPricingReport(result: PricingCheckResult): string { + const status = result.issues.length === 0 ? "PASS" : "REVIEW REQUIRED" + const lines: string[] = [ + "# Command Code pricing check", + "", + `- Pricing source: ${PRICING_SOURCE_URL}`, + `- Model catalog: ${DEFAULT_MODELS_URL}`, + `- Static snapshot verified: ${PRICING_LAST_VERIFIED}`, + `- Models checked: ${result.checkedModelCount}`, + "", + `**Status: ${status}**`, + "", + ] + + if (result.issues.length === 0) { + lines.push("No pricing drift detected.", "") + } else { + lines.push("| Kind | Model | Detail |", "| --- | --- | --- |") + for (const issue of result.issues) { + lines.push( + `| ${formatCell(issue.kind)} | ${formatId(issue.modelId)} | ${formatCell(issue.detail)} |`, + ) + } + lines.push("") + } + + lines.push("## Promotions", "") + if (result.deals.length === 0) { + lines.push("_No promotions reported._", "") + } else { + lines.push( + "| Model | Deal | Discount | Free | Expires | Ends when |", + "| --- | --- | --- | --- | --- | --- |", + ) + for (const { modelId, deal } of result.deals) { + lines.push( + `| ${formatId(modelId)} | ${formatId(deal.id)} | ${deal.discountPercent}% | ${ + deal.free ? "yes" : "no" + } | ${deal.expires ? formatCell(deal.expires) : "—"} | ${ + deal.endsWhen ? formatCell(deal.endsWhen) : "—" + } |`, + ) + } + lines.push("") + } + + lines.push( + "## Unused pricing-page rows", + "", + "Informational only: these site rows do not match an advertised API model.", + "", + ) + if (result.unusedPageIds.length === 0) { + lines.push("_None._", "") + } else { + for (const id of result.unusedPageIds) lines.push(`- ${formatId(id)}`) + lines.push("") + } + + lines.push( + "---", + "", + "Static prices in `src/pricing.ts` are unchanged by this check. The Command Code billing Usage page remains authoritative for each request, and every drift listed above requires manual review before prices are edited.", + "", + ) + return lines.join("\n") +} + +function renderPricingFailure(message: string): string { + return [ + "# Command Code pricing check", + "", + `- Pricing source: ${PRICING_SOURCE_URL}`, + `- Model catalog: ${DEFAULT_MODELS_URL}`, + "", + "**ERROR: the pricing check could not be completed.**", + "", + `> ${formatCell(message)}`, + "", + ].join("\n") +} + +export async function runPricingCheck( + options: { + fetchImpl?: typeof fetch + atMs?: number + localCosts?: Readonly> + } = {}, +): Promise { + const fetchImpl = options.fetchImpl ?? fetch + const atMs = options.atMs ?? Date.now() + const localCosts = options.localCosts ?? MODEL_COSTS + + const models = await fetchCommandCodeModels({ fetchImpl, timeoutMs: DEFAULT_MODELS_TIMEOUT_MS }) + const modelIds = models.map((model) => model.id) + const seen = new Set() + for (const modelId of modelIds) { + if (seen.has(modelId)) { + throw new Error(`Duplicate model id in the Command Code catalog: ${modelId}`) + } + seen.add(modelId) + } + + const response = await fetchImpl(PRICING_SOURCE_URL, { + headers: { accept: "text/html" }, + signal: AbortSignal.timeout(DEFAULT_MODELS_TIMEOUT_MS), + }) + if (!response.ok) { + throw new Error( + `Failed to fetch the Command Code pricing page: ${response.status} ${response.statusText}`, + ) + } + const pageRows = parsePricingPage(await response.text()) + return checkCommandCodePricing(modelIds, pageRows, localCosts, atMs) +} + +export async function runPricingCheckCli(options: { + args: readonly string[] + fetchImpl?: typeof fetch + atMs?: number + localCosts?: Readonly> + stdout: (text: string) => void + stderr: (text: string) => void +}): Promise<0 | 1 | 2> { + if (options.args.length > 0) { + options.stderr("Usage: npm run check:commandcode-pricing\n") + return 2 + } + try { + const result = await runPricingCheck({ + fetchImpl: options.fetchImpl, + atMs: options.atMs, + localCosts: options.localCosts, + }) + options.stdout(renderPricingReport(result)) + return result.issues.length > 0 ? 1 : 0 + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + options.stdout(renderPricingFailure(message)) + options.stderr(`${message}\n`) + return 2 + } +} + +function isMainModule(): boolean { + const entrypoint = process.argv[1] + return entrypoint !== undefined && pathToFileURL(resolve(entrypoint)).href === import.meta.url +} + +if (isMainModule()) { + process.exitCode = await runPricingCheckCli({ + args: process.argv.slice(2), + stdout: (text) => console.log(text), + stderr: (text) => console.error(text), + }) +} diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index b9d39b1..33cccd1 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -5,14 +5,20 @@ on: branches: [main] paths: - ".github/scripts/check-commandcode-model-metadata.ts" + - ".github/scripts/check-commandcode-pricing.ts" - ".github/workflows/model-metadata.yml" - "README.md" + - "package.json" - "src/commandcode-catalog-overrides.ts" - "src/commandcode-catalog.ts" - "src/core.ts" + - "src/cost.ts" - "src/models.ts" + - "src/pricing.ts" + - "tests/fixtures/commandcode-pricing-page.html" - "tests/test-model-metadata-check.ts" - "tests/test-models.ts" + - "tests/test-pricing-check.ts" schedule: - cron: "17 6 * * *" workflow_dispatch: @@ -41,6 +47,14 @@ jobs: - name: Publish catalog report if: always() run: cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY" + - name: Compare official pricing + if: always() + shell: bash + run: npm run check:commandcode-pricing | tee commandcode-pricing-report.md + - name: Publish pricing report + if: always() + run: | + if [ -f commandcode-pricing-report.md ]; then cat commandcode-pricing-report.md >> "$GITHUB_STEP_SUMMARY"; fi sync: if: github.event_name != 'pull_request' @@ -92,3 +106,11 @@ jobs: src/commandcode-catalog.ts src/commandcode-catalog-overrides.ts README.md + - name: Compare official pricing + if: always() + shell: bash + run: npm run check:commandcode-pricing | tee commandcode-pricing-report.md + - name: Publish pricing report + if: always() + run: | + if [ -f commandcode-pricing-report.md ]; then cat commandcode-pricing-report.md >> "$GITHUB_STEP_SUMMARY"; fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 128a891..1a86ad4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add a read-only `npm run check:commandcode-pricing` command that compares the live Provider API catalog and the official pricing page against `MODEL_COSTS`, reporting missing local prices, changed base rates and context tiers, time-policy drift, and expired promotions. It runs in the daily catalog workflow and applicable pull requests, publishes its report to the Actions summary, and never edits prices. - Use the Provider API's OpenAI Responses wire (`/provider/v1/responses`) for models whose `/provider/v1/models` entry advertises `/responses`, keeping OpenAI Chat Completions for chat-only models and Anthropic Messages for Claude. The wire is resolved per model from `supported_endpoints`, falling back to Chat Completions when the catalog omits the field. ## 0.7.3 - 2026-09-28 diff --git a/README.md b/README.md index cfb0264..6eef126 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,8 @@ The Command Code Provider API does not currently include prices in its model cat Models missing from that table display zero cost in pi. This does **not** mean that Command Code will bill the request at zero. The Command Code Usage page remains authoritative for each request. Check the current [Command Code pricing](https://commandcode.ai/docs/resources/pricing-limits) before relying on the displayed value. +A read-only companion check, `npm run check:commandcode-pricing`, compares the live Provider API catalog and the official pricing page with the static table. It never writes prices: exit code `0` means every compared rate matches, `1` means drift or an expired promotion needs manual review, and `2` means the price could not be checked. The daily catalog workflow and applicable pull requests publish the Markdown report to the GitHub Actions summary. The pricing page has no public pricing API, so the check parses its private Next.js Flight payload; if that format changes the check fails loudly instead of reporting a false pass, and prices stay manually reviewed. + ## Update and remove Update installed pi packages: diff --git a/package.json b/package.json index eea63d1..9f622fd 100644 --- a/package.json +++ b/package.json @@ -29,21 +29,23 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && node --test tests/test-release.mjs && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && node --test tests/test-release.mjs && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", "check:commandcode-catalog": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest", + "check:commandcode-pricing": "tsx .github/scripts/check-commandcode-pricing.ts", "sync:commandcode-catalog": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest --write", "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts", "test:usage": "tsx tests/test-usage.ts", - "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", + "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", "test:models": "tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts", "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", + "test:pricing-check": "tsx tests/test-pricing-check.ts", "test:oauth": "tsx tests/test-oauth.ts", "test:abort": "tsx tests/test-abort.ts", "test:overflow": "tsx tests/test-overflow.ts", diff --git a/tests/fixtures/commandcode-pricing-page.html b/tests/fixtures/commandcode-pricing-page.html new file mode 100644 index 0000000..cda1167 --- /dev/null +++ b/tests/fixtures/commandcode-pricing-page.html @@ -0,0 +1,7 @@ + + diff --git a/tests/test-pricing-check.ts b/tests/test-pricing-check.ts new file mode 100644 index 0000000..96c9abb --- /dev/null +++ b/tests/test-pricing-check.ts @@ -0,0 +1,877 @@ +import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" +import { describe, it } from "node:test" + +import { + checkCommandCodePricing, + parsePricingPage, + pricingPageIdCandidates, + renderPricingReport, + runPricingCheck, + runPricingCheckCli, + type PricingPageRow, +} from "../.github/scripts/check-commandcode-pricing.ts" +import { DEFAULT_MODELS_URL } from "../src/models.ts" +import { PRICING_SOURCE_URL, type CommandCodeModelCost } from "../src/pricing.ts" + +const fixtureUrl = new URL("./fixtures/commandcode-pricing-page.html", import.meta.url) +const fixtureHtml = await readFile(fixtureUrl, "utf-8") +const AT_MS = Date.UTC(2026, 9, 5, 12, 0, 0) +const SUPPORTED_WINDOWS = "01–04 & 06–10 UTC, Mon–Fri" + +function cost( + input: number, + output: number, + cacheRead: number, + cacheWrite: number, +): CommandCodeModelCost { + return { input, output, cacheRead, cacheWrite } +} + +function ratesJson(rates: { + input: number + output: number + cacheRead: number + cacheWrite: number +}) { + return { + input: rates.input, + output: rates.output, + cacheRead: rates.cacheRead, + cacheWrite: rates.cacheWrite, + } +} + +function pageRow( + id: string, + rowCost: CommandCodeModelCost, + extra: Partial = {}, +): PricingPageRow { + return { id, cost: rowCost, ...extra } +} + +function findRow(rows: readonly PricingPageRow[], id: string): PricingPageRow { + const row = rows.find((candidate) => candidate.id === id) + assert.ok(row, `expected fixture row ${id}`) + return row +} + +function rowsRecord(rows: unknown): string { + return `34:${JSON.stringify(["$", "$L53", null, { rows }])}\n` +} + +function pushScript(flightText: string): string { + return `` +} + +function pageWithRows(rows: unknown, extraFlight = ""): string { + return pushScript(extraFlight + rowsRecord(rows)) +} + +function modelsBody(ids: readonly string[]) { + return { object: "list", data: ids.map((id) => ({ id, name: id, context_length: 1_000 })) } +} + +function fakeFetch(routes: { models?: unknown; page?: string; pageStatus?: number }) { + const calls: { url: string; init?: RequestInit }[] = [] + const impl = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url + calls.push({ url, init }) + if (url === DEFAULT_MODELS_URL) { + return new Response(JSON.stringify(routes.models ?? modelsBody(["model"])), { + status: 200, + headers: { "content-type": "application/json" }, + }) + } + if (url === PRICING_SOURCE_URL) { + const status = routes.pageStatus ?? 200 + return new Response(routes.page ?? "", { + status, + statusText: status === 200 ? "OK" : "Service Unavailable", + }) + } + throw new Error(`Unexpected fetch: ${url}`) + }) as typeof fetch + return { impl, calls } +} + +async function captureCli( + fetchImpl: typeof fetch, + options: { atMs?: number; localCosts?: Readonly> } = {}, +) { + let stdout = "" + let stderr = "" + const code = await runPricingCheckCli({ + args: [], + fetchImpl, + atMs: options.atMs ?? AT_MS, + localCosts: options.localCosts, + stdout: (text) => { + stdout += text + }, + stderr: (text) => { + stderr += text + }, + }) + return { code, stdout, stderr } +} + +describe("parsePricingPage with the live snapshot fixture", () => { + const rows = parsePricingPage(fixtureHtml) + + it("parses exactly 89 unique rows", () => { + assert.equal(rows.length, 89) + assert.equal(new Set(rows.map((row) => row.id.toLowerCase())).size, 89) + }) + + it("extracts DeepSeek V4.1 Flash Fast base and peak rates", () => { + const row = findRow(rows, "deepseek-v4.1-flash-fast") + assert.deepEqual(ratesJson(row.cost), { + input: 0.16, + output: 0.58, + cacheRead: 0.016, + cacheWrite: 0, + }) + assert.ok(row.timeOfDay) + assert.deepEqual(row.timeOfDay.offPeak, { + input: 0.16, + output: 0.58, + cacheRead: 0.016, + cacheWrite: 0, + }) + assert.deepEqual(row.timeOfDay.peak, { + input: 0.32, + output: 1.16, + cacheRead: 0.032, + cacheWrite: 0, + }) + }) + + it("extracts the gpt-6.1-sol long-context tier", () => { + const row = findRow(rows, "gpt-6.1-sol") + assert.deepEqual(ratesJson(row.cost), { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 }) + assert.deepEqual(row.cost.tiers, [ + { inputTokensAbove: 272_000, input: 4, output: 15, cacheRead: 0.2, cacheWrite: 5 }, + ]) + }) + + it("extracts Qwen 3.7 Flash thresholds", () => { + const row = findRow(rows, "qwen-3.7-flash") + assert.deepEqual( + row.cost.tiers?.map((tier) => tier.inputTokensAbove), + [32_000, 256_000], + ) + }) + + it("collapses MiniMax M3 tiers that repeat the base rates", () => { + const row = findRow(rows, "minimax-m3") + assert.equal(row.cost.tiers, undefined) + }) +}) + +describe("parsePricingPage shape handling", () => { + it("reconstructs one row split across two push chunks", () => { + const record = rowsRecord([{ id: "split-model", tiers: [{ rates: { input: 1, output: 2 } }] }]) + const split = Math.floor(record.length / 2) + const rows = parsePricingPage( + pushScript(record.slice(0, split)) + pushScript(record.slice(split)), + ) + assert.deepEqual( + rows.map((row) => row.id), + ["split-model"], + ) + }) + + it("ignores unrelated props.models payloads", () => { + const payload = ["$", "$L9", null, { models: [{ id: "budget" }] }] + const modelsRecord = `35:${JSON.stringify(payload)}\n` + const rows = parsePricingPage( + pageWithRows([{ id: "only-row", tiers: [{ rates: { input: 1, output: 2 } }] }], modelsRecord), + ) + assert.deepEqual( + rows.map((row) => row.id), + ["only-row"], + ) + }) + + it("rejects pages without exactly one valid rows payload", () => { + assert.throws(() => parsePricingPage(""), /exactly one pricing rows payload/) + assert.throws( + () => + parsePricingPage(pushScript(`35:${JSON.stringify(["$", "$L9", null, { models: [] }])}\n`)), + /exactly one pricing rows payload/, + ) + assert.throws(() => parsePricingPage(pageWithRows([])), /non-empty array/) + assert.throws( + () => + parsePricingPage( + pageWithRows([{ id: "a", tiers: [{ rates: { input: 1, output: 1 } }] }]) + + pageWithRows([{ id: "b", tiers: [{ rates: { input: 1, output: 1 } }] }]), + ), + /exactly one pricing rows payload/, + ) + assert.throws(() => parsePricingPage(pushScript('34:[{"rows":')), /Malformed pricing page/) + assert.throws( + () => + parsePricingPage( + pageWithRows([ + { id: "Dup", tiers: [{ rates: { input: 1, output: 1 } }] }, + { id: "dup", tiers: [{ rates: { input: 1, output: 1 } }] }, + ]), + ), + /Duplicate pricing row id/, + ) + }) +}) + +describe("pricing rate and context validation", () => { + it("treats an absent cache column as explicit zero", () => { + const rows = parsePricingPage( + pageWithRows([{ id: "zero", tiers: [{ rates: { input: 0, output: 0 } }] }]), + ) + assert.deepEqual(rows[0]!.cost, { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }) + }) + + it("rejects malformed rate and context shapes", () => { + const rateCases: [string, unknown][] = [ + ["missing input", { output: 1 }], + ["missing output", { input: 1 }], + ["null cacheWrite", { input: 1, output: 1, cacheWrite: null }], + ["negative input", { input: -1, output: 1 }], + ["string input", { input: "1", output: 1 }], + ] + for (const [label, rates] of rateCases) { + assert.throws( + () => parsePricingPage(pageWithRows([{ id: "m", tiers: [{ rates }] }])), + Error, + label, + ) + } + + const contextCases: [string, unknown][] = [ + ["lone above band", [{ context: "> 272K", rates: { input: 1, output: 1 } }]], + [ + "non-increasing bounds", + [ + { context: "≤ 256K", rates: { input: 1, output: 1 } }, + { context: "≤ 128K", rates: { input: 2, output: 2 } }, + { context: "> 128K", rates: { input: 3, output: 3 } }, + ], + ], + [ + "last above does not match", + [ + { context: "≤ 272K", rates: { input: 1, output: 1 } }, + { context: "> 128K", rates: { input: 2, output: 2 } }, + ], + ], + ["unknown single context", [{ context: "up to 32K", rates: { input: 1, output: 1 } }]], + ["zero single context", [{ context: "≤ 0K", rates: { input: 1, output: 1 } }]], + ] + for (const [label, tiers] of contextCases) { + assert.throws(() => parsePricingPage(pageWithRows([{ id: "m", tiers }])), Error, label) + } + }) +}) + +describe("pricing page id matching", () => { + it("derives page id candidates from API ids and aliases", () => { + assert.deepEqual(pricingPageIdCandidates("tencent/hy4-preview"), [ + "tencent/hy4-preview", + "hy4-preview", + ]) + assert.deepEqual(pricingPageIdCandidates("moonshotai/Kimi-K3"), [ + "moonshotai/kimi-k3", + "kimi-k3", + ]) + assert.deepEqual(pricingPageIdCandidates("Qwen/Qwen3.8-Max"), [ + "qwen/qwen3.8-max", + "qwen-3.8-max", + ]) + assert.deepEqual(pricingPageIdCandidates("claude-haiku-4-5-20251001"), [ + "claude-haiku-4-5-20251001", + "claude-haiku-4-5", + ]) + assert.deepEqual(pricingPageIdCandidates("Qwen/Qwen3.6-Max-Preview"), [ + "qwen/qwen3.6-max-preview", + "qwen-3.6-max", + ]) + assert.deepEqual(pricingPageIdCandidates("nvidia/nemotron-3-ultra-550b-a55b"), [ + "nvidia/nemotron-3-ultra-550b-a55b", + "nemotron-3-ultra", + ]) + }) + + it("flags ambiguous page rows instead of picking the first match", () => { + const pageRows = [pageRow("vendor/model", cost(1, 1, 0, 0)), pageRow("model", cost(1, 1, 0, 0))] + const result = checkCommandCodePricing( + ["vendor/model"], + pageRows, + { "vendor/model": cost(1, 1, 0, 0) }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["ambiguous-page-row"], + ) + }) + + it("flags two API ids that share one page row", () => { + const pageRows = [pageRow("model", cost(1, 1, 0, 0))] + const result = checkCommandCodePricing( + ["a/model", "b/model"], + pageRows, + { "a/model": cost(1, 1, 0, 0), "b/model": cost(1, 1, 0, 0) }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["ambiguous-page-row", "ambiguous-page-row"], + ) + }) + + it("reports a missing page row", () => { + const result = checkCommandCodePricing( + ["missing/model"], + [], + { "missing/model": cost(1, 1, 0, 0) }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["missing-page-row"], + ) + }) +}) + +describe("advertised model without a local price", () => { + const apiModels = ["deepseek/deepseek-v4.1-flash-fast"] + const fixtureRows = parsePricingPage(fixtureHtml) + const flashFastRow = findRow(fixtureRows, "deepseek-v4.1-flash-fast") + + it("reports missing-local-price and time-policy, then CLI exit 1", async () => { + const { impl } = fakeFetch({ models: modelsBody(apiModels), page: fixtureHtml }) + const result = await runPricingCheck({ fetchImpl: impl, atMs: AT_MS, localCosts: {} }) + assert.deepEqual(result.issues.map((issue) => issue.kind).sort(), [ + "missing-local-price", + "time-policy", + ]) + + const cli = await captureCli(impl, { localCosts: {} }) + assert.equal(cli.code, 1) + assert.match(cli.stdout, /deepseek\/deepseek-v4\.1-flash-fast/) + }) + + it("reports only time-policy when the local rates are exact", async () => { + const localCosts = { "deepseek/deepseek-v4.1-flash-fast": flashFastRow.cost } + const { impl } = fakeFetch({ models: modelsBody(apiModels), page: fixtureHtml }) + const result = await runPricingCheck({ fetchImpl: impl, atMs: AT_MS, localCosts }) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["time-policy"], + ) + + const cli = await captureCli(impl, { localCosts }) + assert.equal(cli.code, 1) + }) + + it("passes for a model whose page row and runtime policy agree", async () => { + const modelId = "deepseek/deepseek-v4-flash-fast" + const row = findRow(fixtureRows, "deepseek-v4-flash-fast") + const localCosts = { [modelId]: row.cost } + const { impl } = fakeFetch({ models: modelsBody([modelId]), page: fixtureHtml }) + const result = await runPricingCheck({ fetchImpl: impl, atMs: AT_MS, localCosts }) + assert.deepEqual(result.issues, []) + + const cli = await captureCli(impl, { localCosts }) + assert.equal(cli.code, 0) + }) + + it("does not import filesystem write APIs", async () => { + const source = await readFile( + new URL("../.github/scripts/check-commandcode-pricing.ts", import.meta.url), + "utf-8", + ) + assert.doesNotMatch(source, /node:fs/) + }) +}) + +describe("changed cost detection", () => { + it("reports a changed flat price with both sides", () => { + const pageRows = [pageRow("flat-model", cost(2, 3, 0, 0))] + const result = checkCommandCodePricing( + ["flat-model"], + pageRows, + { "flat-model": cost(1, 3, 0, 0) }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["changed-cost"], + ) + assert.match(result.issues[0]!.detail, /current=/) + assert.match(result.issues[0]!.detail, /upstream=/) + }) + + it("reports a changed local threshold", () => { + const pageRows = [ + pageRow("gpt-6.1-sol", { + input: 2, + output: 10, + cacheRead: 0.1, + cacheWrite: 2.5, + tiers: [{ inputTokensAbove: 272_000, input: 4, output: 15, cacheRead: 0.2, cacheWrite: 5 }], + }), + ] + const local = { + input: 2, + output: 10, + cacheRead: 0.1, + cacheWrite: 2.5, + tiers: [{ inputTokensAbove: 272_001, input: 4, output: 15, cacheRead: 0.2, cacheWrite: 5 }], + } + const result = checkCommandCodePricing( + ["gpt-6.1-sol"], + pageRows, + { "gpt-6.1-sol": local }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["changed-cost"], + ) + }) + + it("reports a missing local long-context tier", () => { + const rows = parsePricingPage( + pageWithRows([ + { + id: "qwen-3.6-plus", + tiers: [ + { context: "≤ 256K", rates: { input: 0.5, output: 3, cacheRead: 0.1 } }, + { context: "> 256K", rates: { input: 2, output: 6, cacheRead: 0.2 } }, + ], + }, + ]), + ) + const result = checkCommandCodePricing( + ["Qwen/Qwen3.6-Plus"], + rows, + { "Qwen/Qwen3.6-Plus": cost(0.5, 3, 0.1, 0) }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["changed-cost"], + ) + }) + + it("accepts two identical upstream tiers after normalization", () => { + const rows = parsePricingPage( + pageWithRows([ + { + id: "minimax-m3", + tiers: [ + { context: "≤ 512K", rates: { input: 0.3, output: 1.2, cacheRead: 0.06 } }, + { context: "> 512K", rates: { input: 0.3, output: 1.2, cacheRead: 0.06 } }, + ], + }, + ]), + ) + const result = checkCommandCodePricing( + ["MiniMaxAI/MiniMax-M3"], + rows, + { "MiniMaxAI/MiniMax-M3": cost(0.3, 1.2, 0.06, 0) }, + AT_MS, + ) + assert.deepEqual(result.issues, []) + }) + + it("lists unused page rows without blocking stale local prices", () => { + const pageRows = [ + pageRow("live-model", cost(1, 2, 0, 0)), + pageRow("informational-row", cost(9, 9, 0, 0)), + ] + const result = checkCommandCodePricing( + ["live-model"], + pageRows, + { "live-model": cost(1, 2, 0, 0), "retired/model": cost(5, 5, 0, 0) }, + AT_MS, + ) + assert.deepEqual(result.issues, []) + assert.deepEqual(result.unusedPageIds, ["informational-row"]) + }) + + it("rejects invalid model ids and local thresholds", () => { + assert.throws(() => checkCommandCodePricing([], [], {}, AT_MS), /at least one model id/) + assert.throws(() => checkCommandCodePricing(["a", "a"], [], {}, AT_MS), /Duplicate model id/) + assert.throws(() => checkCommandCodePricing([""], [], {}, AT_MS), /non-empty strings/) + assert.throws(() => checkCommandCodePricing(["m"], [], {}, Number.NaN), /finite timestamp/) + + const pageRows = [pageRow("m", cost(1, 2, 0, 0))] + assert.throws( + () => + checkCommandCodePricing( + ["m"], + pageRows, + { m: { ...cost(1, 2, 0, 0), input: Number.POSITIVE_INFINITY } }, + AT_MS, + ), + /finite non-negative/, + ) + assert.throws( + () => + checkCommandCodePricing( + ["m"], + pageRows, + { m: { ...cost(1, 2, 0, 0), tiers: [{ inputTokensAbove: 0, ...cost(1, 2, 0, 0) }] } }, + AT_MS, + ), + /positive integer/, + ) + assert.throws( + () => + checkCommandCodePricing( + ["m"], + pageRows, + { + m: { + ...cost(1, 2, 0, 0), + tiers: [ + { inputTokensAbove: 10, ...cost(2, 2, 0, 0) }, + { inputTokensAbove: 10, ...cost(3, 2, 0, 0) }, + ], + }, + }, + AT_MS, + ), + /strictly increase/, + ) + }) +}) + +describe("time-priced models", () => { + const fixtureRows = parsePricingPage(fixtureHtml) + + it("accepts the current DeepSeek page and runtime policy", () => { + const modelIds = ["deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-flash-fast"] + const localCosts = { + "deepseek/deepseek-v4-flash": findRow(fixtureRows, "deepseek-v4-flash").cost, + "deepseek/deepseek-v4-flash-fast": findRow(fixtureRows, "deepseek-v4-flash-fast").cost, + } + const result = checkCommandCodePricing(modelIds, fixtureRows, localCosts, AT_MS) + assert.deepEqual(result.issues, []) + }) + + it("flags a page-only time policy the runtime does not implement", () => { + const modelId = "new-time-model" + const pageRows = [ + pageRow(modelId, cost(0.1, 0.2, 0, 0), { + timeOfDay: { + offPeak: cost(0.1, 0.2, 0, 0), + peak: cost(0.2, 0.4, 0, 0), + windows: SUPPORTED_WINDOWS, + effective: "2026-08-16T16:00:00Z", + }, + }), + ] + const result = checkCommandCodePricing( + [modelId], + pageRows, + { [modelId]: cost(0.1, 0.2, 0, 0) }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["time-policy"], + ) + assert.match(result.issues[0]!.detail, /2026-10-05T01:00:00\.000Z/) + }) + + it("detects a removed time policy for a runtime time-priced model", () => { + const row = findRow(fixtureRows, "deepseek-v4-flash") + const result = checkCommandCodePricing( + ["deepseek/deepseek-v4-flash"], + [pageRow(row.id, row.cost)], + { "deepseek/deepseek-v4-flash": row.cost }, + AT_MS, + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["time-policy"], + ) + }) + + it("detects a changed peak multiplier and an unsupported window", () => { + const row = findRow(fixtureRows, "deepseek-v4-flash") + const changedPeak = pageRow(row.id, row.cost, { + timeOfDay: { ...row.timeOfDay!, peak: cost(0.99, 1.2, 0.006, 0) }, + }) + const peakResult = checkCommandCodePricing( + ["deepseek/deepseek-v4-flash"], + [changedPeak], + { "deepseek/deepseek-v4-flash": row.cost }, + AT_MS, + ) + assert.deepEqual( + peakResult.issues.map((issue) => issue.kind), + ["time-policy"], + ) + + const unsupported = pageRow(row.id, row.cost, { + timeOfDay: { ...row.timeOfDay!, windows: "01:00-04:00 UTC" }, + }) + const windowResult = checkCommandCodePricing( + ["deepseek/deepseek-v4-flash"], + [unsupported], + { "deepseek/deepseek-v4-flash": row.cost }, + AT_MS, + ) + assert.deepEqual( + windowResult.issues.map((issue) => issue.kind), + ["time-policy"], + ) + assert.match(windowResult.issues[0]!.detail, /window/i) + }) + + it("allows effective exactly at atMs and flags a future effective date", () => { + const row = findRow(fixtureRows, "deepseek-v4-flash") + const localCosts = { "deepseek/deepseek-v4-flash": row.cost } + + const atNow = pageRow(row.id, row.cost, { + timeOfDay: { ...row.timeOfDay!, effective: new Date(AT_MS).toISOString() }, + }) + assert.deepEqual( + checkCommandCodePricing(["deepseek/deepseek-v4-flash"], [atNow], localCosts, AT_MS).issues, + [], + ) + + const future = pageRow(row.id, row.cost, { + timeOfDay: { ...row.timeOfDay!, effective: new Date(AT_MS + 3_600_000).toISOString() }, + }) + const flagged = checkCommandCodePricing( + ["deepseek/deepseek-v4-flash"], + [future], + localCosts, + AT_MS, + ) + assert.deepEqual( + flagged.issues.map((issue) => issue.kind), + ["time-policy"], + ) + assert.match(flagged.issues[0]!.detail, /Future effective/) + }) +}) + +describe("promotions and deals", () => { + const modelId = "deal-model" + const localCosts = { [modelId]: cost(1, 2, 0, 0) } + + function dealRow(expires?: string, endsWhen?: string): PricingPageRow { + return pageRow(modelId, cost(1, 2, 0, 0), { + deal: { + id: "deal-1", + discountPercent: 50, + free: false, + ...(expires ? { expires } : {}), + ...(endsWhen ? { endsWhen } : {}), + }, + }) + } + + it("treats a date-only deal as active through the last moment of the UTC day", () => { + const row = dealRow("2026-06-22") + const active = checkCommandCodePricing( + [modelId], + [row], + localCosts, + Date.UTC(2026, 5, 22, 23, 59, 59, 999), + ) + assert.deepEqual(active.issues, []) + assert.equal(active.deals.length, 1) + + const expired = checkCommandCodePricing([modelId], [row], localCosts, Date.UTC(2026, 5, 23)) + assert.deepEqual( + expired.issues.map((issue) => issue.kind), + ["expired-deal"], + ) + }) + + it("treats an ISO timestamp deal as expired exactly at the timestamp", () => { + const expires = "2026-06-22T12:00:00Z" + const row = dealRow(expires) + const before = checkCommandCodePricing([modelId], [row], localCosts, Date.parse(expires) - 1) + assert.deepEqual(before.issues, []) + const at = checkCommandCodePricing([modelId], [row], localCosts, Date.parse(expires)) + assert.deepEqual( + at.issues.map((issue) => issue.kind), + ["expired-deal"], + ) + }) + + it("lists an undated deal without inventing an expiration", () => { + const result = checkCommandCodePricing( + [modelId], + [dealRow(undefined, "while capacity lasts")], + localCosts, + AT_MS, + ) + assert.deepEqual(result.issues, []) + assert.equal(result.deals[0]!.deal.endsWhen, "while capacity lasts") + assert.equal(result.deals[0]!.deal.expires, undefined) + }) + + it("uses the literal discounted rates instead of listRates or discountPercent", () => { + const rows = parsePricingPage( + pageWithRows([ + { + id: modelId, + tiers: [{ rates: { input: 1, output: 2 }, listRates: { input: 9, output: 9 } }], + deal: { id: "d", discountPercent: 90, free: false }, + }, + ]), + ) + assert.deepEqual(ratesJson(rows[0]!.cost), { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 }) + }) + + it("rejects an impossible calendar date", () => { + assert.throws( + () => + parsePricingPage( + pageWithRows([ + { + id: "m", + tiers: [{ rates: { input: 1, output: 1 } }], + deal: { id: "d", discountPercent: 1, free: false, expires: "2026-02-30" }, + }, + ]), + ), + /valid calendar date/, + ) + }) + + it("reports the real Qwen 3.7 Max expired deal even when rates match", () => { + const rows = parsePricingPage(fixtureHtml) + const row = findRow(rows, "qwen-3.7-max") + const result = checkCommandCodePricing( + ["Qwen/Qwen3.7-Max"], + rows, + { "Qwen/Qwen3.7-Max": row.cost }, + Date.UTC(2026, 9, 5, 12, 0, 0), + ) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["expired-deal"], + ) + }) +}) + +describe("runner and report", () => { + it("checks the API and pricing routes without authorization headers", async () => { + const { impl, calls } = fakeFetch({ + models: modelsBody(["m"]), + page: pageWithRows([{ id: "m", tiers: [{ rates: { input: 1, output: 2 } }] }]), + }) + await runPricingCheck({ fetchImpl: impl, atMs: AT_MS, localCosts: { m: cost(1, 2, 0, 0) } }) + assert.deepEqual( + calls.map((call) => call.url), + [DEFAULT_MODELS_URL, PRICING_SOURCE_URL], + ) + for (const call of calls) { + const headers = call.init?.headers as Record | undefined + assert.equal(headers?.authorization, undefined) + assert.equal(headers?.Authorization, undefined) + } + }) + + it("returns CLI code 2 for HTTP failures and network rejections", async () => { + const pageFail = fakeFetch({ models: modelsBody(["m"]), pageStatus: 503 }) + const pageOut = await captureCli(pageFail.impl, {}) + assert.equal(pageOut.code, 2) + assert.match(pageOut.stdout, /ERROR/) + + const reject = (async () => { + throw new Error("network down") + }) as typeof fetch + const netOut = await captureCli(reject, {}) + assert.equal(netOut.code, 2) + assert.match(netOut.stdout, /ERROR/) + + const badApi = (async (input: RequestInfo | URL) => { + const url = String(input) + if (url === DEFAULT_MODELS_URL) { + return new Response(JSON.stringify({ object: "wrong" }), { status: 200 }) + } + return new Response("", { status: 200 }) + }) as typeof fetch + const badOut = await captureCli(badApi, {}) + assert.equal(badOut.code, 2) + assert.match(badOut.stdout, /ERROR/) + }) + + it("rejects unknown CLI arguments without fetching", async () => { + let calls = 0 + const impl = (async () => { + calls += 1 + throw new Error("should not fetch") + }) as typeof fetch + let stderr = "" + const code = await runPricingCheckCli({ + args: ["--write"], + fetchImpl: impl, + stdout: () => {}, + stderr: (text) => { + stderr += text + }, + }) + assert.equal(code, 2) + assert.equal(calls, 0) + assert.match(stderr, /Usage/) + }) + + it("renders deterministic reports and escapes table cells", () => { + const result = { + checkedModelCount: 1, + issues: [{ kind: "changed-cost" as const, modelId: "m", detail: "a|b\nc" }], + unusedPageIds: [], + deals: [], + } + const report = renderPricingReport(result) + assert.match(report, /REVIEW REQUIRED/) + assert.match(report, /a\\\|b c/) + assert.equal(report, renderPricingReport(result)) + }) + + it("renders PASS and no drift for a clean result", () => { + const report = renderPricingReport({ + checkedModelCount: 1, + issues: [], + unusedPageIds: ["informational"], + deals: [], + }) + assert.match(report, /PASS/) + assert.match(report, /No pricing drift detected\./) + assert.doesNotMatch(report, /REVIEW REQUIRED/) + }) + + it("does not fetch or change process.exitCode on import", async () => { + const originalFetch = globalThis.fetch + let calls = 0 + globalThis.fetch = (async () => { + calls += 1 + throw new Error("unexpected fetch") + }) as typeof fetch + try { + const exitCodeBefore = process.exitCode + const moduleUrl = new URL( + "../.github/scripts/check-commandcode-pricing.ts?import-side-effect-check", + import.meta.url, + ) + await import(moduleUrl.href) + assert.equal(calls, 0) + assert.equal(process.exitCode, exitCodeBefore) + } finally { + globalThis.fetch = originalFetch + } + }) +}) From 90b8e006f508971bc2cb1562cb74d07a2dc9ef10 Mon Sep 17 00:00:00 2001 From: Aleksandr Zverev Date: Tue, 6 Oct 2026 10:32:46 +0300 Subject: [PATCH 2/2] fix(models): resolve reviewed pricing drift --- .github/scripts/check-commandcode-pricing.ts | 34 +++++++++++++---- CHANGELOG.md | 1 + src/pricing.ts | 12 ++++-- tests/fixtures/commandcode-pricing.json | 5 ++- tests/test-pricing-check.ts | 40 ++++++++++++++++++-- tests/test-pricing.ts | 12 +++++- 6 files changed, 86 insertions(+), 18 deletions(-) diff --git a/.github/scripts/check-commandcode-pricing.ts b/.github/scripts/check-commandcode-pricing.ts index 6b5e8f6..6d83c5d 100644 --- a/.github/scripts/check-commandcode-pricing.ts +++ b/.github/scripts/check-commandcode-pricing.ts @@ -51,7 +51,11 @@ export interface PricingCheckResult { issues: readonly PricingCheckIssue[] unusedPageIds: readonly string[] // Promotions are visible even when there is no price drift. - deals: readonly { modelId: string; deal: NonNullable }[] + deals: readonly { + modelId: string + deal: NonNullable + resolvedExpired?: boolean + }[] } /** @@ -569,7 +573,11 @@ export function checkCommandCodePricing( } const issues: PricingCheckIssue[] = [] - const deals: { modelId: string; deal: NonNullable }[] = [] + const deals: { + modelId: string + deal: NonNullable + resolvedExpired?: boolean + }[] = [] for (const modelId of sortedModelIds) { const hasLocal = Object.prototype.hasOwnProperty.call(localCosts, modelId) @@ -609,9 +617,19 @@ export function checkCommandCodePricing( } if (pageRow.deal) { - deals.push({ modelId, deal: pageRow.deal }) const expiry = dealExpiryMs(pageRow.deal) - if (expiry !== undefined && atMs >= expiry) { + // Reviewed on 2026-10-06: the page retains this expired badge; its table rates remain current. + const resolvedExpired = + expiry !== undefined && + atMs >= expiry && + modelId === "Qwen/Qwen3.7-Max" && + pageRow.deal.id === "qwen-3.7-max-2x-usage" && + pageRow.deal.expires === "2026-06-22" && + pageRow.deal.discountPercent === 50 && + pageRow.deal.free === false && + pageRow.deal.endsWhen === undefined + deals.push({ modelId, deal: pageRow.deal, ...(resolvedExpired ? { resolvedExpired } : {}) }) + if (expiry !== undefined && atMs >= expiry && !resolvedExpired) { issues.push({ kind: "expired-deal", modelId, @@ -693,16 +711,16 @@ export function renderPricingReport(result: PricingCheckResult): string { lines.push("_No promotions reported._", "") } else { lines.push( - "| Model | Deal | Discount | Free | Expires | Ends when |", - "| --- | --- | --- | --- | --- | --- |", + "| Model | Deal | Discount | Free | Expires | Ends when | Review |", + "| --- | --- | --- | --- | --- | --- | --- |", ) - for (const { modelId, deal } of result.deals) { + for (const { modelId, deal, resolvedExpired } of result.deals) { lines.push( `| ${formatId(modelId)} | ${formatId(deal.id)} | ${deal.discountPercent}% | ${ deal.free ? "yes" : "no" } | ${deal.expires ? formatCell(deal.expires) : "—"} | ${ deal.endsWhen ? formatCell(deal.endsWhen) : "—" - } |`, + } | ${resolvedExpired ? "Reviewed expired deal" : "—"} |`, ) } lines.push("") diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dbfcca..b5c0f65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Correct the display input price for `stepfun/Step-3.5-Flash` to $0.09 per million tokens and add the `Qwen/Qwen3.6-Plus` long-context tier above 256K tokens, verified against the official pricing page on 2026-10-06. Mark the unchanged, expired `qwen-3.7-max-2x-usage` badge as reviewed in the pricing checker while keeping it visible in reports; changed deal terms and price drift still require review. - Restore incremental reasoning on the Provider API OpenAI Responses wire by translating Command Code's non-standard `response.reasoning.delta` events into the canonical `response.reasoning_text.delta` before they reach pi-ai, including valid trailing frames without a final blank line. Reasoning now streams while the reasoning item is still open instead of appearing only at `response.output_item.done`. Unsupported reasoning event shapes remain unchanged and produce a content-free warning in `/commandcode-status`. Reproduced with `deepseek/deepseek-v4.1-flash-fast`; standard Responses events, Chat Completions, Anthropic Messages, and the generate fallback are unchanged. - Add a read-only `npm run check:commandcode-pricing` command that compares the live Provider API catalog and the official pricing page against `MODEL_COSTS`, reporting missing local prices, changed base rates and context tiers, time-policy drift, and expired promotions. It runs in the daily catalog workflow and applicable pull requests, publishes its report to the Actions summary, and never edits prices. - Add reviewed display pricing for the October catalog additions — `claude-sonnet-5-5`, `gpt-6.1-sol` (with its 272K long-context tier), `deepseek/deepseek-v4.1-flash-fast`, and the free `inclusionai/ling-3.1-flash:free`. Apply the DeepSeek V4 weekday peak-pricing window to `deepseek/deepseek-v4.1-flash-fast`. Models absent from `MODEL_COSTS` silently fall back to a zero display cost, so the snapshot now covers all 85 advertised models. diff --git a/src/pricing.ts b/src/pricing.ts index d25e2e9..34d9f9e 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -20,7 +20,7 @@ export interface TemporaryPricing { } export const PRICING_SOURCE_URL = "https://commandcode.ai/docs/resources/pricing-limits" -export const PRICING_LAST_VERIFIED = "2026-10-05" +export const PRICING_LAST_VERIFIED = "2026-10-06" export const ZERO_MODEL_COST: CommandCodeModelCost = { input: 0, @@ -154,11 +154,17 @@ export const MODEL_COSTS: Readonly> = { cacheRead: 0.26, cacheWrite: 1.63, }, - "Qwen/Qwen3.6-Plus": { input: 0.5, output: 3, cacheRead: 0.1, cacheWrite: 0 }, + "Qwen/Qwen3.6-Plus": { + input: 0.5, + output: 3, + cacheRead: 0.1, + cacheWrite: 0, + tiers: [{ inputTokensAbove: 256_000, input: 2, output: 6, cacheRead: 0.2, cacheWrite: 0 }], + }, "meituan/LongCat-2.0": { input: 0.3, output: 1.2, cacheRead: 0.006, cacheWrite: 0 }, "stepfun/Step-5-Preview": { input: 1, output: 2.7, cacheRead: 0.05, cacheWrite: 0 }, "stepfun/Step-3.7-Flash": { input: 0.2, output: 1.15, cacheRead: 0.04, cacheWrite: 0 }, - "stepfun/Step-3.5-Flash": { input: 0.1, output: 0.3, cacheRead: 0.02, cacheWrite: 0 }, + "stepfun/Step-3.5-Flash": { input: 0.09, output: 0.3, cacheRead: 0.02, cacheWrite: 0 }, // Permanent discounted rates. "xiaomi/mimo-v2.5-pro": { input: 0.435, output: 0.87, cacheRead: 0.0036, cacheWrite: 0 }, "xiaomi/mimo-v2.5": { input: 0.14, output: 0.28, cacheRead: 0.0028, cacheWrite: 0 }, diff --git a/tests/fixtures/commandcode-pricing.json b/tests/fixtures/commandcode-pricing.json index d427e24..4924630 100644 --- a/tests/fixtures/commandcode-pricing.json +++ b/tests/fixtures/commandcode-pricing.json @@ -1,5 +1,5 @@ { - "verifiedAt": "2026-10-05", + "verifiedAt": "2026-10-06", "source": "https://commandcode.ai/docs/resources/pricing-limits", "tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", "tiers": { @@ -8,6 +8,7 @@ [32000, 0.1, 0.4, 0.02, 0.125], [256000, 0.2, 0.8, 0.04, 0.25] ], + "Qwen/Qwen3.6-Plus": [[256000, 2, 6, 0.2, 0]], "gpt-6-astra": [[272000, 20, 75, 2, 25]], "gpt-6.1-sol": [[272000, 4, 15, 0.2, 5]], "gpt-6-sol": [[272000, 4, 15, 0.4, 5]], @@ -59,7 +60,7 @@ "meituan/LongCat-2.0": [0.3, 1.2, 0.006, 0], "stepfun/Step-5-Preview": [1, 2.7, 0.05, 0], "stepfun/Step-3.7-Flash": [0.2, 1.15, 0.04, 0], - "stepfun/Step-3.5-Flash": [0.1, 0.3, 0.02, 0], + "stepfun/Step-3.5-Flash": [0.09, 0.3, 0.02, 0], "xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0], "xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0], "xiaomi/mimo-v2.6-pro": [0.435, 0.87, 0.0036, 0], diff --git a/tests/test-pricing-check.ts b/tests/test-pricing-check.ts index 7fe61f4..2e467d0 100644 --- a/tests/test-pricing-check.ts +++ b/tests/test-pricing-check.ts @@ -12,7 +12,7 @@ import { type PricingPageRow, } from "../.github/scripts/check-commandcode-pricing.ts" import { DEFAULT_MODELS_URL } from "../src/models.ts" -import { PRICING_SOURCE_URL, type CommandCodeModelCost } from "../src/pricing.ts" +import { MODEL_COSTS, PRICING_SOURCE_URL, type CommandCodeModelCost } from "../src/pricing.ts" const fixtureUrl = new URL("./fixtures/commandcode-pricing-page.html", import.meta.url) const fixtureHtml = await readFile(fixtureUrl, "utf-8") @@ -755,20 +755,52 @@ describe("promotions and deals", () => { ) }) - it("reports the real Qwen 3.7 Max expired deal even when rates match", () => { + it("marks the reviewed Qwen 3.7 Max expired deal resolved without hiding it", () => { const rows = parsePricingPage(fixtureHtml) const row = findRow(rows, "qwen-3.7-max") const result = checkCommandCodePricing( ["Qwen/Qwen3.7-Max"], rows, { "Qwen/Qwen3.7-Max": row.cost }, - Date.UTC(2026, 9, 5, 12, 0, 0), + AT_MS, ) + assert.deepEqual(result.issues, []) + assert.equal(result.deals[0]!.resolvedExpired, true) + assert.match(renderPricingReport(result), /Reviewed expired deal/) + assert.match(renderPricingReport(result), /qwen-3\.7-max-2x-usage/) + }) + + it("still flags renewed or changed Qwen deals and changed rates", () => { + const row = findRow(parsePricingPage(fixtureHtml), "qwen-3.7-max") + const modelId = "Qwen/Qwen3.7-Max" + for (const change of [ + { id: "new-deal" }, + { expires: "2026-09-30" }, + { discountPercent: 75 }, + { free: true }, + { endsWhen: "while capacity lasts" }, + ]) { + const changed = { ...row, deal: { ...row.deal!, ...change } } + const result = checkCommandCodePricing([modelId], [changed], { [modelId]: row.cost }, AT_MS) + assert.deepEqual( + result.issues.map((issue) => issue.kind), + ["expired-deal"], + ) + } + const result = checkCommandCodePricing([modelId], [row], { [modelId]: cost(1, 2, 0, 0) }, AT_MS) assert.deepEqual( result.issues.map((issue) => issue.kind), - ["expired-deal"], + ["changed-cost"], ) }) + + it("passes the reviewed pricing corrections through the CLI", async () => { + const modelIds = ["Qwen/Qwen3.6-Plus", "stepfun/Step-3.5-Flash", "Qwen/Qwen3.7-Max"] + const { impl } = fakeFetch({ models: modelsBody(modelIds), page: fixtureHtml }) + const cli = await captureCli(impl, { localCosts: MODEL_COSTS }) + assert.equal(cli.code, 0, cli.stdout) + assert.match(cli.stdout, /Reviewed expired deal/) + }) }) describe("runner and report", () => { diff --git a/tests/test-pricing.ts b/tests/test-pricing.ts index 7bce7b9..8fb01c1 100644 --- a/tests/test-pricing.ts +++ b/tests/test-pricing.ts @@ -131,6 +131,12 @@ describe("MODEL_COSTS pricing overlay", () => { cacheRead: 0.003, cacheWrite: 0, }) + assertCost("stepfun/Step-3.5-Flash", { + input: 0.09, + output: 0.3, + cacheRead: 0.02, + cacheWrite: 0, + }) assertCost("Qwen/Qwen3.7-Max", { input: 2.5, output: 7.5, @@ -200,6 +206,10 @@ describe("MODEL_COSTS pricing overlay", () => { }) it("uses the documented base rates for context-dependent models", () => { + assertCost("Qwen/Qwen3.6-Plus", { input: 0.5, output: 3, cacheRead: 0.1, cacheWrite: 0 }) + assert.deepEqual(MODEL_COSTS["Qwen/Qwen3.6-Plus"]?.tiers, [ + { inputTokensAbove: 256_000, input: 2, output: 6, cacheRead: 0.2, cacheWrite: 0 }, + ]) assertCost("Qwen/Qwen3.7-Plus", { input: 0.4, output: 1.6, @@ -312,7 +322,7 @@ describe("MODEL_COSTS pricing overlay", () => { it("tracks pricing provenance", () => { assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits") - assert.equal(PRICING_LAST_VERIFIED, "2026-10-05") + assert.equal(PRICING_LAST_VERIFIED, "2026-10-06") }) it("fails once temporary pricing needs review", () => {