diff --git a/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md b/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md index 913ba5ead3..563fc8d790 100644 --- a/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md +++ b/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md @@ -9,105 +9,326 @@ closes: "(split from #1412)" # 010 — M0-1: Model-aware input admission gate +## Audit history + +**P stale check (base `origin/dev` @ v2.17.0)** replaced three WP0 assumptions: reuse +`estimateTokens` (`src/lib/token-estimate.ts`) instead of a `chars / 4` heuristic; +measure `OcxParsedRequest.context`, not `input[]`; call `formatErrorResponse` with 3 args +(its 4th is `{code, retryAfter}`, so a metadata object would have been silently dropped — +`src/bridge.ts:1825`). + +**A round 1 — FAIL, 5 blockers. All folded.** + +| # | Blocker | Disposition | +|---|---------|-------------| +| 1 | Gate would 413 compaction-recovery turns, deadlocking the client | Folded — full bypass; reviewer confirmed both paths in round 2 | +| 2 | A heuristic estimate cannot justify "could not have succeeded upstream" | Folded — tolerance margin; claim removed | +| 3 | `candidateCapabilityEvidence` merges registry data even for same-named custom providers | Folded — read `route.provider` | +| 4 | Merged `modelMaxInputTokens` lives on `route.provider`, not raw config | Folded — read `route.provider` | +| 5 | `candidateCapabilityEvidence` does sync `existsSync`/`readFileSync`/`statSync` per call | Folded — no filesystem access on the request path | + +**A round 3 — GO-WITH-FIXES, 1 blocker, folded below. Audit loop exits here** (AUDIT-LOOP-01: +near-pass with every blocker folded). Round 3 also confirmed 2.5 is above the estimator's +maximum internal divergence — base ratio 4 over clamp 2.5 is exactly 1.6 — so no worse +CJK-mode overshoot can be constructed, and that `nativeOpenAiContextWindow` touches no +filesystem. + +| # | Blocker | Disposition | +|---|---------|-------------| +| 8 | "canonical native provider" underspecified — a custom provider named `openai` could take the native fallback | Folded — exact 3-clause predicate | + +### Blocker 8: name alone does not identify the native route + +Bare native-family routing accepts `config.providers.openai` (`src/router.ts:615`), and a +transport-mismatched custom provider named `openai` is deliberately preserved +(`src/router.ts:251`). Gating the native fallback on `providerName === "openai"` alone would +hand built-in native limits to that custom provider — reintroducing blocker 3 through the +back door. The condition is therefore all three of: + +```ts +providerName === OPENAI_CODEX_PROVIDER_ID // src/providers/openai-tiers.ts:5 + && isCanonicalOpenAiForwardProvider(provider) // adapter + forward auth + canonical base URL + && !modelId.includes("/") // bare native slug, not a routed id +``` + +Account-qualified routes stay covered: routing strips the namespace into the native model id +and keeps `providerName: "openai"` (`src/router.ts:543`), with `codexAccountNamespace` as a +separate field — so the shape predicate, not the name, is what decides. + +**A round 2 — FAIL, 2 new blockers.** Round-1 fixes 1, 3, 4, 5 verified by the reviewer as +real. Two new findings, both reproduced independently before acceptance: + +| # | Blocker | Disposition | +|---|---------|-------------| +| 6 | `ADMISSION_TOLERANCE = 1.5` is under the estimator's own 1.6x aliasing overshoot | Folded — tolerance raised to 2.5 with a measured basis | +| 7 | Dropping native metadata makes the gate inert on every native OpenAI model | Folded — pure static native fallback added | + +### Blocker 6: the estimator can overshoot 1.6x, so 1.5x was below its own error bar + +`cjkRatio` (`src/lib/token-estimate.ts:47-53`) samples with +`stride = ceil(length / 2048)` and tests only `text[i]`. When record length aligns with the +stride, every sampled character can be Korean while the text is almost entirely ASCII. +Reproduced locally on Bun 1.3.14 with a 126,046-char payload of 62-char records each +starting with one Hangul character: + +``` +{"length":126046,"stride":62,"sampledRatio":1,"trueRatio":0.0161, + "clampFires":true,"withClamp":50419,"honest":31512,"overshoot":1.6} +``` + +A payload that is **1.6% Korean samples as 100% Korean**, fires the 2.5-chars/token clamp +(`src/lib/token-estimate.ts:67`), and inflates the estimate by 1.6x. Fixed-width records +with a leading Korean label are ordinary data, not a contrived attack. + +`ADMISSION_TOLERANCE` is therefore **2.5**, not 1.5: strictly above the 1.6x demonstrated +internal divergence, with margin left for the ~10% model-family ratio spread documented at +`src/lib/token-estimate.ts:8-11`. The #1412 shape (10x inflation) still clears 2.5x by a +factor of four, so the gate keeps the case it exists for while becoming much harder to +trip by accident. + +Fixing `cjkRatio` itself is the better long-term fix and is deliberately NOT done here: +`estimateTokens` also feeds usage accounting and auto-compact +(`src/server/chat-completions.ts:140`), so changing its output is a behavior change to +unrelated subsystems and belongs in its own layer with its own tests. Recorded as +follow-up FU-1 rather than smuggled into an admission-gate PR. + +### Blocker 7: without native metadata the gate is inert where it matters most + +The canonical `openai` registry entry (`src/providers/registry.ts:937-947`) declares +`adapter`, `baseUrl`, `authKind` — and no `contextWindow`, `modelContextWindows`, or +`modelMaxInputTokens`. Native model limits live only in static metadata: +`NATIVE_OPENAI_CONTEXT_OVERRIDES` (`src/codex/catalog/metadata.ts:104`) gives `gpt-5.5` +272k, `gpt-5.4` 1M, and the GPT-5.6 family 372k. + +So a `route.provider`-only ceiling returns null for the default Codex route, and the gate +silently never fires on the most-used path. Round 1 accepted "thin evidence fails open" as +a trade; that reasoning does not survive contact with the fact that the primary route has +no evidence at all. + +The fix keeps blocker 5 intact by taking only the **static** half of what +`candidateCapabilityEvidence` consults: `nativeOpenAiContextWindow` +(`src/codex/catalog/metadata.ts:135`) reads two in-memory maps and touches no filesystem. +The catalog row — the part that costs `existsSync`/`readFileSync`/`statSync` — stays +excluded. + ## Thesis -Reject oversized requests BEFORE upstream dispatch. A 1.3M-token request that exceeds -the model's context window should never reach the provider — it wastes bandwidth, holds -a turn slot, and the provider will reject it anyway with a less useful error. +Refuse a request whose input alone cannot plausibly fit the model context window, before +spending auth, circuit budget, or upstream bandwidth on a turn the provider will reject. + +Deliberately narrow: not a context manager, not a compaction trigger. It catches the +pathological case (#1412 reported 127k of real context inflating to 1.3M-1.6M tokens) and +stays out of the way otherwise. ## Current state -- `src/server/responses/core.ts:1843` has `acquireUpstreamHostAdmission` — per-host - circuit breaker, not size-based -- `src/server/responses/core.ts:680` returns 413 for translator buffer overflow - (post-translation, not pre-dispatch) -- `src/types.ts:1340` has `contextWindow` and `modelContextWindows` on OcxProviderConfig -- `src/types.ts:1347` has `modelMaxInputTokens` on OcxProviderConfig -- No code path compares request input size against model context window before dispatch +- `src/server/responses/core.ts:1842` acquires the per-host circuit admission — failure-rate + based, not size based +- `src/server/responses/core.ts:680` and `:1552` return 413 for translator buffer overflow — + post-translation and byte-based +- `src/routing/evaluator.ts:192` compares a context window for routing ELIGIBILITY, not + request admission +- `src/server/responses/core.ts:3500` retries images one tier lower after an UPSTREAM 413 — + after dispatch, no overlap +- `route.provider` already carries merged, transport-guarded `modelContextWindows` and + `modelMaxInputTokens` (`src/router.ts:250-284`, attached at `:455`) +- `src/lib/token-estimate.ts` yields model-aware, CJK-aware estimates +- No estimated-token admission exists anywhere (reviewer-confirmed) + +## Design decisions + +### Compaction turns bypass the gate (blocker 1) + +Codex sends `{type:"compaction_trigger"}` precisely BECAUSE context is full. +413-ing it is a deadlock: the client is told to compact, and compaction is refused. + +Reviewer verified in round 2 that both paths set the flag before the gate: v2 sets +`compactionRequest = true` at `src/responses/parser.ts:355` and emits `_compactionRequest` +at `:711`; routed `/v1/responses/compact` appends the same trigger +(`src/server/responses/compact.ts:659`) and re-enters `handleResponses` at `:671`, parsed +at `src/server/responses/core.ts:1538` — before the `:1840` guard. + +### The ceiling comes from the routed provider, plus static native metadata (blockers 3, 4, 5, 7) + +`routedProviderConfig` (`src/router.ts:250`) already refuses registry merging when +`providerMatchesRegistryTransport` is false (`:252`), so a user-defined provider sharing a +built-in name keeps its own limits, and it merges registry + user caps (`:278-284`). +Reading `route.provider` gets all of that for free, as pure record lookups. + +Native models then fall back to `nativeOpenAiContextWindow`, which is static maps only. + +`min()` of the defined positive values is correct: `modelMaxInputTokens` is documented as a +per-model maximum INPUT limit (`docs-site/.../providers.md:83`) and catalog construction +already bounds it against context (`src/codex/catalog/provider-fetch.ts:800`), so the +tighter of the two is the real admission ceiling. ## File change map ### NEW: src/server/responses/input-admission.ts -Purpose: Pre-dispatch input size estimation and admission gate. - ```ts +/** + * Multiplier applied to the ceiling before refusing. + * + * 2.5, not 1.5: estimateTokens can overshoot by 1.6x on its own. cjkRatio + * (src/lib/token-estimate.ts:47) samples every `stride`-th character, so a payload of + * fixed-width records whose length aligns with the stride can sample as 100% CJK while + * being ~1.6% CJK, firing the 2.5-chars/token clamp. Measured on Bun 1.3.14: + * 126,046 chars, true CJK ratio 0.0161, sampled ratio 1.0, estimate inflated 1.6x. + * + * A threshold under that turns the estimator error bar into false 413s. 2.5 sits above + * it with room for the ~10% model-ratio spread, and still catches the #1412 case (10x). + */ +export const ADMISSION_TOLERANCE = 2.5; + export interface InputAdmissionResult { admitted: boolean; estimatedTokens: number; - contextWindow: number | null; - reason?: string; + /** Resolved ceiling, or null when unknown (=> always admitted). */ + ceiling: number | null; } /** - * Estimate token count from a parsed Responses request and compare - * against the model's advertised context window. - * - * Token estimation: count characters / 4 as a rough upper bound, - * with base64 image data counted at its decoded byte size / 750. - * This is intentionally conservative (overestimates) — better to - * reject a request that's close than to let a too-large one through. + * Estimate input tokens over the parsed request, delegating every text blob to + * estimateTokens(text, modelId) so model-aware and CJK-aware ratios apply. + * + * Covers the full OcxMessage union (src/types.ts:96), not user text only: + * - user / developer: string or OcxContentPart[] + * - assistant: OcxAssistantContentPart[] = text | thinking | toolCall, so thinking + * blocks and JSON tool-call arguments are counted, plus kiroRedactedReasoning + * - toolResult: string or OcxContentPart[] + * Tools are charged name + description + JSON.stringify(parameters): all three reach + * the upstream (src/types.ts:181). + * + * Images are not text. A data: URL is charged by DECODED byte size / 750; a remote + * https URL is charged a small fixed cost because its bytes are not in this request. */ -export function estimateInputTokens(parsed: OcxParsedRequest): number; +export function estimateInputTokens(parsed: OcxParsedRequest, modelId: string): number; /** - * Resolve the effective context window for a provider+model pair. - * Priority: modelContextWindows[model] > contextWindow > null. + * Resolve the admission ceiling. Pure: no filesystem, no catalog, no registry scan. + * + * Order: + * 1. provider.modelContextWindows[modelId] ?? provider.contextWindow + * — route.provider is routedProviderConfig output (src/router.ts:455), already + * transport-guarded and merged, so a same-named custom provider keeps its own + * limits instead of inheriting built-in ones. + * 2. nativeOpenAiContextWindow(modelId) when step 1 found nothing AND all three hold: + * providerName === OPENAI_CODEX_PROVIDER_ID + * && isCanonicalOpenAiForwardProvider(provider) + * && !modelId.includes("/") + * The `openai` registry entry carries no context fields + * (src/providers/registry.ts:937), so without this the gate is inert on the + * default Codex route. All three clauses are required: a transport-mismatched + * custom provider named "openai" is preserved verbatim by routing + * (src/router.ts:251) and must NOT inherit built-in native limits. + * Static maps only (src/codex/catalog/metadata.ts:135) — no filesystem, + * preserving blocker 5. + * + * providerContextCaps is deliberately NOT applied: it is a Codex-visible + * presentation cap (src/types.ts:790), not upstream capacity, so honoring it + * here would let a display setting cause false 413s. + * 3. Tighten by provider.modelMaxInputTokens[modelId] when present (min of defined + * positive values). + * Returns null when nothing resolves — unknown models stay admissible. */ -export function resolveContextWindow( +export function resolveInputCeiling( provider: OcxProviderConfig, - model: string, + providerName: string, + modelId: string, ): number | null; /** - * Check whether the estimated input fits within the model's context window. - * Returns admitted:true if no context window is known (fail-open for unconfigured models). + * Fail-open when no ceiling is known. Refuses only when + * estimate > ceiling * ADMISSION_TOLERANCE. + * + * Caller skips compaction turns; see the core.ts call site. */ export function checkInputAdmission( parsed: OcxParsedRequest, provider: OcxProviderConfig, - model: string, + providerName: string, + modelId: string, ): InputAdmissionResult; ``` ### MODIFY: src/server/responses/core.ts -Location: Inside `handleResponsesInner`, after route resolution and before -`acquireUpstreamHostAdmission` (around line 1840). +Location: inside `handleResponsesInner`, after `applyFinalRouteRequestNormalization` +settles the final route and before the `preAuthHostKey` circuit block (`:1840`). After +normalization the gate measures what will actually be sent, including replay expansion; +before auth and the circuit, an oversized turn costs no credential resolution and burns no +circuit budget. Reviewer confirmed `parsed`, `route`, `config`, and `formatErrorResponse` +are in scope there (`:1824-1836`, import at `:2`). ```diff -+ // Pre-dispatch input admission: reject requests whose estimated token count -+ // exceeds the resolved model context window. -+ const admission = checkInputAdmission(parsed, providerConfig, resolvedModel); -+ if (!admission.admitted) { -+ return formatErrorResponse(413, "request_too_large", -+ `Estimated input (~${admission.estimatedTokens} tokens) exceeds the model context window (${admission.contextWindow} tokens). ` -+ + `Reduce the conversation size or choose a model with a larger context window.`, -+ { estimated_tokens: admission.estimatedTokens, context_window: admission.contextWindow }); ++ // Refuse an input that cannot plausibly fit the model context window before spending ++ // auth, circuit budget, or upstream bandwidth on a turn the provider will reject anyway. ++ // ++ // Compaction turns are exempt: Codex sends compaction_trigger BECAUSE context is full ++ // (src/responses/parser.ts:355), so refusing the turn that shrinks the context would ++ // deadlock the client against the very limit this gate reports. ++ if (parsed._compactionRequest !== true) { ++ const inputAdmission = checkInputAdmission(parsed, route.provider, route.providerName, parsed.modelId); ++ if (!inputAdmission.admitted) { ++ return formatErrorResponse( ++ 413, ++ "request_too_large", ++ `Estimated input (~${inputAdmission.estimatedTokens} tokens) is far past the context ` ++ + `window of ${parsed.modelId} (${inputAdmission.ceiling} tokens). Start a new session ` ++ + `or choose a model with a larger context window.`, ++ ); ++ } + } ``` ### NEW: tests/input-admission.test.ts -Test cases: -1. Request within context window → admitted -2. Request exceeding context window → 413 with token estimates -3. No context window configured → admitted (fail-open) -4. Base64 image data counted at reduced rate -5. Tool results included in estimate -6. `modelMaxInputTokens` caps below context window -7. Edge: exactly at boundary → admitted -8. Edge: 1 token over → rejected +1. Input under the ceiling → admitted +2. Input past `ceiling * ADMISSION_TOLERANCE` → refused, reporting both numbers +3. Input over the ceiling but inside tolerance → admitted (estimator error bar) +4. No ceiling resolvable → admitted (fail-open) +5. `modelMaxInputTokens` tightens the ceiling below the context window +6. A same-named custom provider uses its OWN limits, not registry limits (blocker 3) +6b. **A custom provider named `openai` with no limits does NOT get the native fallback (blocker 8)** +7. **Native `gpt-5.6-sol` resolves 372k from static metadata (blocker 7)** +7b. **An account-namespaced native route still resolves the native ceiling (blocker 8)** +7c. **A routed `provider/model` id does not take the native fallback (blocker 8)** +8. **The stride-aliased 1.6x CJK payload does NOT trip the gate at its true size (blocker 6)** +9. Assistant thinking blocks and tool-call arguments are counted +10. Tool name + description + parameters all counted +11. A `data:` image is charged by decoded size, not URL character length +12. A remote https image is charged a small fixed cost +13. **A `_compactionRequest` turn is admitted no matter how large (blocker 1)** +14. Ceiling resolution performs no filesystem access (blocker 5) ## Activation scenario -A Codex turn with 500k tokens of conversation history targeting a model with a -128k context window hits `checkInputAdmission` → returns `admitted: false` → -413 response returned to client before any upstream fetch. +A turn carrying ~1.3M estimated tokens (the #1412 shape) targets a 372k-window native +model. `1_300_000 > 372_000 * 2.5 = 930_000`, so the client gets 413 before any upstream +fetch. + +Observable proof required in C: the focused test asserts the 413 AND that no upstream fetch +was attempted — the test installs a fetch that throws if called, so a gate placed after +dispatch fails rather than passing quietly. + +Blocker-1 proof: a `_compactionRequest` turn of the same size returns non-413. +Blocker-6 proof: the aliased payload from the probe above is admitted. + +## Follow-ups (not this layer) + +- **FU-1: fix `cjkRatio` periodic aliasing** (`src/lib/token-estimate.ts:47`). Sampling + every `stride`-th character aliases against fixed-width records. `estimateTokens` also + feeds usage accounting and auto-compact (`src/server/chat-completions.ts:140`), so this + is a behavior change to unrelated subsystems and needs its own layer and tests. Until it + lands, `ADMISSION_TOLERANCE` absorbs the error. ## Scope boundary -IN: New admission module + core.ts insertion point + test file -OUT: Changing existing translator buffer limits, modifying provider configs, - adding UI for admission settings +IN: the admission module, its single guarded call site, the test file +OUT: the translator buffer limit, provider config defaults, GUI surface, any second copy of +context-window resolution, changing `estimateTokens` behavior (FU-1), and making +`src/routing/capability.ts` transport-aware (sidestepped by reading `route.provider`), and +applying `providerContextCaps` to admission + diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index fa09bf2e6c..3b9c4c8349 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1,6 +1,7 @@ import type { Server } from "bun"; import { bridgeToResponsesSSE, buildResponseJSON, formatErrorResponse, type ResponsesTerminalStatus } from "../../bridge"; import { formatPassthroughUpstreamError } from "./passthrough-error"; +import { checkInputAdmission } from "./input-admission"; import { describeUpstreamConnectFailure } from "./upstream-error"; import { getConfigPath, @@ -1837,6 +1838,24 @@ async function handleResponsesInner( } if (options.abortSignal?.aborted) return clientCancelledResponse(); + // Refuse an input that cannot plausibly fit the model context window before spending auth, + // circuit budget, or upstream bandwidth on a turn the provider will reject anyway (#1412). + // + // Compaction turns are exempt: Codex sends compaction_trigger BECAUSE context is full, so + // refusing the turn that shrinks the context would deadlock the client against the very + // limit this gate reports — it would be told to compact and then denied the compaction. + if (parsed._compactionRequest !== true) { + const inputAdmission = checkInputAdmission(parsed, route.provider, route.providerName, parsed.modelId); + if (!inputAdmission.admitted) { + return formatErrorResponse( + 413, + "request_too_large", + `Estimated input (~${inputAdmission.estimatedTokens} tokens) is far past the context window ` + + `of ${parsed.modelId} (${inputAdmission.ceiling} tokens). Start a new session or choose a ` + + `model with a larger context window.`, + ); + } + } const preAuthHostKey = preAuthUpstreamHostCircuitKey(route, config); if (preAuthHostKey) { const admission = acquireUpstreamHostAdmission( diff --git a/src/server/responses/input-admission.ts b/src/server/responses/input-admission.ts new file mode 100644 index 0000000000..af00d91200 --- /dev/null +++ b/src/server/responses/input-admission.ts @@ -0,0 +1,169 @@ +/** + * Pre-dispatch input admission (#1412). + * + * Refuses a turn whose estimated input cannot plausibly fit the model context window, + * BEFORE auth resolution, circuit admission, or any upstream I/O. #1412 reported ~127k of + * real context compounding to 1.3M-1.6M tokens and crashing the proxy; the provider would + * reject such a turn anyway, so paying for the round trip buys nothing. + * + * Deliberately narrow. This is not a context manager and not a compaction trigger: it + * catches the pathological case and stays out of the way otherwise. Every uncertainty + * resolves toward admitting. + */ +import { nativeOpenAiContextWindow } from "../../codex/catalog/metadata"; +import { estimateTokens } from "../../lib/token-estimate"; +import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; +import type { OcxContentPart, OcxParsedRequest, OcxProviderConfig } from "../../types"; + +/** + * Multiplier applied to the ceiling before refusing. + * + * 2.5, not something tighter, because `estimateTokens` can overshoot by 1.6x on its own. + * `cjkRatio` samples every `stride`-th character, so a payload of fixed-width records whose + * length aligns with the stride samples as 100% CJK while being ~1.6% CJK, firing the + * 2.5-chars/token clamp instead of the 4.0 default. Measured on Bun 1.3.14: 126,046 chars, + * true CJK ratio 0.0161, sampled ratio 1.0, estimate inflated 1.6x. Since 4.0 / 2.5 = 1.6 + * is that branch maximum divergence, a threshold at or under 1.6 would convert the + * estimator error bar into false 413s. + * + * 2.5 sits above it with room for the ~10% model-family ratio spread, and still refuses the + * #1412 shape (10x) four times over. + */ +export const ADMISSION_TOLERANCE = 2.5; + +/** + * Token cost charged for a remote image URL. The bytes are not in this request — the + * provider fetches them — so the URL own length is not the cost. A small flat charge + * acknowledges the tiles the image will occupy without pretending to know its dimensions. + */ +const REMOTE_IMAGE_TOKENS = 850; + +/** Decoded image bytes per token. Coarse tile-count proxy, not a provider formula. */ +const IMAGE_BYTES_PER_TOKEN = 750; + +export interface InputAdmissionResult { + admitted: boolean; + estimatedTokens: number; + /** Resolved ceiling, or null when nothing could be resolved (=> always admitted). */ + ceiling: number | null; +} + +function positive(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; +} + +/** + * Charge a `data:` URL by its DECODED size rather than its character length: base64 inflates + * by 4/3, so charging the string would overcount by a third. A remote URL is charged flat. + */ +function imageTokens(imageUrl: string): number { + if (!imageUrl.startsWith("data:")) return REMOTE_IMAGE_TOKENS; + const comma = imageUrl.indexOf(","); + if (comma < 0) return REMOTE_IMAGE_TOKENS; + const payload = imageUrl.length - comma - 1; + if (payload <= 0) return 0; + const decoded = Math.floor((payload * 3) / 4); + return Math.max(1, Math.ceil(decoded / IMAGE_BYTES_PER_TOKEN)); +} + +function contentPartTokens(part: OcxContentPart, modelId: string): number { + return part.type === "image" ? imageTokens(part.imageUrl) : estimateTokens(part.text, modelId); +} + +function contentTokens(content: string | readonly OcxContentPart[], modelId: string): number { + if (typeof content === "string") return estimateTokens(content, modelId); + let total = 0; + for (const part of content) total += contentPartTokens(part, modelId); + return total; +} + +/** + * Estimate the input tokens of a parsed request. + * + * Walks the whole `OcxMessage` union rather than user text alone. Assistant turns carry + * their content as `OcxAssistantContentPart[]` — text, thinking blocks, and tool calls whose + * JSON arguments are frequently the largest single item in an agent conversation. A walk + * that counted only `{type:"text"}` would undercount exactly the turns that trigger this + * gate. + */ +export function estimateInputTokens(parsed: OcxParsedRequest, modelId: string): number { + const { context } = parsed; + let total = 0; + + for (const prompt of context.systemPrompt ?? []) total += estimateTokens(prompt, modelId); + + for (const message of context.messages) { + if (message.role === "assistant") { + for (const part of message.content) { + if (part.type === "text") total += estimateTokens(part.text, modelId); + else if (part.type === "thinking") total += estimateTokens(part.thinking, modelId); + else total += estimateTokens(part.name, modelId) + estimateTokens(JSON.stringify(part.arguments), modelId); + } + // Opaque provider blob replayed verbatim upstream, so it costs real input tokens. + if (message.kiroRedactedReasoning) total += estimateTokens(message.kiroRedactedReasoning, modelId); + continue; + } + total += contentTokens(message.content, modelId); + } + + // Tool schemas ride every turn: name, description, and the JSON parameter schema all + // reach the upstream, and a large MCP catalog can dominate a short conversation. + for (const tool of context.tools ?? []) { + total += estimateTokens(tool.name, modelId) + + estimateTokens(tool.description, modelId) + + estimateTokens(JSON.stringify(tool.parameters), modelId); + } + + return total; +} + +/** + * Resolve the admission ceiling. Pure: no filesystem, no catalog, no registry scan. + * + * `provider` must be the ROUTED config (`route.provider`), which `routedProviderConfig` + * has already transport-guarded and merged. Re-deriving from `config.providers[name]` would + * reject a user-defined provider that merely shares a built-in name using limits that + * belong to a different service. + */ +export function resolveInputCeiling( + provider: OcxProviderConfig, + providerName: string, + modelId: string, +): number | null { + const configured = positive(provider.modelContextWindows?.[modelId]) ?? positive(provider.contextWindow); + + // The canonical `openai` registry entry declares no context fields, so without this the + // gate would be inert on the default Codex route. All three clauses are load-bearing: a + // transport-mismatched custom provider named "openai" is preserved verbatim by routing + // and must not inherit built-in native limits, and a routed `provider/model` id is not a + // native slug. Static maps only — no catalog read. + const native = configured === null + && providerName === OPENAI_CODEX_PROVIDER_ID + && isCanonicalOpenAiForwardProvider(provider) + && !modelId.includes("/") + ? positive(nativeOpenAiContextWindow(modelId)) + : null; + + const window = configured ?? native; + // modelMaxInputTokens is an input-only cap, so it can only tighten the window. + const maxInput = positive(provider.modelMaxInputTokens?.[modelId]); + if (window === null) return maxInput; + return maxInput === null ? window : Math.min(window, maxInput); +} + +/** + * Fail-open when no ceiling is known; refuse only past `ceiling * ADMISSION_TOLERANCE`. + * + * The caller is responsible for skipping compaction turns — see the call site in core.ts. + */ +export function checkInputAdmission( + parsed: OcxParsedRequest, + provider: OcxProviderConfig, + providerName: string, + modelId: string, +): InputAdmissionResult { + const ceiling = resolveInputCeiling(provider, providerName, modelId); + if (ceiling === null) return { admitted: true, estimatedTokens: 0, ceiling: null }; + const estimatedTokens = estimateInputTokens(parsed, modelId); + return { admitted: estimatedTokens <= ceiling * ADMISSION_TOLERANCE, estimatedTokens, ceiling }; +} diff --git a/tests/input-admission.test.ts b/tests/input-admission.test.ts new file mode 100644 index 0000000000..3669d46122 --- /dev/null +++ b/tests/input-admission.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, test } from "bun:test"; +import { + ADMISSION_TOLERANCE, + checkInputAdmission, + estimateInputTokens, + resolveInputCeiling, +} from "../src/server/responses/input-admission"; +import type { OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTool } from "../src/types"; + +const CANONICAL_NATIVE: OcxProviderConfig = { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", +}; + +function request(messages: OcxMessage[], tools?: OcxTool[]): OcxParsedRequest { + return { + modelId: "test-model", + context: { messages, ...(tools ? { tools } : {}) }, + stream: false, + options: {}, + } as OcxParsedRequest; +} + +function userText(text: string): OcxMessage { + return { role: "user", content: text, timestamp: 0 }; +} + +/** Roughly `tokens` worth of plain ASCII at the default 4 chars/token ratio. */ +function asciiTokens(tokens: number): string { + return "a".repeat(tokens * 4); +} + +describe("resolveInputCeiling", () => { + test("prefers the per-model window over the provider-wide one", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + contextWindow: 8_000, + modelContextWindows: { "m": 32_000 }, + }; + expect(resolveInputCeiling(provider, "custom", "m")).toBe(32_000); + expect(resolveInputCeiling(provider, "custom", "other")).toBe(8_000); + }); + + test("modelMaxInputTokens can only tighten the window", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": 32_000 }, + modelMaxInputTokens: { "m": 20_000 }, + }; + expect(resolveInputCeiling(provider, "custom", "m")).toBe(20_000); + }); + + test("a looser modelMaxInputTokens never widens the window", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": 32_000 }, + modelMaxInputTokens: { "m": 90_000 }, + }; + expect(resolveInputCeiling(provider, "custom", "m")).toBe(32_000); + }); + + test("returns null when nothing is configured", () => { + const provider: OcxProviderConfig = { adapter: "openai-chat", baseUrl: "https://example.test/v1" }; + expect(resolveInputCeiling(provider, "custom", "m")).toBeNull(); + }); + + test("resolves the native window from static metadata for a canonical route", () => { + // The `openai` registry entry carries no context fields, so without the native + // fallback the gate would be inert on the default Codex route. + expect(resolveInputCeiling(CANONICAL_NATIVE, "openai", "gpt-5.6-sol")).toBe(372_000); + }); + + test("a custom provider merely NAMED openai does not inherit native limits", () => { + const impostor: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://impostor.test/v1", + authMode: "key", + }; + expect(resolveInputCeiling(impostor, "openai", "gpt-5.6-sol")).toBeNull(); + }); + + test("a routed provider/model id does not take the native fallback", () => { + expect(resolveInputCeiling(CANONICAL_NATIVE, "openai", "vendor/gpt-5.6-sol")).toBeNull(); + }); + + test("an explicit user window still wins over native metadata", () => { + const pinned: OcxProviderConfig = { ...CANONICAL_NATIVE, modelContextWindows: { "gpt-5.6-sol": 50_000 } }; + expect(resolveInputCeiling(pinned, "openai", "gpt-5.6-sol")).toBe(50_000); + }); +}); + +describe("estimateInputTokens", () => { + test("counts assistant thinking blocks and tool-call arguments", () => { + // A walk that only counted {type:"text"} would report ~0 here, which is exactly the + // shape of an agent conversation that triggers this gate. + const parsed = request([{ + role: "assistant", + timestamp: 0, + content: [ + { type: "thinking", thinking: asciiTokens(500) }, + { type: "toolCall", id: "c1", name: "apply_patch", arguments: { patch: asciiTokens(500) } }, + ], + }]); + expect(estimateInputTokens(parsed, "test-model")).toBeGreaterThan(900); + }); + + test("counts tool name, description, and parameter schema", () => { + const bare = estimateInputTokens(request([userText("hi")]), "test-model"); + const withTools = estimateInputTokens(request([userText("hi")], [{ + name: "search", + description: asciiTokens(300), + parameters: { type: "object", properties: { q: { type: "string", description: asciiTokens(300) } } }, + }]), "test-model"); + expect(withTools - bare).toBeGreaterThan(500); + }); + + test("charges a data: image by decoded size, not URL length", () => { + // base64 inflates by 4/3, so charging the string would overcount by a third. + const base64 = "A".repeat(75_000); + const parsed = request([{ + role: "user", + timestamp: 0, + content: [{ type: "image", imageUrl: `data:image/png;base64,${base64}` }], + }]); + const decoded = Math.floor((75_000 * 3) / 4); + expect(estimateInputTokens(parsed, "test-model")).toBe(Math.ceil(decoded / 750)); + }); + + test("charges a remote image a flat cost, not its URL length", () => { + const short = request([{ + role: "user", + timestamp: 0, + content: [{ type: "image", imageUrl: "https://example.test/a.png" }], + }]); + const long = request([{ + role: "user", + timestamp: 0, + content: [{ type: "image", imageUrl: `https://example.test/${"b".repeat(4_000)}.png` }], + }]); + expect(estimateInputTokens(short, "test-model")).toBe(estimateInputTokens(long, "test-model")); + }); + + test("Korean text costs more than the same number of ASCII characters", () => { + const korean = estimateInputTokens(request([userText("한".repeat(4_000))]), "test-model"); + const ascii = estimateInputTokens(request([userText("a".repeat(4_000))]), "test-model"); + expect(korean).toBeGreaterThan(ascii); + }); +}); + +describe("checkInputAdmission", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": 10_000 }, + }; + + test("admits input under the ceiling", () => { + const result = checkInputAdmission(request([userText(asciiTokens(5_000))]), provider, "custom", "m"); + expect(result.admitted).toBe(true); + expect(result.ceiling).toBe(10_000); + }); + + test("admits input over the ceiling but inside the tolerance", () => { + // The estimator has a real error bar, so 1.0x-2.5x is deliberately not refused. + const result = checkInputAdmission(request([userText(asciiTokens(15_000))]), provider, "custom", "m"); + expect(result.admitted).toBe(true); + }); + + test("refuses input past the tolerance", () => { + const result = checkInputAdmission(request([userText(asciiTokens(40_000))]), provider, "custom", "m"); + expect(result.admitted).toBe(false); + expect(result.ceiling).toBe(10_000); + expect(result.estimatedTokens).toBeGreaterThan(10_000 * ADMISSION_TOLERANCE); + }); + + test("admits everything when no ceiling resolves", () => { + const unknown: OcxProviderConfig = { adapter: "openai-chat", baseUrl: "https://example.test/v1" }; + const result = checkInputAdmission(request([userText(asciiTokens(5_000_000))]), unknown, "custom", "m"); + expect(result.admitted).toBe(true); + expect(result.ceiling).toBeNull(); + }); + + test("the estimator CJK sampling alias does not trip the gate", () => { + // cjkRatio samples every stride-th character. A payload of fixed-width records whose + // length aligns with the stride samples as 100% CJK while being ~1.6% CJK, inflating + // the estimate by 1.6x. ADMISSION_TOLERANCE exists to absorb exactly this; a payload + // whose HONEST size fits must still be admitted. + const record = "\uAC00" + "x".repeat(61); + const text = record.repeat(2_033); + const honestTokens = Math.ceil(text.length / 4); + const aliased: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": honestTokens }, + }; + const result = checkInputAdmission(request([userText(text)]), aliased, "custom", "m"); + expect(result.estimatedTokens).toBeGreaterThan(honestTokens); // the overshoot is real + expect(result.admitted).toBe(true); // and absorbed + }); + + test("resolving a ceiling touches no filesystem", () => { + // The native fallback must read static maps only; a catalog read here would put + // synchronous file I/O on every request. + const fs = require("node:fs"); + const watched = ["readFileSync", "existsSync", "statSync"] as const; + const originals = watched.map(name => [name, fs[name]] as const); + let calls = 0; + for (const [name, fn] of originals) { + fs[name] = (...args: unknown[]) => { calls += 1; return (fn as (...a: unknown[]) => unknown)(...args); }; + } + try { + resolveInputCeiling(CANONICAL_NATIVE, "openai", "gpt-5.6-sol"); + } finally { + for (const [name, fn] of originals) fs[name] = fn; + } + expect(calls).toBe(0); + }); +});