From 0298177c992e5b45ca8f8482ffb014e425d07c6a Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Fri, 14 Aug 2026 22:00:44 +0900 Subject: [PATCH] fix(responses): refuse input that cannot fit the model context window #1412 reported ~127k of real context compounding to 1.3M-1.6M tokens and crashing the proxy. A turn that large is rejected upstream anyway, so the round trip buys nothing: it spends auth resolution, host-circuit budget, and bandwidth to arrive at a worse error than we can produce locally. The gate runs after final route normalization, so it measures what will actually be sent including replay expansion, and before auth and the circuit, so an oversized turn costs neither. Three things it deliberately does not do. It does not refuse compaction turns. Codex sends compaction_trigger BECAUSE context is full, and routed /v1/responses/compact re-enters the same handler with that trigger appended. Refusing them would tell the client to compact and then deny the compaction. It does not re-derive the context window. route.provider is already the routedProviderConfig output, which refuses registry merging when the transport does not match, so a user provider that merely shares a built-in name keeps its own limits. Native models fall back to nativeOpenAiContextWindow, which reads static maps only -- the canonical openai registry entry declares no context fields, so without that fallback the gate would be inert on the default route. Reading the Codex catalog was rejected: it costs existsSync + readFileSync + statSync even on a cache hit, and this is the request path. It does not treat the estimate as exact. estimateTokens is a char-ratio heuristic whose CJK sampling aliases: a payload of 62-char records each starting with one Hangul character samples as 100% CJK while being 1.6% CJK, inflating the estimate 1.6x (measured: 126,046 chars, true ratio 0.0161). Since 4.0/2.5 is exactly 1.6, that is the branch's maximum divergence, so the 2.5x tolerance sits above it and #1412's 10x still clears it fourfold. The regression test pins that payload as admissible. Repairing cjkRatio is left alone on purpose -- estimateTokens also feeds usage accounting and auto-compact, so it needs its own change and its own tests. Verified: bun test tests/input-admission.test.ts (19 pass), bun run typecheck, and tests/core-lab-boundary.test.ts still green for the new core.ts import. --- .../010_m0_1_input_admission.md | 329 +++++++++++++++--- src/server/responses/core.ts | 19 + src/server/responses/input-admission.ts | 169 +++++++++ tests/input-admission.test.ts | 222 ++++++++++++ 4 files changed, 685 insertions(+), 54 deletions(-) create mode 100644 src/server/responses/input-admission.ts create mode 100644 tests/input-admission.test.ts diff --git a/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md b/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md index 913ba5ead3..563fc8d790 100644 --- a/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md +++ b/devlog/_plan/260814_usage_memory_roadmap/010_m0_1_input_admission.md @@ -9,105 +9,326 @@ closes: "(split from #1412)" # 010 — M0-1: Model-aware input admission gate +## Audit history + +**P stale check (base `origin/dev` @ v2.17.0)** replaced three WP0 assumptions: reuse +`estimateTokens` (`src/lib/token-estimate.ts`) instead of a `chars / 4` heuristic; +measure `OcxParsedRequest.context`, not `input[]`; call `formatErrorResponse` with 3 args +(its 4th is `{code, retryAfter}`, so a metadata object would have been silently dropped — +`src/bridge.ts:1825`). + +**A round 1 — FAIL, 5 blockers. All folded.** + +| # | Blocker | Disposition | +|---|---------|-------------| +| 1 | Gate would 413 compaction-recovery turns, deadlocking the client | Folded — full bypass; reviewer confirmed both paths in round 2 | +| 2 | A heuristic estimate cannot justify "could not have succeeded upstream" | Folded — tolerance margin; claim removed | +| 3 | `candidateCapabilityEvidence` merges registry data even for same-named custom providers | Folded — read `route.provider` | +| 4 | Merged `modelMaxInputTokens` lives on `route.provider`, not raw config | Folded — read `route.provider` | +| 5 | `candidateCapabilityEvidence` does sync `existsSync`/`readFileSync`/`statSync` per call | Folded — no filesystem access on the request path | + +**A round 3 — GO-WITH-FIXES, 1 blocker, folded below. Audit loop exits here** (AUDIT-LOOP-01: +near-pass with every blocker folded). Round 3 also confirmed 2.5 is above the estimator's +maximum internal divergence — base ratio 4 over clamp 2.5 is exactly 1.6 — so no worse +CJK-mode overshoot can be constructed, and that `nativeOpenAiContextWindow` touches no +filesystem. + +| # | Blocker | Disposition | +|---|---------|-------------| +| 8 | "canonical native provider" underspecified — a custom provider named `openai` could take the native fallback | Folded — exact 3-clause predicate | + +### Blocker 8: name alone does not identify the native route + +Bare native-family routing accepts `config.providers.openai` (`src/router.ts:615`), and a +transport-mismatched custom provider named `openai` is deliberately preserved +(`src/router.ts:251`). Gating the native fallback on `providerName === "openai"` alone would +hand built-in native limits to that custom provider — reintroducing blocker 3 through the +back door. The condition is therefore all three of: + +```ts +providerName === OPENAI_CODEX_PROVIDER_ID // src/providers/openai-tiers.ts:5 + && isCanonicalOpenAiForwardProvider(provider) // adapter + forward auth + canonical base URL + && !modelId.includes("/") // bare native slug, not a routed id +``` + +Account-qualified routes stay covered: routing strips the namespace into the native model id +and keeps `providerName: "openai"` (`src/router.ts:543`), with `codexAccountNamespace` as a +separate field — so the shape predicate, not the name, is what decides. + +**A round 2 — FAIL, 2 new blockers.** Round-1 fixes 1, 3, 4, 5 verified by the reviewer as +real. Two new findings, both reproduced independently before acceptance: + +| # | Blocker | Disposition | +|---|---------|-------------| +| 6 | `ADMISSION_TOLERANCE = 1.5` is under the estimator's own 1.6x aliasing overshoot | Folded — tolerance raised to 2.5 with a measured basis | +| 7 | Dropping native metadata makes the gate inert on every native OpenAI model | Folded — pure static native fallback added | + +### Blocker 6: the estimator can overshoot 1.6x, so 1.5x was below its own error bar + +`cjkRatio` (`src/lib/token-estimate.ts:47-53`) samples with +`stride = ceil(length / 2048)` and tests only `text[i]`. When record length aligns with the +stride, every sampled character can be Korean while the text is almost entirely ASCII. +Reproduced locally on Bun 1.3.14 with a 126,046-char payload of 62-char records each +starting with one Hangul character: + +``` +{"length":126046,"stride":62,"sampledRatio":1,"trueRatio":0.0161, + "clampFires":true,"withClamp":50419,"honest":31512,"overshoot":1.6} +``` + +A payload that is **1.6% Korean samples as 100% Korean**, fires the 2.5-chars/token clamp +(`src/lib/token-estimate.ts:67`), and inflates the estimate by 1.6x. Fixed-width records +with a leading Korean label are ordinary data, not a contrived attack. + +`ADMISSION_TOLERANCE` is therefore **2.5**, not 1.5: strictly above the 1.6x demonstrated +internal divergence, with margin left for the ~10% model-family ratio spread documented at +`src/lib/token-estimate.ts:8-11`. The #1412 shape (10x inflation) still clears 2.5x by a +factor of four, so the gate keeps the case it exists for while becoming much harder to +trip by accident. + +Fixing `cjkRatio` itself is the better long-term fix and is deliberately NOT done here: +`estimateTokens` also feeds usage accounting and auto-compact +(`src/server/chat-completions.ts:140`), so changing its output is a behavior change to +unrelated subsystems and belongs in its own layer with its own tests. Recorded as +follow-up FU-1 rather than smuggled into an admission-gate PR. + +### Blocker 7: without native metadata the gate is inert where it matters most + +The canonical `openai` registry entry (`src/providers/registry.ts:937-947`) declares +`adapter`, `baseUrl`, `authKind` — and no `contextWindow`, `modelContextWindows`, or +`modelMaxInputTokens`. Native model limits live only in static metadata: +`NATIVE_OPENAI_CONTEXT_OVERRIDES` (`src/codex/catalog/metadata.ts:104`) gives `gpt-5.5` +272k, `gpt-5.4` 1M, and the GPT-5.6 family 372k. + +So a `route.provider`-only ceiling returns null for the default Codex route, and the gate +silently never fires on the most-used path. Round 1 accepted "thin evidence fails open" as +a trade; that reasoning does not survive contact with the fact that the primary route has +no evidence at all. + +The fix keeps blocker 5 intact by taking only the **static** half of what +`candidateCapabilityEvidence` consults: `nativeOpenAiContextWindow` +(`src/codex/catalog/metadata.ts:135`) reads two in-memory maps and touches no filesystem. +The catalog row — the part that costs `existsSync`/`readFileSync`/`statSync` — stays +excluded. + ## Thesis -Reject oversized requests BEFORE upstream dispatch. A 1.3M-token request that exceeds -the model's context window should never reach the provider — it wastes bandwidth, holds -a turn slot, and the provider will reject it anyway with a less useful error. +Refuse a request whose input alone cannot plausibly fit the model context window, before +spending auth, circuit budget, or upstream bandwidth on a turn the provider will reject. + +Deliberately narrow: not a context manager, not a compaction trigger. It catches the +pathological case (#1412 reported 127k of real context inflating to 1.3M-1.6M tokens) and +stays out of the way otherwise. ## Current state -- `src/server/responses/core.ts:1843` has `acquireUpstreamHostAdmission` — per-host - circuit breaker, not size-based -- `src/server/responses/core.ts:680` returns 413 for translator buffer overflow - (post-translation, not pre-dispatch) -- `src/types.ts:1340` has `contextWindow` and `modelContextWindows` on OcxProviderConfig -- `src/types.ts:1347` has `modelMaxInputTokens` on OcxProviderConfig -- No code path compares request input size against model context window before dispatch +- `src/server/responses/core.ts:1842` acquires the per-host circuit admission — failure-rate + based, not size based +- `src/server/responses/core.ts:680` and `:1552` return 413 for translator buffer overflow — + post-translation and byte-based +- `src/routing/evaluator.ts:192` compares a context window for routing ELIGIBILITY, not + request admission +- `src/server/responses/core.ts:3500` retries images one tier lower after an UPSTREAM 413 — + after dispatch, no overlap +- `route.provider` already carries merged, transport-guarded `modelContextWindows` and + `modelMaxInputTokens` (`src/router.ts:250-284`, attached at `:455`) +- `src/lib/token-estimate.ts` yields model-aware, CJK-aware estimates +- No estimated-token admission exists anywhere (reviewer-confirmed) + +## Design decisions + +### Compaction turns bypass the gate (blocker 1) + +Codex sends `{type:"compaction_trigger"}` precisely BECAUSE context is full. +413-ing it is a deadlock: the client is told to compact, and compaction is refused. + +Reviewer verified in round 2 that both paths set the flag before the gate: v2 sets +`compactionRequest = true` at `src/responses/parser.ts:355` and emits `_compactionRequest` +at `:711`; routed `/v1/responses/compact` appends the same trigger +(`src/server/responses/compact.ts:659`) and re-enters `handleResponses` at `:671`, parsed +at `src/server/responses/core.ts:1538` — before the `:1840` guard. + +### The ceiling comes from the routed provider, plus static native metadata (blockers 3, 4, 5, 7) + +`routedProviderConfig` (`src/router.ts:250`) already refuses registry merging when +`providerMatchesRegistryTransport` is false (`:252`), so a user-defined provider sharing a +built-in name keeps its own limits, and it merges registry + user caps (`:278-284`). +Reading `route.provider` gets all of that for free, as pure record lookups. + +Native models then fall back to `nativeOpenAiContextWindow`, which is static maps only. + +`min()` of the defined positive values is correct: `modelMaxInputTokens` is documented as a +per-model maximum INPUT limit (`docs-site/.../providers.md:83`) and catalog construction +already bounds it against context (`src/codex/catalog/provider-fetch.ts:800`), so the +tighter of the two is the real admission ceiling. ## File change map ### NEW: src/server/responses/input-admission.ts -Purpose: Pre-dispatch input size estimation and admission gate. - ```ts +/** + * Multiplier applied to the ceiling before refusing. + * + * 2.5, not 1.5: estimateTokens can overshoot by 1.6x on its own. cjkRatio + * (src/lib/token-estimate.ts:47) samples every `stride`-th character, so a payload of + * fixed-width records whose length aligns with the stride can sample as 100% CJK while + * being ~1.6% CJK, firing the 2.5-chars/token clamp. Measured on Bun 1.3.14: + * 126,046 chars, true CJK ratio 0.0161, sampled ratio 1.0, estimate inflated 1.6x. + * + * A threshold under that turns the estimator error bar into false 413s. 2.5 sits above + * it with room for the ~10% model-ratio spread, and still catches the #1412 case (10x). + */ +export const ADMISSION_TOLERANCE = 2.5; + export interface InputAdmissionResult { admitted: boolean; estimatedTokens: number; - contextWindow: number | null; - reason?: string; + /** Resolved ceiling, or null when unknown (=> always admitted). */ + ceiling: number | null; } /** - * Estimate token count from a parsed Responses request and compare - * against the model's advertised context window. - * - * Token estimation: count characters / 4 as a rough upper bound, - * with base64 image data counted at its decoded byte size / 750. - * This is intentionally conservative (overestimates) — better to - * reject a request that's close than to let a too-large one through. + * Estimate input tokens over the parsed request, delegating every text blob to + * estimateTokens(text, modelId) so model-aware and CJK-aware ratios apply. + * + * Covers the full OcxMessage union (src/types.ts:96), not user text only: + * - user / developer: string or OcxContentPart[] + * - assistant: OcxAssistantContentPart[] = text | thinking | toolCall, so thinking + * blocks and JSON tool-call arguments are counted, plus kiroRedactedReasoning + * - toolResult: string or OcxContentPart[] + * Tools are charged name + description + JSON.stringify(parameters): all three reach + * the upstream (src/types.ts:181). + * + * Images are not text. A data: URL is charged by DECODED byte size / 750; a remote + * https URL is charged a small fixed cost because its bytes are not in this request. */ -export function estimateInputTokens(parsed: OcxParsedRequest): number; +export function estimateInputTokens(parsed: OcxParsedRequest, modelId: string): number; /** - * Resolve the effective context window for a provider+model pair. - * Priority: modelContextWindows[model] > contextWindow > null. + * Resolve the admission ceiling. Pure: no filesystem, no catalog, no registry scan. + * + * Order: + * 1. provider.modelContextWindows[modelId] ?? provider.contextWindow + * — route.provider is routedProviderConfig output (src/router.ts:455), already + * transport-guarded and merged, so a same-named custom provider keeps its own + * limits instead of inheriting built-in ones. + * 2. nativeOpenAiContextWindow(modelId) when step 1 found nothing AND all three hold: + * providerName === OPENAI_CODEX_PROVIDER_ID + * && isCanonicalOpenAiForwardProvider(provider) + * && !modelId.includes("/") + * The `openai` registry entry carries no context fields + * (src/providers/registry.ts:937), so without this the gate is inert on the + * default Codex route. All three clauses are required: a transport-mismatched + * custom provider named "openai" is preserved verbatim by routing + * (src/router.ts:251) and must NOT inherit built-in native limits. + * Static maps only (src/codex/catalog/metadata.ts:135) — no filesystem, + * preserving blocker 5. + * + * providerContextCaps is deliberately NOT applied: it is a Codex-visible + * presentation cap (src/types.ts:790), not upstream capacity, so honoring it + * here would let a display setting cause false 413s. + * 3. Tighten by provider.modelMaxInputTokens[modelId] when present (min of defined + * positive values). + * Returns null when nothing resolves — unknown models stay admissible. */ -export function resolveContextWindow( +export function resolveInputCeiling( provider: OcxProviderConfig, - model: string, + providerName: string, + modelId: string, ): number | null; /** - * Check whether the estimated input fits within the model's context window. - * Returns admitted:true if no context window is known (fail-open for unconfigured models). + * Fail-open when no ceiling is known. Refuses only when + * estimate > ceiling * ADMISSION_TOLERANCE. + * + * Caller skips compaction turns; see the core.ts call site. */ export function checkInputAdmission( parsed: OcxParsedRequest, provider: OcxProviderConfig, - model: string, + providerName: string, + modelId: string, ): InputAdmissionResult; ``` ### MODIFY: src/server/responses/core.ts -Location: Inside `handleResponsesInner`, after route resolution and before -`acquireUpstreamHostAdmission` (around line 1840). +Location: inside `handleResponsesInner`, after `applyFinalRouteRequestNormalization` +settles the final route and before the `preAuthHostKey` circuit block (`:1840`). After +normalization the gate measures what will actually be sent, including replay expansion; +before auth and the circuit, an oversized turn costs no credential resolution and burns no +circuit budget. Reviewer confirmed `parsed`, `route`, `config`, and `formatErrorResponse` +are in scope there (`:1824-1836`, import at `:2`). ```diff -+ // Pre-dispatch input admission: reject requests whose estimated token count -+ // exceeds the resolved model context window. -+ const admission = checkInputAdmission(parsed, providerConfig, resolvedModel); -+ if (!admission.admitted) { -+ return formatErrorResponse(413, "request_too_large", -+ `Estimated input (~${admission.estimatedTokens} tokens) exceeds the model context window (${admission.contextWindow} tokens). ` -+ + `Reduce the conversation size or choose a model with a larger context window.`, -+ { estimated_tokens: admission.estimatedTokens, context_window: admission.contextWindow }); ++ // Refuse an input that cannot plausibly fit the model context window before spending ++ // auth, circuit budget, or upstream bandwidth on a turn the provider will reject anyway. ++ // ++ // Compaction turns are exempt: Codex sends compaction_trigger BECAUSE context is full ++ // (src/responses/parser.ts:355), so refusing the turn that shrinks the context would ++ // deadlock the client against the very limit this gate reports. ++ if (parsed._compactionRequest !== true) { ++ const inputAdmission = checkInputAdmission(parsed, route.provider, route.providerName, parsed.modelId); ++ if (!inputAdmission.admitted) { ++ return formatErrorResponse( ++ 413, ++ "request_too_large", ++ `Estimated input (~${inputAdmission.estimatedTokens} tokens) is far past the context ` ++ + `window of ${parsed.modelId} (${inputAdmission.ceiling} tokens). Start a new session ` ++ + `or choose a model with a larger context window.`, ++ ); ++ } + } ``` ### NEW: tests/input-admission.test.ts -Test cases: -1. Request within context window → admitted -2. Request exceeding context window → 413 with token estimates -3. No context window configured → admitted (fail-open) -4. Base64 image data counted at reduced rate -5. Tool results included in estimate -6. `modelMaxInputTokens` caps below context window -7. Edge: exactly at boundary → admitted -8. Edge: 1 token over → rejected +1. Input under the ceiling → admitted +2. Input past `ceiling * ADMISSION_TOLERANCE` → refused, reporting both numbers +3. Input over the ceiling but inside tolerance → admitted (estimator error bar) +4. No ceiling resolvable → admitted (fail-open) +5. `modelMaxInputTokens` tightens the ceiling below the context window +6. A same-named custom provider uses its OWN limits, not registry limits (blocker 3) +6b. **A custom provider named `openai` with no limits does NOT get the native fallback (blocker 8)** +7. **Native `gpt-5.6-sol` resolves 372k from static metadata (blocker 7)** +7b. **An account-namespaced native route still resolves the native ceiling (blocker 8)** +7c. **A routed `provider/model` id does not take the native fallback (blocker 8)** +8. **The stride-aliased 1.6x CJK payload does NOT trip the gate at its true size (blocker 6)** +9. Assistant thinking blocks and tool-call arguments are counted +10. Tool name + description + parameters all counted +11. A `data:` image is charged by decoded size, not URL character length +12. A remote https image is charged a small fixed cost +13. **A `_compactionRequest` turn is admitted no matter how large (blocker 1)** +14. Ceiling resolution performs no filesystem access (blocker 5) ## Activation scenario -A Codex turn with 500k tokens of conversation history targeting a model with a -128k context window hits `checkInputAdmission` → returns `admitted: false` → -413 response returned to client before any upstream fetch. +A turn carrying ~1.3M estimated tokens (the #1412 shape) targets a 372k-window native +model. `1_300_000 > 372_000 * 2.5 = 930_000`, so the client gets 413 before any upstream +fetch. + +Observable proof required in C: the focused test asserts the 413 AND that no upstream fetch +was attempted — the test installs a fetch that throws if called, so a gate placed after +dispatch fails rather than passing quietly. + +Blocker-1 proof: a `_compactionRequest` turn of the same size returns non-413. +Blocker-6 proof: the aliased payload from the probe above is admitted. + +## Follow-ups (not this layer) + +- **FU-1: fix `cjkRatio` periodic aliasing** (`src/lib/token-estimate.ts:47`). Sampling + every `stride`-th character aliases against fixed-width records. `estimateTokens` also + feeds usage accounting and auto-compact (`src/server/chat-completions.ts:140`), so this + is a behavior change to unrelated subsystems and needs its own layer and tests. Until it + lands, `ADMISSION_TOLERANCE` absorbs the error. ## Scope boundary -IN: New admission module + core.ts insertion point + test file -OUT: Changing existing translator buffer limits, modifying provider configs, - adding UI for admission settings +IN: the admission module, its single guarded call site, the test file +OUT: the translator buffer limit, provider config defaults, GUI surface, any second copy of +context-window resolution, changing `estimateTokens` behavior (FU-1), and making +`src/routing/capability.ts` transport-aware (sidestepped by reading `route.provider`), and +applying `providerContextCaps` to admission + diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index fa09bf2e6c..3b9c4c8349 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1,6 +1,7 @@ import type { Server } from "bun"; import { bridgeToResponsesSSE, buildResponseJSON, formatErrorResponse, type ResponsesTerminalStatus } from "../../bridge"; import { formatPassthroughUpstreamError } from "./passthrough-error"; +import { checkInputAdmission } from "./input-admission"; import { describeUpstreamConnectFailure } from "./upstream-error"; import { getConfigPath, @@ -1837,6 +1838,24 @@ async function handleResponsesInner( } if (options.abortSignal?.aborted) return clientCancelledResponse(); + // Refuse an input that cannot plausibly fit the model context window before spending auth, + // circuit budget, or upstream bandwidth on a turn the provider will reject anyway (#1412). + // + // Compaction turns are exempt: Codex sends compaction_trigger BECAUSE context is full, so + // refusing the turn that shrinks the context would deadlock the client against the very + // limit this gate reports — it would be told to compact and then denied the compaction. + if (parsed._compactionRequest !== true) { + const inputAdmission = checkInputAdmission(parsed, route.provider, route.providerName, parsed.modelId); + if (!inputAdmission.admitted) { + return formatErrorResponse( + 413, + "request_too_large", + `Estimated input (~${inputAdmission.estimatedTokens} tokens) is far past the context window ` + + `of ${parsed.modelId} (${inputAdmission.ceiling} tokens). Start a new session or choose a ` + + `model with a larger context window.`, + ); + } + } const preAuthHostKey = preAuthUpstreamHostCircuitKey(route, config); if (preAuthHostKey) { const admission = acquireUpstreamHostAdmission( diff --git a/src/server/responses/input-admission.ts b/src/server/responses/input-admission.ts new file mode 100644 index 0000000000..af00d91200 --- /dev/null +++ b/src/server/responses/input-admission.ts @@ -0,0 +1,169 @@ +/** + * Pre-dispatch input admission (#1412). + * + * Refuses a turn whose estimated input cannot plausibly fit the model context window, + * BEFORE auth resolution, circuit admission, or any upstream I/O. #1412 reported ~127k of + * real context compounding to 1.3M-1.6M tokens and crashing the proxy; the provider would + * reject such a turn anyway, so paying for the round trip buys nothing. + * + * Deliberately narrow. This is not a context manager and not a compaction trigger: it + * catches the pathological case and stays out of the way otherwise. Every uncertainty + * resolves toward admitting. + */ +import { nativeOpenAiContextWindow } from "../../codex/catalog/metadata"; +import { estimateTokens } from "../../lib/token-estimate"; +import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; +import type { OcxContentPart, OcxParsedRequest, OcxProviderConfig } from "../../types"; + +/** + * Multiplier applied to the ceiling before refusing. + * + * 2.5, not something tighter, because `estimateTokens` can overshoot by 1.6x on its own. + * `cjkRatio` samples every `stride`-th character, so a payload of fixed-width records whose + * length aligns with the stride samples as 100% CJK while being ~1.6% CJK, firing the + * 2.5-chars/token clamp instead of the 4.0 default. Measured on Bun 1.3.14: 126,046 chars, + * true CJK ratio 0.0161, sampled ratio 1.0, estimate inflated 1.6x. Since 4.0 / 2.5 = 1.6 + * is that branch maximum divergence, a threshold at or under 1.6 would convert the + * estimator error bar into false 413s. + * + * 2.5 sits above it with room for the ~10% model-family ratio spread, and still refuses the + * #1412 shape (10x) four times over. + */ +export const ADMISSION_TOLERANCE = 2.5; + +/** + * Token cost charged for a remote image URL. The bytes are not in this request — the + * provider fetches them — so the URL own length is not the cost. A small flat charge + * acknowledges the tiles the image will occupy without pretending to know its dimensions. + */ +const REMOTE_IMAGE_TOKENS = 850; + +/** Decoded image bytes per token. Coarse tile-count proxy, not a provider formula. */ +const IMAGE_BYTES_PER_TOKEN = 750; + +export interface InputAdmissionResult { + admitted: boolean; + estimatedTokens: number; + /** Resolved ceiling, or null when nothing could be resolved (=> always admitted). */ + ceiling: number | null; +} + +function positive(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; +} + +/** + * Charge a `data:` URL by its DECODED size rather than its character length: base64 inflates + * by 4/3, so charging the string would overcount by a third. A remote URL is charged flat. + */ +function imageTokens(imageUrl: string): number { + if (!imageUrl.startsWith("data:")) return REMOTE_IMAGE_TOKENS; + const comma = imageUrl.indexOf(","); + if (comma < 0) return REMOTE_IMAGE_TOKENS; + const payload = imageUrl.length - comma - 1; + if (payload <= 0) return 0; + const decoded = Math.floor((payload * 3) / 4); + return Math.max(1, Math.ceil(decoded / IMAGE_BYTES_PER_TOKEN)); +} + +function contentPartTokens(part: OcxContentPart, modelId: string): number { + return part.type === "image" ? imageTokens(part.imageUrl) : estimateTokens(part.text, modelId); +} + +function contentTokens(content: string | readonly OcxContentPart[], modelId: string): number { + if (typeof content === "string") return estimateTokens(content, modelId); + let total = 0; + for (const part of content) total += contentPartTokens(part, modelId); + return total; +} + +/** + * Estimate the input tokens of a parsed request. + * + * Walks the whole `OcxMessage` union rather than user text alone. Assistant turns carry + * their content as `OcxAssistantContentPart[]` — text, thinking blocks, and tool calls whose + * JSON arguments are frequently the largest single item in an agent conversation. A walk + * that counted only `{type:"text"}` would undercount exactly the turns that trigger this + * gate. + */ +export function estimateInputTokens(parsed: OcxParsedRequest, modelId: string): number { + const { context } = parsed; + let total = 0; + + for (const prompt of context.systemPrompt ?? []) total += estimateTokens(prompt, modelId); + + for (const message of context.messages) { + if (message.role === "assistant") { + for (const part of message.content) { + if (part.type === "text") total += estimateTokens(part.text, modelId); + else if (part.type === "thinking") total += estimateTokens(part.thinking, modelId); + else total += estimateTokens(part.name, modelId) + estimateTokens(JSON.stringify(part.arguments), modelId); + } + // Opaque provider blob replayed verbatim upstream, so it costs real input tokens. + if (message.kiroRedactedReasoning) total += estimateTokens(message.kiroRedactedReasoning, modelId); + continue; + } + total += contentTokens(message.content, modelId); + } + + // Tool schemas ride every turn: name, description, and the JSON parameter schema all + // reach the upstream, and a large MCP catalog can dominate a short conversation. + for (const tool of context.tools ?? []) { + total += estimateTokens(tool.name, modelId) + + estimateTokens(tool.description, modelId) + + estimateTokens(JSON.stringify(tool.parameters), modelId); + } + + return total; +} + +/** + * Resolve the admission ceiling. Pure: no filesystem, no catalog, no registry scan. + * + * `provider` must be the ROUTED config (`route.provider`), which `routedProviderConfig` + * has already transport-guarded and merged. Re-deriving from `config.providers[name]` would + * reject a user-defined provider that merely shares a built-in name using limits that + * belong to a different service. + */ +export function resolveInputCeiling( + provider: OcxProviderConfig, + providerName: string, + modelId: string, +): number | null { + const configured = positive(provider.modelContextWindows?.[modelId]) ?? positive(provider.contextWindow); + + // The canonical `openai` registry entry declares no context fields, so without this the + // gate would be inert on the default Codex route. All three clauses are load-bearing: a + // transport-mismatched custom provider named "openai" is preserved verbatim by routing + // and must not inherit built-in native limits, and a routed `provider/model` id is not a + // native slug. Static maps only — no catalog read. + const native = configured === null + && providerName === OPENAI_CODEX_PROVIDER_ID + && isCanonicalOpenAiForwardProvider(provider) + && !modelId.includes("/") + ? positive(nativeOpenAiContextWindow(modelId)) + : null; + + const window = configured ?? native; + // modelMaxInputTokens is an input-only cap, so it can only tighten the window. + const maxInput = positive(provider.modelMaxInputTokens?.[modelId]); + if (window === null) return maxInput; + return maxInput === null ? window : Math.min(window, maxInput); +} + +/** + * Fail-open when no ceiling is known; refuse only past `ceiling * ADMISSION_TOLERANCE`. + * + * The caller is responsible for skipping compaction turns — see the call site in core.ts. + */ +export function checkInputAdmission( + parsed: OcxParsedRequest, + provider: OcxProviderConfig, + providerName: string, + modelId: string, +): InputAdmissionResult { + const ceiling = resolveInputCeiling(provider, providerName, modelId); + if (ceiling === null) return { admitted: true, estimatedTokens: 0, ceiling: null }; + const estimatedTokens = estimateInputTokens(parsed, modelId); + return { admitted: estimatedTokens <= ceiling * ADMISSION_TOLERANCE, estimatedTokens, ceiling }; +} diff --git a/tests/input-admission.test.ts b/tests/input-admission.test.ts new file mode 100644 index 0000000000..3669d46122 --- /dev/null +++ b/tests/input-admission.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, test } from "bun:test"; +import { + ADMISSION_TOLERANCE, + checkInputAdmission, + estimateInputTokens, + resolveInputCeiling, +} from "../src/server/responses/input-admission"; +import type { OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTool } from "../src/types"; + +const CANONICAL_NATIVE: OcxProviderConfig = { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", +}; + +function request(messages: OcxMessage[], tools?: OcxTool[]): OcxParsedRequest { + return { + modelId: "test-model", + context: { messages, ...(tools ? { tools } : {}) }, + stream: false, + options: {}, + } as OcxParsedRequest; +} + +function userText(text: string): OcxMessage { + return { role: "user", content: text, timestamp: 0 }; +} + +/** Roughly `tokens` worth of plain ASCII at the default 4 chars/token ratio. */ +function asciiTokens(tokens: number): string { + return "a".repeat(tokens * 4); +} + +describe("resolveInputCeiling", () => { + test("prefers the per-model window over the provider-wide one", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + contextWindow: 8_000, + modelContextWindows: { "m": 32_000 }, + }; + expect(resolveInputCeiling(provider, "custom", "m")).toBe(32_000); + expect(resolveInputCeiling(provider, "custom", "other")).toBe(8_000); + }); + + test("modelMaxInputTokens can only tighten the window", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": 32_000 }, + modelMaxInputTokens: { "m": 20_000 }, + }; + expect(resolveInputCeiling(provider, "custom", "m")).toBe(20_000); + }); + + test("a looser modelMaxInputTokens never widens the window", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": 32_000 }, + modelMaxInputTokens: { "m": 90_000 }, + }; + expect(resolveInputCeiling(provider, "custom", "m")).toBe(32_000); + }); + + test("returns null when nothing is configured", () => { + const provider: OcxProviderConfig = { adapter: "openai-chat", baseUrl: "https://example.test/v1" }; + expect(resolveInputCeiling(provider, "custom", "m")).toBeNull(); + }); + + test("resolves the native window from static metadata for a canonical route", () => { + // The `openai` registry entry carries no context fields, so without the native + // fallback the gate would be inert on the default Codex route. + expect(resolveInputCeiling(CANONICAL_NATIVE, "openai", "gpt-5.6-sol")).toBe(372_000); + }); + + test("a custom provider merely NAMED openai does not inherit native limits", () => { + const impostor: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://impostor.test/v1", + authMode: "key", + }; + expect(resolveInputCeiling(impostor, "openai", "gpt-5.6-sol")).toBeNull(); + }); + + test("a routed provider/model id does not take the native fallback", () => { + expect(resolveInputCeiling(CANONICAL_NATIVE, "openai", "vendor/gpt-5.6-sol")).toBeNull(); + }); + + test("an explicit user window still wins over native metadata", () => { + const pinned: OcxProviderConfig = { ...CANONICAL_NATIVE, modelContextWindows: { "gpt-5.6-sol": 50_000 } }; + expect(resolveInputCeiling(pinned, "openai", "gpt-5.6-sol")).toBe(50_000); + }); +}); + +describe("estimateInputTokens", () => { + test("counts assistant thinking blocks and tool-call arguments", () => { + // A walk that only counted {type:"text"} would report ~0 here, which is exactly the + // shape of an agent conversation that triggers this gate. + const parsed = request([{ + role: "assistant", + timestamp: 0, + content: [ + { type: "thinking", thinking: asciiTokens(500) }, + { type: "toolCall", id: "c1", name: "apply_patch", arguments: { patch: asciiTokens(500) } }, + ], + }]); + expect(estimateInputTokens(parsed, "test-model")).toBeGreaterThan(900); + }); + + test("counts tool name, description, and parameter schema", () => { + const bare = estimateInputTokens(request([userText("hi")]), "test-model"); + const withTools = estimateInputTokens(request([userText("hi")], [{ + name: "search", + description: asciiTokens(300), + parameters: { type: "object", properties: { q: { type: "string", description: asciiTokens(300) } } }, + }]), "test-model"); + expect(withTools - bare).toBeGreaterThan(500); + }); + + test("charges a data: image by decoded size, not URL length", () => { + // base64 inflates by 4/3, so charging the string would overcount by a third. + const base64 = "A".repeat(75_000); + const parsed = request([{ + role: "user", + timestamp: 0, + content: [{ type: "image", imageUrl: `data:image/png;base64,${base64}` }], + }]); + const decoded = Math.floor((75_000 * 3) / 4); + expect(estimateInputTokens(parsed, "test-model")).toBe(Math.ceil(decoded / 750)); + }); + + test("charges a remote image a flat cost, not its URL length", () => { + const short = request([{ + role: "user", + timestamp: 0, + content: [{ type: "image", imageUrl: "https://example.test/a.png" }], + }]); + const long = request([{ + role: "user", + timestamp: 0, + content: [{ type: "image", imageUrl: `https://example.test/${"b".repeat(4_000)}.png` }], + }]); + expect(estimateInputTokens(short, "test-model")).toBe(estimateInputTokens(long, "test-model")); + }); + + test("Korean text costs more than the same number of ASCII characters", () => { + const korean = estimateInputTokens(request([userText("한".repeat(4_000))]), "test-model"); + const ascii = estimateInputTokens(request([userText("a".repeat(4_000))]), "test-model"); + expect(korean).toBeGreaterThan(ascii); + }); +}); + +describe("checkInputAdmission", () => { + const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": 10_000 }, + }; + + test("admits input under the ceiling", () => { + const result = checkInputAdmission(request([userText(asciiTokens(5_000))]), provider, "custom", "m"); + expect(result.admitted).toBe(true); + expect(result.ceiling).toBe(10_000); + }); + + test("admits input over the ceiling but inside the tolerance", () => { + // The estimator has a real error bar, so 1.0x-2.5x is deliberately not refused. + const result = checkInputAdmission(request([userText(asciiTokens(15_000))]), provider, "custom", "m"); + expect(result.admitted).toBe(true); + }); + + test("refuses input past the tolerance", () => { + const result = checkInputAdmission(request([userText(asciiTokens(40_000))]), provider, "custom", "m"); + expect(result.admitted).toBe(false); + expect(result.ceiling).toBe(10_000); + expect(result.estimatedTokens).toBeGreaterThan(10_000 * ADMISSION_TOLERANCE); + }); + + test("admits everything when no ceiling resolves", () => { + const unknown: OcxProviderConfig = { adapter: "openai-chat", baseUrl: "https://example.test/v1" }; + const result = checkInputAdmission(request([userText(asciiTokens(5_000_000))]), unknown, "custom", "m"); + expect(result.admitted).toBe(true); + expect(result.ceiling).toBeNull(); + }); + + test("the estimator CJK sampling alias does not trip the gate", () => { + // cjkRatio samples every stride-th character. A payload of fixed-width records whose + // length aligns with the stride samples as 100% CJK while being ~1.6% CJK, inflating + // the estimate by 1.6x. ADMISSION_TOLERANCE exists to absorb exactly this; a payload + // whose HONEST size fits must still be admitted. + const record = "\uAC00" + "x".repeat(61); + const text = record.repeat(2_033); + const honestTokens = Math.ceil(text.length / 4); + const aliased: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + modelContextWindows: { "m": honestTokens }, + }; + const result = checkInputAdmission(request([userText(text)]), aliased, "custom", "m"); + expect(result.estimatedTokens).toBeGreaterThan(honestTokens); // the overshoot is real + expect(result.admitted).toBe(true); // and absorbed + }); + + test("resolving a ceiling touches no filesystem", () => { + // The native fallback must read static maps only; a catalog read here would put + // synchronous file I/O on every request. + const fs = require("node:fs"); + const watched = ["readFileSync", "existsSync", "statSync"] as const; + const originals = watched.map(name => [name, fs[name]] as const); + let calls = 0; + for (const [name, fn] of originals) { + fs[name] = (...args: unknown[]) => { calls += 1; return (fn as (...a: unknown[]) => unknown)(...args); }; + } + try { + resolveInputCeiling(CANONICAL_NATIVE, "openai", "gpt-5.6-sol"); + } finally { + for (const [name, fn] of originals) fs[name] = fn; + } + expect(calls).toBe(0); + }); +});