diff --git a/src/inference-error-message.test.ts b/src/inference-error-message.test.ts index 19779728d..81e5b2f32 100644 --- a/src/inference-error-message.test.ts +++ b/src/inference-error-message.test.ts @@ -160,6 +160,50 @@ describe("terminalProviderFailureMessage", () => { ); }); + test("terminal Codex credential 404 names the profile with a re-login hint", () => { + const normalized = normalizeInferenceErrorForTerminal( + { + category: "fatal", + message: "Not Found", + statusCode: 404, + raw: { + error: { + code: "invalid_token", + message: "Not authorized: the access token has been revoked", + }, + }, + }, + "codex/work", + ); + const message = terminalProviderFailureMessage("codex/work", normalized); + expect(message).toContain('Codex profile "work"'); + expect(message).toContain("Not Found"); + expect(message).not.toContain("/model"); + // One re-login hint, not a stutter: the branded diagnostic dedups via + // the shared carriesCodexReLoginHint predicate. + expect(message.toLowerCase().match(/log in again/g)).toHaveLength(1); + }); + + test("terminal bare Codex 404 without an auth signal keeps switch-models guidance", () => { + const normalized = normalizeInferenceErrorForTerminal( + { category: "fatal", message: "Not Found", statusCode: 404 }, + "codex/work", + ); + const message = terminalProviderFailureMessage("codex/work", normalized); + expect(message).toContain('"/model"'); + expect(message.toLowerCase()).not.toMatch(/log in again/); + }); + + test("terminal genuine unknown-model 404 keeps switch-models guidance", () => { + const message = terminalProviderFailureMessage("codex/work", { + category: "fatal", + message: "The model 'gpt-99' does not exist", + statusCode: 404, + providerId: "codex/work", + }); + expect(message).toContain('"/model"'); + }); + test.each([ { name: "Bearer header", diff --git a/src/inference-error-message.ts b/src/inference-error-message.ts index cf1d8ef4e..0cd6126cd 100644 --- a/src/inference-error-message.ts +++ b/src/inference-error-message.ts @@ -14,6 +14,7 @@ import { import { stripTerminalControlSequences } from "./util/control-char-strip.js"; import { scrubSecretShapedContent } from "./plugins/tool-result-secret-scrub.js"; import { + carriesCodexReLoginHint, gatewayOverloadUserMessage, isCodexShortRateLimitInferenceError, isGatewayOverloadInferenceError, @@ -146,14 +147,22 @@ export function terminalProviderFailureMessage( ? diagnostic : `${diagnostic}.`; const guidance = terminalProviderFailureGuidance(error, category); - return `${label} Provider failed (${category}): ${diagnosticSentence} ${guidance}`; + const tail = guidance.length > 0 ? ` ${guidance}` : ""; + return `${label} Provider failed (${category}): ${diagnosticSentence}${tail}`; } function terminalProviderFailureGuidance( error: InferenceErrorLike, category: string, ): string { - if (category === "credential_failure") return CREDENTIAL_FAILURE_USER_MESSAGE; + if (category === "credential_failure") { + // Normalized credential failures already carry the re-login hint in the + // diagnostic (e.g. Codex profile copy); repeating it reads as a stutter. + // Shared with the classifier via carriesCodexReLoginHint — one predicate. + return carriesCodexReLoginHint(error.message ?? "") + ? "" + : CREDENTIAL_FAILURE_USER_MESSAGE; + } if (category === "context_overflow") return "Try /clear to start fresh."; // A 429 that survived the harness's paced retries is a wait-it-out rate // limit, not a generic flake: say so instead of the bare "Try again." @@ -180,7 +189,9 @@ function terminalProviderFailureSummary( ): string { const label = terminalProviderFailureLabel(providerId, displayLabel); const category = terminalProviderFailureCategory(error); - return `${label} Provider failed (${category}). ${terminalProviderFailureGuidance(error, category)}`; + const guidance = terminalProviderFailureGuidance(error, category); + const tail = guidance.length > 0 ? ` ${guidance}` : ""; + return `${label} Provider failed (${category}).${tail}`; } export type ResolvedProviderFailureError = Error & { diff --git a/src/inference-gateway-error.test.ts b/src/inference-gateway-error.test.ts index 622795f2c..0a6ac1225 100644 --- a/src/inference-gateway-error.test.ts +++ b/src/inference-gateway-error.test.ts @@ -1,9 +1,12 @@ import { describe, expect, test } from "bun:test"; import { + carriesCodexReLoginHint, + codexCredential404ReclassifiedStats, GATEWAY_OVERLOAD_USER_MESSAGE, isGatewayOverloadInferenceError, looksLikeHtmlGatewayBody, normalizeInferenceErrorForRetry, + resetCodexCredential404StatsForTests, XAI_CAPACITY_USER_MESSAGE, } from "./inference-gateway-error.js"; @@ -279,6 +282,178 @@ describe("normalizeInferenceErrorForRetry", () => { expect(normalized).toBe(error); }); + /** + * Wire shape for a revoked Codex credential: the harness classifies the + * HTTP 404 as fatal with the statusText message while the JSON body rides + * on raw. The body carries the auth-rejection signal; the status line + * alone ("Not Found") must never reclassify. + */ + const REVOKED_CREDENTIAL_404_RAW = { + error: { + code: "invalid_token", + message: "Not authorized: the access token has been revoked", + type: "invalid_request_error", + }, + }; + + test("Codex 404 with a revoked-credential body reclassifies as credential_failure", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "fatal", + message: "Not Found", + statusCode: 404, + providerId: "codex/work", + raw: REVOKED_CREDENTIAL_404_RAW, + }); + expect(normalized.category).toBe("credential_failure"); + expect(normalized.message).toContain('Codex profile "work"'); + expect(carriesCodexReLoginHint(normalized.message)).toBe(true); + // Original diagnostic rides along so the failure stays debuggable, and + // the wire body stays on raw for logs. + expect(normalized.message).toContain("Not Found"); + expect(normalized.raw).toEqual(REVOKED_CREDENTIAL_404_RAW); + }); + + test.each([ + { name: "not authorized", body: "Not authorized" }, + { name: "unauthorized", body: "401 Unauthorized" }, + { name: "invalid token", body: "Invalid token" }, + { name: "expired", body: "The access token expired" }, + { name: "revoked", body: "Token has been revoked" }, + ])("Codex 404 with $name signal reclassifies", ({ body }) => { + const normalized = normalizeInferenceErrorForRetry({ + category: "fatal", + message: "Not Found", + statusCode: 404, + providerId: "codex/work", + raw: { error: { message: body } }, + }); + expect(normalized.category).toBe("credential_failure"); + expect(carriesCodexReLoginHint(normalized.message)).toBe(true); + }); + + test("Codex 404 with an auth signal in the message reclassifies", () => { + const normalized = normalizeInferenceErrorForRetry({ + category: "fatal", + message: "Invalid token: expired", + statusCode: 404, + providerId: "codex/work", + }); + expect(normalized.category).toBe("credential_failure"); + expect(normalized.message).toContain("Invalid token: expired"); + }); + + test("Codex bare 404 without an auth signal stays fatal", () => { + const error = { + category: "fatal" as const, + message: "Not Found", + statusCode: 404, + providerId: "codex/work", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("Codex routing 404 without an auth signal stays fatal", () => { + const error = { + category: "fatal" as const, + message: "Not Found", + statusCode: 404, + providerId: "codex/work", + raw: { error: { code: "not_found", message: "No such endpoint" } }, + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("Codex 404 naming a dotted unknown model stays fatal", () => { + const error = { + category: "fatal" as const, + message: "The model 'gpt-3.5-turbo' does not exist", + statusCode: 404, + providerId: "codex/work", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("Codex 404 naming an unknown model keeps fatal switch-models guidance", () => { + const error = { + category: "fatal" as const, + message: "The model 'gpt-99' does not exist", + statusCode: 404, + providerId: "codex/work", + raw: { + error: { + code: "model_not_found", + message: "The model 'gpt-99' does not exist", + type: "invalid_request_error", + }, + }, + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("Codex model-deprecation 404 containing 'expired' stays fatal", () => { + // Keeper: a retired model names itself with the credential marker word, + // but logging in again cannot resurrect it — the fatal switch-models + // path must win over the expired-credential reclassification. + const error = { + category: "fatal" as const, + message: + "The model 'gpt-4o' has expired. Migrate to 'gpt-5' to continue.", + statusCode: 404, + providerId: "codex/work", + raw: { + error: { + code: "model_expired", + message: "The model 'gpt-4o' has expired (2025-02-01).", + type: "invalid_request_error", + }, + }, + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("non-Codex 404 keeps fatal switch-models guidance", () => { + const error = { + category: "fatal" as const, + message: "Not Found", + statusCode: 404, + providerId: "custom-provider", + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("non-Codex 404 with an auth signal keeps provider scoping", () => { + const error = { + category: "fatal" as const, + message: "Not Found", + statusCode: 404, + providerId: "custom-provider", + raw: { error: { message: "Token has been revoked" } }, + }; + expect(normalizeInferenceErrorForRetry(error)).toBe(error); + }); + + test("reclassified Codex 404s bump the counter with a body sample", () => { + resetCodexCredential404StatsForTests(); + expect(codexCredential404ReclassifiedStats().count).toBe(0); + normalizeInferenceErrorForRetry({ + category: "fatal", + message: "Not Found", + statusCode: 404, + providerId: "codex/work", + raw: REVOKED_CREDENTIAL_404_RAW, + }); + // A fatal 404 without an auth signal must not bump the counter. + normalizeInferenceErrorForRetry({ + category: "fatal", + message: "Not Found", + statusCode: 404, + providerId: "codex/work", + }); + const stats = codexCredential404ReclassifiedStats(); + expect(stats.count).toBe(1); + expect(stats.lastSample).toContain("revoked"); + }); + test("known-xAI message-only capacity protocol error becomes retryable", () => { const normalized = normalizeInferenceErrorForRetry({ category: "protocol_mismatch", diff --git a/src/inference-gateway-error.ts b/src/inference-gateway-error.ts index 2e97ebbd6..12c652831 100644 --- a/src/inference-gateway-error.ts +++ b/src/inference-gateway-error.ts @@ -519,13 +519,164 @@ function normalizeCodexUsageLimitError( }; } +/** + * Positive auth-rejection signals for the Codex credential-404 classifier. A + * known-Codex fatal 404 reclassifies to credential_failure ONLY when the + * message or raw body carries one of these markers — bare / routing / config + * 404s and genuine unknown-model rejections stay fatal with switch-models + * guidance. Negative unknown-model matching is deliberately not used here: + * every new backend phrasing would otherwise need an allowlist entry. + */ +const CODEX_CREDENTIAL_404_MARKERS = [ + "not authorized", + "unauthorized", + "unauthorised", + "invalid token", + "invalid_token", + "expired", + "revoked", +] as const; + +function hasCodexCredentialAuthSignal(error: InferenceErrorLike): boolean { + return combinedTextIncludesMarker( + [error.message ?? "", stringFromRaw(error.raw)], + CODEX_CREDENTIAL_404_MARKERS, + ); +} + +/** + * Model-deprecation signals that veto the credential-404 classifier. A + * retired-model 404 can itself carry the word "expired" ("model 'gpt-4o' + * has expired — migrate to 'gpt-5'"), and reclassifying it as + * credential_failure would send the operator to log in again for a model + * that no longer exists. The veto needs a model mention plus a deprecation + * signal so bare credential texts ("the access token expired") still + * reclassify. + */ +const CODEX_MODEL_DEPRECATION_MARKERS = [ + "model_expired", + "model_deprecated", + "deprecated", + "deprecation", + "retired", + "sunset", + "no longer supported", + "no longer available", + "has expired", + "end of life", +] as const; + +function looksLikeCodexModelDeprecation(error: InferenceErrorLike): boolean { + const combined = [error.message ?? "", stringFromRaw(error.raw)] + .join("\n") + .toLowerCase(); + if (!combined.includes("model")) return false; + return CODEX_MODEL_DEPRECATION_MARKERS.some((marker) => + combined.includes(marker), + ); +} + +/** + * Single shared predicate behind the Codex credential-404 re-login copy: the + * classifier brands with it (formatCodexCredential404Message) and the + * terminal-guidance dedup checks with it, so the two cannot drift. + */ +export function carriesCodexReLoginHint(text: string): boolean { + return /log in again|sign in again/i.test(text); +} + +/** Branded re-login line for a Codex credential 404, diagnostic appended. */ +function formatCodexCredential404Message( + profile: string, + originalDiagnostic: string, +): string { + const branded = `Codex profile "${profile}" is not authorized. Log in again.`; + const oneLine = originalDiagnostic.replace(/\s+/g, " ").trim(); + if (oneLine.length === 0 || branded.includes(oneLine)) return branded; + const clipped = oneLine.length > 200 ? `${oneLine.slice(0, 199)}…` : oneLine; + return `${branded} (${clipped})`; +} + +/** + * Reclassification telemetry for the Codex credential-404 classifier. The + * backend invents new 404 reasons over time; the counter plus the last-body + * sample let future unknown-404 waves be spotted without guessing. + */ +let codexCredential404ReclassifiedCount = 0; +let lastReclassifiedCodex404Sample = ""; + +export function codexCredential404ReclassifiedStats(): { + readonly count: number; + readonly lastSample: string; +} { + return { + count: codexCredential404ReclassifiedCount, + lastSample: lastReclassifiedCodex404Sample, + }; +} + +export function resetCodexCredential404StatsForTests(): void { + codexCredential404ReclassifiedCount = 0; + lastReclassifiedCodex404Sample = ""; +} + +function recordCodexCredential404Reclassification( + error: InferenceErrorLike, +): void { + codexCredential404ReclassifiedCount += 1; + const sample = [error.message ?? "", stringFromRaw(error.raw)] + .join("\n") + .replace(/\s+/g, " ") + .trim(); + lastReclassifiedCodex404Sample = + sample.length > 500 ? `${sample.slice(0, 499)}…` : sample; +} + +/** + * Codex answers unauthenticated requests with 426/404, so a fatal 404 in a + * known-Codex context whose body carries an auth-rejection signal is an + * expired, invalid, or revoked credential — not a bad model name. The + * re-login copy matches the CodexAuthError shape so the TUI names the + * affected profile through its existing auth matchers; the original + * diagnostic rides along in parens so the model name stays debuggable. + * Anything without an auth signal keeps the fatal switch-models path, as + * does a model-deprecation 404 even when it carries the word "expired". + */ +function normalizeCodexCredential404Error( + error: InferenceErrorWithGoContext, +): InferenceError { + if (error.category !== "fatal") return error; + if (error.statusCode !== 404) return error; + const providerId = error.providerId; + if (providerId === undefined || !isCodexProviderName(providerId)) + return error; + // A retired model is a fatal switch-models failure, never a credential + // failure: the veto runs before the auth markers so deprecation wins over + // a merely expired-sounding word. + if (looksLikeCodexModelDeprecation(error)) return error; + if (!hasCodexCredentialAuthSignal(error)) return error; + const profile = codexProfileFromProviderName(providerId) ?? providerId; + const message = formatCodexCredential404Message(profile, error.message ?? ""); + recordCodexCredential404Reclassification(error); + return { + category: "credential_failure", + message, + statusCode: 404, + ...(error.raw !== undefined ? { raw: error.raw } : {}), + ...(error.retryAfterMs !== undefined + ? { retryAfterMs: error.retryAfterMs } + : {}), + }; +} + /** * Reclassify gateway overload errors so the default retry policy treats them as * transient instead of aborting on protocol_mismatch. Also normalizes OpenCode * Go quota/rate-limit shapes (including HTTP 400 mis-status), known-xAI short * 429s, attributable xAI capacity protocol_mismatch, Codex usage limits - * (nested detail.error with resets_in_seconds), and known-Codex short 429s that - * are not usage_limit_reached. + * (nested detail.error with resets_in_seconds), known-Codex short 429s that + * are not usage_limit_reached, and known-Codex 404s carrying an + * auth-rejection signal (expired/revoked credential). */ export function normalizeInferenceErrorForRetry( error: InferenceErrorWithGoContext, @@ -545,6 +696,9 @@ export function normalizeInferenceErrorForRetry( const codexRateLimit = normalizeCodexRateLimitError(error); if (codexRateLimit !== error) return codexRateLimit; + const codexCredential = normalizeCodexCredential404Error(error); + if (codexCredential !== error) return codexCredential; + if (!isGatewayOverloadInferenceError(error)) return error; if (error.category === "retryable" || error.category === "timeout") return error;