diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index 812cadd22f..a7a9fb93fd 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -455,6 +455,7 @@ export async function resolveCodexAuthContext( const excludeAccountIds = nativeMainReadsForbidden ? new Set([MAIN_CODEX_ACCOUNT_ID]) : undefined; + const mainModelGrantUnobserved = excludeAccountIds?.has(MAIN_CODEX_ACCOUNT_ID) === true; const entitlementSnapshot = options.modelId && ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(options.modelId) ? await (options.resolveCodexModelEntitlements ?? resolveCodexModelEntitlements)(config, { excludeAccountIds, @@ -507,7 +508,15 @@ export async function resolveCodexAuthContext( if (resolution.status === "expired") throw new CodexThreadAffinityExpiredError(resolution.accountId); const selected = resolution.status === "selected" ? resolution.accountId : null; if (!selected) { - if (requestScopedMainCredential && fixedAccountId === undefined && !options.excludeAccountId) { + // A retry that excluded a failed Pool account may still use the validated caller-owned + // main credential. Treating every exclusion as if main itself had failed strands a healthy + // native bearer after the first Pool attempt. Preserve the exactly-once boundary by refusing + // this fallback only when the excluded credential is main. + if ( + requestScopedMainCredential + && fixedAccountId === undefined + && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID + ) { return await resolveCallerOwnedMainContext(); } if (fixedAccountId !== undefined) { @@ -527,7 +536,11 @@ export async function resolveCodexAuthContext( throw new CodexMainProfileDrainingError(); } throw new CodexPoolAuthenticationError( - modelEligibleAccountIds ? "No eligible Codex account supports this model" : undefined, + modelEligibleAccountIds === undefined + ? undefined + : entitledAccountIds?.size === 0 && !mainModelGrantUnobserved + ? "No eligible Codex account supports this model" + : "Codex accounts that support this model are currently unavailable", ); } accountId = selected; diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index b14800fc7b..f0e798828f 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -382,7 +382,10 @@ async function resolveAlternateCompactContext(args: { requestScopedMainCredential: hasForwardableCodexBearer(req.headers, config), beginCodexAccountSelection: codexAccountSelectionForTurn(turnAdmissionLease), }); - if (!authCtx.accountId || authCtx.accountId === excludeAccountId) return null; + // Caller-owned main has no Pool account id. It is still a valid one-shot alternate after a + // stored account fails; resolveCodexAuthContext already prevents returning it when main is the + // excluded credential. + if (authCtx.accountId === excludeAccountId) return null; const provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); const headers = new Headers({ "content-type": "application/json" }); const selected = headersForCodexAuthContext(req.headers, authCtx); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 1f56ed979a..5296e9307f 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -945,7 +945,7 @@ interface CodexPoolAccountRetryArgs { stream: boolean; onResponse?: ( response: Response, - authCtx: Extract, + authCtx: CodexAuthContext, request: Awaited["buildRequest"]>>, ) => void; } @@ -953,7 +953,7 @@ interface CodexPoolAccountRetryArgs { type CodexPoolAccountRetryResult = | { kind: "retried"; - authCtx: Extract; + authCtx: CodexAuthContext; request: Awaited["buildRequest"]>>; upstreamResponse: Response; selectedForwardHeaders: Headers; @@ -962,7 +962,7 @@ type CodexPoolAccountRetryResult = | { kind: "transport"; error: unknown; - authCtx: Extract; + authCtx: CodexAuthContext; }; /** Keep retry-stage entitlement snapshots inside the native-main selection fence. */ @@ -1129,7 +1129,14 @@ async function retryCodexPoolOnAlternateAccount( throw error; } } - if (retryAuthCtx?.kind !== "pool" && retryAuthCtx?.kind !== "main-pool") { + // A validated request-owned main bearer is a real alternate when the failed credential was a + // stored Pool account. It has no Pool account id to promote or cool, but it can own this one + // bounded replay. The resolver already refuses it when main itself is the excluded credential. + if ( + retryAuthCtx?.kind !== "pool" + && retryAuthCtx?.kind !== "main-pool" + && retryAuthCtx?.kind !== "main" + ) { return { kind: "no-alternate" }; } @@ -1243,6 +1250,9 @@ async function retryCodexPoolOnAlternateAccount( retrySendCount += 1; args.onResponse?.(upstreamResponse, retryAuthCtx, request); if (!retrySameConfirmedAccount || retrySendCount >= maxRetrySends) break; + // Caller-owned main is an alternate-account replay and can never enter the bounded + // same-stored-account 400 loop above. Keep that invariant explicit for the account-id reads. + if (retryAuthCtx.kind === "main") break; if (!await shouldRetryCodexPoolAccountModel400( upstreamResponse, route.modelId, @@ -4285,7 +4295,12 @@ async function handleResponsesInner( passthroughEstimate, stream: parsed.stream, onResponse: (response, retryAuthCtx, retryRequest) => { - captureAffinityResponse(response, retryAuthCtx, retryRequest, true); + captureAffinityResponse( + response, + retryAuthCtx, + retryRequest, + retryAuthCtx.kind !== "main", + ); }, }); if (retry.kind === "transport") { diff --git a/tests/codex-auth-context.test.ts b/tests/codex-auth-context.test.ts index 93653a3002..be19f74c0e 100644 --- a/tests/codex-auth-context.test.ts +++ b/tests/codex-auth-context.test.ts @@ -633,6 +633,50 @@ describe("Codex auth context", () => { }); }); + test("account-gated routing distinguishes an unavailable grant from no grant", async () => { + const cfg = config(); + saveCodexAccountCredential("pool-a", { + accessToken: "pool-token", + refreshToken: "pool-refresh", + expiresAt: Date.now() + 5 * 60_000, + chatgptAccountId: "pool-account", + }); + const snapshot = (models: string[]): CodexModelEntitlementSnapshot => ({ + modelsByAccount: new Map([["pool-a", new Set(models)]]), + confirmedAccountIds: new Set(["pool-a"]), + credentialIdentities: new Map(), + }); + const resolve = (models: string[]) => resolveCodexAuthContext(new Headers(), cfg, "pool", { + excludeAccountId: "pool-a", + modelId: "gpt-daybreak-blue-latest", + resolveCodexModelEntitlements: async () => snapshot(models), + }); + + await expect(resolve(["gpt-daybreak-blue-latest"])) + .rejects.toThrow("Codex accounts that support this model are currently unavailable"); + await expect(resolve(["gpt-5.6-sol"])) + .rejects.toThrow("No eligible Codex account supports this model"); + + const mainExcludedSnapshot: CodexModelEntitlementSnapshot = { + modelsByAccount: new Map(), + confirmedAccountIds: new Set(), + credentialIdentities: new Map(), + }; + await expect(resolveCodexAuthContext(new Headers(), cfg, "pool", { + excludeAccountId: "pool-a", + modelId: "gpt-daybreak-blue-latest", + beginCodexAccountSelection: () => ({ + mainProfileDraining: true, + claimMainProfile: () => false, + release: () => {}, + }), + resolveCodexModelEntitlements: async (_config, options) => { + expect(options?.excludeAccountIds?.has(MAIN_CODEX_ACCOUNT_ID)).toBeTrue(); + return mainExcludedSnapshot; + }, + })).rejects.toThrow("Codex accounts that support this model are currently unavailable"); + }); + test("auth resolution preserves per-model detours without replacing ordinary affinity", async () => { const cfg = config(); cfg.accountPoolStrategy = "round-robin"; @@ -1088,6 +1132,46 @@ describe("Codex auth context", () => { clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); } }); + + test("a failed Pool account may fall back once to the validated caller-owned main credential", async () => { + const cfg = config(); + cfg.codexAccounts = [ + { id: "pool-a", email: "pool-a@example.test", isMain: false, chatgptAccountId: "pool-account" }, + ]; + const inbound = new Headers({ + authorization: "Bearer caller-keyring-token", + "chatgpt-account-id": "caller-keyring-account", + }); + const emptyEntitlements: CodexModelEntitlementSnapshot = { + modelsByAccount: new Map(), + confirmedAccountIds: new Set(), + credentialIdentities: new Map(), + }; + let directEntitlementChecks = 0; + const options = { + requestScopedMainCredential: true, + modelId: "gpt-daybreak-blue-latest", + resolveCodexModelEntitlements: async () => emptyEntitlements, + isDirectCallerEntitledToCodexModel: async () => { + directEntitlementChecks += 1; + return true; + }, + }; + + await expect(resolveCodexAuthContext(inbound, cfg, "pool", { + ...options, + excludeAccountId: "pool-a", + })).resolves.toMatchObject({ kind: "main", accountId: null }); + expect(directEntitlementChecks).toBe(1); + + // If main itself was the failed credential, the retry must not loop back to it. + await expect(resolveCodexAuthContext(inbound, { ...cfg, codexAccounts: [] }, "pool", { + ...options, + excludeAccountId: MAIN_CODEX_ACCOUNT_ID, + })).rejects.toThrow("Codex accounts that support this model are currently unavailable"); + expect(directEntitlementChecks).toBe(1); + }); + test("selects pool auth independently of the routed provider", async () => { saveCodexAccountCredential("pool-a", { accessToken: "pool_token", diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 448246e1b2..d1dc0b26a8 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -3009,6 +3009,121 @@ describe("server local API auth", () => { } }); + test.each([429, 402] as const)( + "a pre-stream %i from the only Pool account retries once with the validated caller main", + async rejection => { + setDebugSettings({ debug: true }); + const model = "gpt-daybreak-blue-latest"; + const observed: Array<{ authorization: string | null; accountId: string | null }> = []; + const harness = await startPoolRetryHarness((_accountId, request) => { + observed.push({ + authorization: request.headers.get("authorization"), + accountId: request.headers.get("chatgpt-account-id"), + }); + if (observed.length === 1) { + return new Response(JSON.stringify({ error: { message: "pool account unavailable" } }), { + status: rejection, + headers: { "content-type": "application/json", "retry-after": "60" }, + }); + } + return Response.json({ id: "caller-main-success", status: "completed", output: [] }); + }, { + secondAccount: false, + modelRosterByAccount: { + "acct-pool-a": [model], + "acct-caller-main": [model], + }, + }); + try { + const response = await harness.request({ + model, + headers: { "chatgpt-account-id": "acct-caller-main" }, + }); + expect(response.status).toBe(200); + expect((await response.json() as { id: string }).id).toBe("caller-main-success"); + expect(observed).toEqual([ + { authorization: "Bearer pool-a-token", accountId: "acct-pool-a" }, + { authorization: "Bearer inbound-token", accountId: "acct-caller-main" }, + ]); + expect(harness.dispatches).toEqual(["acct-pool-a", "acct-caller-main"]); + expect(loadConfig().activeCodexAccountId).toBe("pool-a"); + const affinity = getDebugLogEntries() + .map(entry => entry.line) + .filter(line => line.startsWith("[ocx:codex:affinity] ")) + .map(line => JSON.parse(line.slice("[ocx:codex:affinity] ".length)) as { + status: number; + authKind: string; + credentialSubstituted: boolean; + }); + expect(affinity.slice(-2)).toEqual([ + expect.objectContaining({ status: rejection, authKind: "pool", credentialSubstituted: true }), + expect.objectContaining({ status: 200, authKind: "main", credentialSubstituted: false }), + ]); + } finally { + await stopPoolRetryHarness(harness); + } + }, + { timeout: SERVER_BUDGET_MS }, + ); + + test.each([429, 402] as const)( + "compact %i from the only Pool account retries once with the validated caller main", + async rejection => { + const model = "gpt-daybreak-blue-latest"; + const observed: Array<{ authorization: string | null; accountId: string | null }> = []; + const harness = await startPoolRetryHarness((_accountId, request) => { + observed.push({ + authorization: request.headers.get("authorization"), + accountId: request.headers.get("chatgpt-account-id"), + }); + if (observed.length === 1) { + return new Response(JSON.stringify({ error: { message: "pool account unavailable" } }), { + status: rejection, + headers: { "content-type": "application/json", "retry-after": "60" }, + }); + } + return new Response([ + "event: response.output_item.done", + 'data: {"type":"response.output_item.done","output_index":0,"item":{"type":"compaction","encrypted_content":"gAAAAAB-caller-main"}}', + "", + "event: response.completed", + 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}', + "", + "data: [DONE]", + "", + ].join("\n"), { + headers: { "content-type": "text/event-stream" }, + }); + }, { + secondAccount: false, + modelRosterByAccount: { + "acct-pool-a": [model], + "acct-caller-main": [model], + }, + }); + try { + const response = await harness.request({ + model, + path: "/v1/responses/compact", + headers: { "chatgpt-account-id": "acct-caller-main" }, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + output: [{ type: "compaction", encrypted_content: "gAAAAAB-caller-main" }], + }); + expect(observed).toEqual([ + { authorization: "Bearer pool-a-token", accountId: "acct-pool-a" }, + { authorization: "Bearer inbound-token", accountId: "acct-caller-main" }, + ]); + expect(harness.dispatches).toEqual(["acct-pool-a", "acct-caller-main"]); + expect(loadConfig().activeCodexAccountId).toBe("pool-a"); + } finally { + await stopPoolRetryHarness(harness); + } + }, + { timeout: SERVER_BUDGET_MS }, + ); + test("#584: Retry-After cools the first account even when its account retry fails", async () => { const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" ? new Response(JSON.stringify({ error: { message: "rate limited" } }), { diff --git a/tests/subagent-fallback-handle-responses.test.ts b/tests/subagent-fallback-handle-responses.test.ts index 9212cbafe3..9bd4dd1705 100644 --- a/tests/subagent-fallback-handle-responses.test.ts +++ b/tests/subagent-fallback-handle-responses.test.ts @@ -263,6 +263,7 @@ async function postDirectCodex( config: OcxConfig, body: Record, options: Parameters[3] = {}, + headers: HeadersInit = {}, ): Promise { return handleResponses( new Request("http://localhost/v1/responses", { @@ -270,6 +271,7 @@ async function postDirectCodex( headers: { "content-type": "application/json", authorization: "Bearer caller-codex-token", + ...headers, }, body: JSON.stringify(body), }), @@ -1776,6 +1778,57 @@ describe("account-gated retry entitlement boundary", () => { expect(selectionReleases).toBe(3); }); + test("a lost Pool model grant retries once with the validated caller-owned main credential", async () => { + const now = 1_800_000_000_000; + Date.now = () => now; + installPoolCredential("pool-a", "pool_acc_a", now); + const cfg = retryConfig(); + let entitlementCalls = 0; + const observed: Array<{ authorization: string | null; accountId: string | null }> = []; + let callerRosterReads = 0; + globalThis.fetch = (async (input, init) => { + const url = new URL(String(input)); + const headers = new Headers(init?.headers); + if (url.pathname.endsWith("/models")) { + callerRosterReads += 1; + expect(headers.get("authorization")).toBe("Bearer caller-codex-token"); + expect(headers.get("chatgpt-account-id")).toBe("caller-main-account"); + return Response.json({ + models: [{ slug: model, supported_in_api: true, visibility: "list" }], + }); + } + observed.push({ + authorization: headers.get("authorization"), + accountId: headers.get("chatgpt-account-id"), + }); + return observed.length === 1 + ? unsupportedCodexModelResponse(model) + : Response.json({ id: "caller-main-success", status: "completed", output: [] }); + }) as typeof fetch; + + const response = await postDirectCodex( + cfg, + { model, input: "hello", stream: false }, + { + resolveCodexModelEntitlements: async () => { + entitlementCalls += 1; + return entitlementCalls === 1 + ? entitlementSnapshot({ "pool-a": [model] }) + : entitlementSnapshot({ "pool-a": ["gpt-5.6-sol"] }); + }, + }, + { "chatgpt-account-id": "caller-main-account" }, + ); + + expect(response.status).toBe(200); + expect(observed).toEqual([ + { authorization: "Bearer pool-a_token", accountId: "pool_acc_a" }, + { authorization: "Bearer caller-codex-token", accountId: "caller-main-account" }, + ]); + expect(callerRosterReads).toBe(1); + expect(entitlementCalls).toBe(3); + }); + test("a first-refresh programmer error cancels the 400 and releases its quota probe", async () => { const cooldownAt = 1_800_000_000_000; const probeAt = cooldownAt + CODEX_QUOTA_PROBE_INTERVAL_MS;