Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 21 additions & 18 deletions src/codex/auth-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import {
entitledCodexAccountIdsForModel,
isDirectCallerEntitledToCodexModel,
resolveCodexModelEntitlements,
type CodexModelEntitlementSnapshot,
} from "./model-entitlements";
import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "./catalog/native-models";
import type { CodexCooldownSource, CodexQuotaScope } from "./routing";
Expand Down Expand Up @@ -334,9 +333,7 @@ export interface ResolveCodexAuthContextOptions {
getMainAccountToken?: typeof getMainAccountToken;
primeCodexPoolQuotas?: (config: OcxConfig, reason: string) => Promise<void>;
/** Test seam for account-gated native model discovery. */
resolveCodexModelEntitlements?: (
config: Pick<OcxConfig, "codexAccounts">,
) => Promise<CodexModelEntitlementSnapshot>;
resolveCodexModelEntitlements?: typeof resolveCodexModelEntitlements;
/** Direct requests admitted with a proxy bearer substitute the stored native-main credential. */
substituteMainCredentialForDirect?: boolean;
/** Test seam for a Direct request's own forwarded ChatGPT credential. */
Expand Down Expand Up @@ -381,28 +378,34 @@ export async function resolveCodexAuthContext(
return { kind: "main", accountId: null };
}
const affinityKey = fixedAccountId === undefined ? codexPoolAffinityKey(headers) : undefined;
const entitlementSnapshot = options.modelId && ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(options.modelId)
? await (options.resolveCodexModelEntitlements ?? resolveCodexModelEntitlements)(config)
: undefined;
const modelEligibleAccountIds = entitlementSnapshot
? entitledCodexAccountIdsForModel(entitlementSnapshot, options.modelId)
: undefined;
// Retained startup recovery makes the physical main identity ineligible. Routing
// can still preserve service by selecting a healthy configured pool account.
const nativeMainTrafficBlocked = isNativeMainTrafficBlocked();
const selectionAdmission = options.beginCodexAccountSelection?.();
const nativeMainReadsForbidden = nativeMainTrafficBlocked || selectionAdmission?.mainProfileDraining === true;
const selectionOptions = {
// Temporary switch drain keeps the candidate until the atomic claim rejects
// it. Retained recovery makes main wholly ineligible so pool routing continues.
nativeMainSelectionOnly: !nativeMainTrafficBlocked
&& selectionAdmission?.mainProfileDraining === true,
isMainAccountTokenLive: options.isMainAccountTokenLive,
modelEligibleAccountIds,
};
let accountId: string;
const quotaScope = codexQuotaScopeForModel(options.modelId);
try {
const excludeAccountIds = nativeMainReadsForbidden
? new Set([MAIN_CODEX_ACCOUNT_ID])
: undefined;
const entitlementSnapshot = options.modelId && ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(options.modelId)
? await (options.resolveCodexModelEntitlements ?? resolveCodexModelEntitlements)(config, { excludeAccountIds })
: undefined;
const entitledAccountIds = entitlementSnapshot
? entitledCodexAccountIdsForModel(entitlementSnapshot, options.modelId)
: undefined;
const modelEligibleAccountIds = entitledAccountIds
? new Set([...entitledAccountIds].filter(candidate => !excludeAccountIds?.has(candidate)))
: undefined;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
const selectionOptions = {
// Temporary switch drain keeps the candidate until the atomic claim rejects
// it. Retained recovery makes main wholly ineligible so pool routing continues.
nativeMainSelectionOnly: !nativeMainTrafficBlocked
&& selectionAdmission?.mainProfileDraining === true,
isMainAccountTokenLive: options.isMainAccountTokenLive,
modelEligibleAccountIds,
};
// A pre-drain selector reserves the native identity while reconciliation and
// routing inspect it. Selectors arriving after the fence skip reconciliation
// and may still route to non-main pool accounts without touching switch state.
Expand Down
11 changes: 9 additions & 2 deletions src/codex/model-entitlements.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ export interface CodexModelEntitlementResolveOptions {
readonly now?: number;
/** Test-only credential seam; production callers enumerate local main + Pool credentials. */
readonly credentials?: readonly CodexModelEntitlementCredentialSnapshot[];
/** Test-only seam for proving lifecycle exclusions happen before credential reads. */
readonly credentialSnapshot?: typeof accountCredentialSnapshot;
/** Accounts whose credentials must not be read while another lifecycle owns them. */
readonly excludeAccountIds?: ReadonlySet<string>;
}

const accountModelsCache = new Map<string, CachedAccountModels>();
Expand Down Expand Up @@ -252,9 +256,12 @@ export async function resolveCodexModelEntitlements(
): Promise<CodexModelEntitlementSnapshot> {
const now = options.now ?? Date.now();
const fetcher = options.fetcher ?? fetch;
const allowedAccountIds = candidateAccountIds(config)
.filter(accountId => !options.excludeAccountIds?.has(accountId));
const credentialSnapshot = options.credentialSnapshot ?? accountCredentialSnapshot;
const credentials = options.credentials
? [...options.credentials]
: (await Promise.all(candidateAccountIds(config).map(accountCredentialSnapshot)))
? [...options.credentials].filter(credential => !options.excludeAccountIds?.has(credential.accountId))
: (await Promise.all(allowedAccountIds.map(credentialSnapshot)))
.filter((value): value is CodexModelEntitlementCredentialSnapshot => value !== null);
const results = await Promise.all(credentials.map(async credential => ({
credential,
Expand Down
77 changes: 62 additions & 15 deletions src/codex/subagent-model-fallback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import {
import { routeModel, type RouteResult } from "../router";
import { sweepExpiredOnWrite } from "../lib/state-store-sweeper";
import { codexAccountNamespaceForModel } from "./account-namespace-match";
import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "./catalog/native-models";
import {
getUpstreamHostHealth,
normalizeUpstreamHostCircuitThreshold,
Expand All @@ -52,7 +53,11 @@ type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise<void>
export type SubagentPoolAccountPreview = (
modelId: string | undefined,
now: number,
modelEligibleAccountIds?: ReadonlySet<string>,
) => string | null;
export type SubagentModelEligibleAccountIds = (
modelId: string | undefined,
) => ReadonlySet<string> | undefined;
let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null;
let quotaPrimeInFlight: Promise<void> | null = null;

Expand Down Expand Up @@ -184,10 +189,11 @@ function resolveRouteFallbackAccountId(
accountId?: string | null,
now = Date.now(),
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIds?: ReadonlySet<string>,
): string | null {
if (route?.codexAccountId !== undefined) return route.codexAccountId;
if (route && isPoolCodexRoute(route) && poolAccountPreview) {
return poolAccountPreview(route.modelId, now);
return poolAccountPreview(route.modelId, now, modelEligibleAccountIds);
}
return resolvePoolFallbackAccountId(config, accountId);
}
Expand Down Expand Up @@ -249,17 +255,26 @@ export function isSubagentModelUnavailable(
now = Date.now(),
accountUsabilityOptions?: CodexAccountUsabilityOptions,
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIdsForModel?: SubagentModelEligibleAccountIds,
): boolean {
if (isDisabledFallbackModel(model, config)) return true;
if (!isRoutableFallbackModel(model, config)) return true;
const route = tryRouteFallbackModel(config, model);
if (!route || route.provider.disabled === true) return true;
const modelEligibleAccountIds = modelEligibleAccountIdsForModel?.(route.modelId);
const candidateAccountUsabilityOptions = modelEligibleAccountIds !== undefined
? {
...accountUsabilityOptions,
modelEligibleAccountIds,
}
: accountUsabilityOptions;
const resolvedAccountId = resolveRouteFallbackAccountId(
route,
config,
accountId,
now,
poolAccountPreview,
candidateAccountUsabilityOptions?.modelEligibleAccountIds,
);
if (isModelHealthBlocked(model, config, resolvedAccountId, now)) return true;
if (!isPoolCodexRoute(route)) return false;
Expand All @@ -268,7 +283,7 @@ export function isSubagentModelUnavailable(
// route (canonical openai defaults to pool even when codexAccountMode is omitted).
if (!resolvedAccountId) return true;
if (isCodexAccountPaused(config, resolvedAccountId)) return true;
if (!isCodexAccountUsable(config, resolvedAccountId, accountUsabilityOptions)) return true;
if (!isCodexAccountUsable(config, resolvedAccountId, candidateAccountUsabilityOptions)) return true;
if (route.codexAccountId !== undefined) {
// An account-qualified route is pinned and cannot consume Pool's recovery-probe
// escape hatch. Honor both account-wide and model-scoped cooldowns so fallback
Expand Down Expand Up @@ -298,8 +313,10 @@ export function selectAvailableSubagentModel(
accountUsabilityOptions?: CodexAccountUsabilityOptions,
trailingFallback: readonly string[] = [],
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIdsForModel?: SubagentModelEligibleAccountIds,
resolvedChain?: readonly string[],
): { model: string; rewritten: boolean; skipped: string[] } {
const chain = normalizedChain(primary, config, extraFallback, trailingFallback);
const chain = resolvedChain ?? normalizedChain(primary, config, extraFallback, trailingFallback);
const skipped: string[] = [];
for (const candidate of chain) {
if (nativeFallbackOnly) {
Expand All @@ -316,6 +333,7 @@ export function selectAvailableSubagentModel(
now,
accountUsabilityOptions,
poolAccountPreview,
modelEligibleAccountIdsForModel,
)) {
skipped.push(candidate);
continue;
Expand Down Expand Up @@ -546,28 +564,26 @@ export function applySubagentModelFallback(
nativeFallbackOnly = false,
accountUsabilityOptions?: CodexAccountUsabilityOptions,
poolAccountPreview?: SubagentPoolAccountPreview,
modelEligibleAccountIdsForModel?: SubagentModelEligibleAccountIds,
resolvedFallbackChain?: readonly string[] | null,
): { from?: string; to?: string; skipped?: string[] } | null {
if (!isThreadSpawnRequest(headers)) return null;
const tomlRoleFallback = resolveAgentModelFallbackForPrimary(
parsed.modelId,
getCodexHome(),
config.codexAccountNamespaces,
);
// Config-keyed chains are the supported per-role home (#1190); TOML `model_fallback`
// stays readable for backwards compatibility with homes written before Codex 0.146.
const configuredFallback = resolveConfiguredModelFallbackForPrimary(parsed.modelId, config);
const globalFallback = config.subagentModelFallback ?? [];
if (globalFallback.length === 0 && configuredFallback.length === 0 && tomlRoleFallback.length === 0) return null;
const fallbackChain = resolvedFallbackChain === undefined
? resolveSubagentFallbackChain(parsed, config)
: resolvedFallbackChain;
if (!fallbackChain) return null;
const selection = selectAvailableSubagentModel(
parsed.modelId,
config,
configuredFallback,
[],
accountId,
now,
nativeFallbackOnly,
accountUsabilityOptions,
tomlRoleFallback,
[],
poolAccountPreview,
modelEligibleAccountIdsForModel,
fallbackChain,
);
if (!selection.rewritten) return selection.skipped.length > 0
? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped }
Expand All @@ -577,6 +593,37 @@ export function applySubagentModelFallback(
return { from, to: selection.model, skipped: selection.skipped };
}

/** Resolve the effective fallback chain once for one logical spawn request. */
export function resolveSubagentFallbackChain(
parsed: OcxParsedRequest,
config: OcxConfig,
): readonly string[] | null {
const tomlRoleFallback = resolveAgentModelFallbackForPrimary(
parsed.modelId,
getCodexHome(),
config.codexAccountNamespaces,
);
// Config-keyed chains are the supported per-role home (#1190); TOML `model_fallback`
// stays readable for backwards compatibility with homes written before Codex 0.146.
const configuredFallback = resolveConfiguredModelFallbackForPrimary(parsed.modelId, config);
const globalFallback = config.subagentModelFallback ?? [];
if (globalFallback.length === 0 && configuredFallback.length === 0 && tomlRoleFallback.length === 0) return null;
return normalizedChain(parsed.modelId, config, configuredFallback, tomlRoleFallback);
}

/** Whether the effective fallback chain crosses an account-gated native Pool model. */
export function subagentFallbackNeedsModelEntitlements(
fallbackChain: readonly string[] | null,
config: OcxConfig,
): boolean {
return fallbackChain?.some((candidate) => {
const route = tryRouteFallbackModel(config, candidate);
return !!route
&& isPoolCodexRoute(route)
&& ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId);
}) === true;
}

export function subagentFallbackGuidanceText(config: OcxConfig): string {
const chain = config.subagentModelFallback ?? [];
if (chain.length === 0) return "";
Expand Down
59 changes: 54 additions & 5 deletions src/server/responses/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ import {
resolveCodexModelEntitlements,
} from "../../codex/model-entitlements";
import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "../../codex/catalog/native-models";
import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account";
import { captureCodexAffinityDiagnostic } from "../../codex/affinity-debug";
import {
computeQuotaCooldown,
Expand Down Expand Up @@ -217,6 +218,9 @@ import {
applySubagentModelFallback,
maybePrimeSubagentQuota,
recordSubagentQuotaFailureForThreadSpawn,
resolveSubagentFallbackChain,
subagentFallbackNeedsModelEntitlements,
type SubagentModelEligibleAccountIds,
type SubagentPoolAccountPreview,
} from "../../codex/subagent-model-fallback";
import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup";
Expand Down Expand Up @@ -1291,6 +1295,8 @@ export interface HandleResponsesOptions {
/** One-shot TTFT callback: first non-empty model output observed (WP4). */
onFirstOutput?: () => void;
onCodexAuthContextResolved?: (context: CodexAuthContext | undefined) => void;
/** Internal deterministic seam for account-gated native fallback tests. */
resolveCodexModelEntitlements?: typeof resolveCodexModelEntitlements;
recordTerminalOutcomes?: boolean;
setTerminalOutcomeRecorder?: (recorder: ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined) => void;
onNativePassthroughTerminal?: (status: ResponsesTerminalStatus) => void;
Expand Down Expand Up @@ -1590,6 +1596,7 @@ async function resolveResponsesCodexAuth(
modelId: route.modelId,
substituteMainCredentialForDirect: substituteMainCredential,
beginCodexAccountSelection: codexAccountSelectionForTurn(options.turnAdmissionLease),
resolveCodexModelEntitlements: options.resolveCodexModelEntitlements,
});
options.onCodexAuthContextResolved?.(authCtx);
} else {
Expand Down Expand Up @@ -1628,6 +1635,25 @@ async function resolveResponsesCodexAuth(
}
}

async function resolveSubagentFallbackModelEligibility(args: {
config: OcxConfig;
fallbackChain: readonly string[] | null;
nativeMainReadsForbidden: boolean;
resolver: typeof resolveCodexModelEntitlements;
}): Promise<SubagentModelEligibleAccountIds | undefined> {
if (!subagentFallbackNeedsModelEntitlements(args.fallbackChain, args.config)) return undefined;
const excludeAccountIds = args.nativeMainReadsForbidden
? new Set([MAIN_CODEX_ACCOUNT_ID])
: undefined;
const snapshot = await args.resolver(args.config, { excludeAccountIds });
return (modelId) => {
const entitledAccountIds = entitledCodexAccountIdsForModel(snapshot, modelId);
return entitledAccountIds
? new Set([...entitledAccountIds].filter(accountId => !excludeAccountIds?.has(accountId)))
: undefined;
};
}

/**
* Apply every route-dependent request mutation against the final selected route.
* Must run only after subagent fallback has settled the model/provider.
Expand Down Expand Up @@ -2435,7 +2461,12 @@ async function handleResponsesInner(
// also fail closed without polling quota upstream. Cached fallback state can still select a
// provider with native continuation support below.
const threadSpawn = isThreadSpawnRequest(req.headers);
const previewSelectionAdmission = threadSpawn && route.codexAccountId === undefined
const initialSubagentFallbackChain = threadSpawn && !options.comboAttempt
? resolveSubagentFallbackChain(parsed, config)
: null;
const previewSelectionAdmission = threadSpawn
&& !options.comboAttempt
&& (route.codexAccountId === undefined || initialSubagentFallbackChain !== null)
? codexAccountSelectionForTurn(options.turnAdmissionLease)?.()
: undefined;
const nativeMainRecoveryBlocked = isNativeMainTrafficBlocked();
Expand All @@ -2448,6 +2479,7 @@ async function handleResponsesInner(
let selectedForwardHeaders = req.headers;
let subagentFallbackAccountId = config.activeCodexAccountId ?? null;
let subagentFallbackAccountPreview: SubagentPoolAccountPreview | undefined;
let subagentFallbackModelEligibleAccountIdsForModel: SubagentModelEligibleAccountIds | undefined;
let subagentQuotaFailureModel = parsed.modelId;
const parentThreadId = req.headers.get("x-codex-parent-thread-id")?.trim() ?? null;
const poolAffinityKey = codexPoolAffinityKey(req.headers) ?? null;
Expand All @@ -2465,20 +2497,35 @@ async function handleResponsesInner(
// normalization (virtual models, effort caps, service tier, wire protocol).
// Preview the preferred Codex account without acquiring a probe lease or refreshing
// tokens — auth is resolved only after the final route is selected.
if (threadSpawn && !options.comboAttempt && route.codexAccountId === undefined) {
if (
threadSpawn
&& !options.comboAttempt
&& (route.codexAccountId === undefined || initialSubagentFallbackChain !== null)
) {
// The final resolveCodexAuthContext binds under codexQuotaScopeForModel(route.modelId),
// so the preview must read the same scope slot — an undefined scope would map to the
// "legacy" affinity bucket and never find a binding made under "shared" or a native
// model scope, making the preview diverge from the account that actually authenticates.
const fallbackChain = initialSubagentFallbackChain;
subagentFallbackModelEligibleAccountIdsForModel = await resolveSubagentFallbackModelEligibility({
config,
fallbackChain,
nativeMainReadsForbidden,
resolver: options.resolveCodexModelEntitlements ?? resolveCodexModelEntitlements,
});
Comment thread
coderabbitai[bot] marked this conversation as resolved.
const fallbackNow = Date.now();
subagentFallbackAccountPreview = (modelId, previewNow) => previewCodexAccountForRequest(
subagentFallbackAccountPreview = (modelId, previewNow, modelEligibleAccountIds) => previewCodexAccountForRequest(
poolAffinityKey,
config,
previewNow,
codexQuotaScopeForModel(modelId),
previewSelectionOptions,
{ ...previewSelectionOptions, modelEligibleAccountIds },
);
const previewAccountId = route.codexAccountId ?? subagentFallbackAccountPreview(
route.modelId,
fallbackNow,
subagentFallbackModelEligibleAccountIdsForModel?.(route.modelId),
);
const previewAccountId = subagentFallbackAccountPreview(route.modelId, fallbackNow);
subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null;
const fallback = applySubagentModelFallback(
parsed,
Expand All @@ -2489,6 +2536,8 @@ async function handleResponsesInner(
unreadableEncryptedAgentTask,
previewSelectionOptions,
subagentFallbackAccountPreview,
subagentFallbackModelEligibleAccountIdsForModel,
fallbackChain,
);
if (fallback) {
(logCtx as unknown as Record<string, unknown>).subagentModelFallbackFrom = fallback.from;
Expand Down
Loading
Loading