feat(trace): audit model calls across routes

CHANGELOG.md

@@ -8,6 +8,10 @@ tagged release also ships native binaries for Linux, macOS, and Windows.
 
 ## Unreleased
 
+- Stop advertising GPT-5.4 and GPT-5.4 Mini through ChatGPT/Codex subscriptions, whose endpoint rejects those models; they remain available through OpenAI API-key connections.
+
+- Model calls now retain provider request IDs across managed, API-key, and ChatGPT subscription routes, and `openscience debug audit <session>` reports content-free call identities for visible, internal, and worker calls.
+
 - Managed model steps record the HTTP call and Ace hold that produced them, making each charge auditable.
 
 - Settings → Usage no longer fails when a saved session or message record has no id; such records are skipped.

backend/cli/src/cli/cmd/debug/audit.ts

@@ -0,0 +1,21 @@
+import { EOL } from "os"
+import { CallAudit } from "../../../session/call-audit"
+import { bootstrap } from "../../bootstrap"
+import { cmd } from "../cmd"
+
+export const AuditCommand = cmd({
+  command: "audit <session>",
+  describe: "show model-call identities recorded for a session and its workers",
+  builder: (yargs) =>
+    yargs.positional("session", {
+      type: "string",
+      demandOption: true,
+      description: "session id",
+    }),
+  async handler(args) {
+    await bootstrap(process.cwd(), async () => {
+      const report = await CallAudit.report(args.session as string)
+      process.stdout.write(JSON.stringify(report, null, 2) + EOL)
+    })
+  },
+})

backend/cli/src/cli/cmd/debug/index.ts

@@ -10,6 +10,7 @@ import { SkillCommand } from "./skill"
 import { SnapshotCommand } from "./snapshot"
 import { AgentCommand } from "./agent"
 import { CapabilityCanaryCommand } from "./capability-canary"
+import { AuditCommand } from "./audit"
 
 export const DebugCommand = cmd({
   command: "debug",
@@ -25,6 +26,7 @@ export const DebugCommand = cmd({
       .command(SnapshotCommand)
       .command(AgentCommand)
       .command(CapabilityCanaryCommand)
+      .command(AuditCommand)
       .command(PathsCommand)
       .command(
         cmd({

backend/cli/src/provider/call-link.ts

@@ -5,17 +5,44 @@
 export namespace CallLink {
   export const CALL_HEADER = "x-openscience-call"
   export const HOLD_HEADER = "x-openscience-hold-id"
+  export const PROVIDER_REQUEST_HEADER = "x-openscience-provider-request-id"
   const HOLD = /^(hold|orgh)_[A-Za-z0-9_]{1,80}$/
   const CALL = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+  const PROVIDER_REQUEST = /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,199}$/
+  const PROVIDER_HEADERS = [
+    "x-generation-id",
+    "x-request-id",
+    "x-oai-request-id",
+    "request-id",
+    "openai-request-id",
+    "x-goog-request-id",
+    "x-amzn-requestid",
+    "x-amzn-request-id",
+  ]
 
   function validHold(value: string | null | undefined) {
     return value && HOLD.test(value) ? value : undefined
   }
 
+  function validProviderRequest(value: string | null | undefined) {
+    return value && PROVIDER_REQUEST.test(value) ? value : undefined
+  }
+
   export function hold(headers: Headers) {
     return validHold(headers.get(HOLD_HEADER))
   }
 
+  /** Provider-owned response identity. Body metadata wins because streaming
+   * APIs often expose their durable generation id only after response headers. */
+  export function providerRequest(headers: Headers, responseID?: string) {
+    const body = validProviderRequest(responseID)
+    if (body) return body
+    for (const header of PROVIDER_HEADERS) {
+      const value = validProviderRequest(headers.get(header))
+      if (value) return value
+    }
+  }
+
   export function headers(
     base: HeadersInit | undefined,
     input: { call: string; sessionID: string; messageID: string; attempt: number },
@@ -30,20 +57,27 @@ export namespace CallLink {
 
   /** Response headers the wrapper returns: always this attempt's call id, and
    * the hold only when it came from the managed gateway in a valid shape. */
-  export function responseHeaders(source: Headers, input: { call: string; hold?: string }) {
+  export function responseHeaders(source: Headers, input: { call: string; hold?: string; providerRequest?: string }) {
     const result = new Headers(source)
     result.set(CALL_HEADER, input.call)
     if (input.hold) result.set(HOLD_HEADER, input.hold)
     else result.delete(HOLD_HEADER)
+    if (input.providerRequest) result.set(PROVIDER_REQUEST_HEADER, input.providerRequest)
+    else result.delete(PROVIDER_REQUEST_HEADER)
     return result
   }
 
   /** The call identity of one step, from the response headers the AI SDK
    * attaches to that step (a lowercase record). */
-  export function fromResponse(headers: Record<string, string | undefined> | undefined) {
+  export function fromResponse(headers: Record<string, string | undefined> | undefined, responseID?: string) {
     const id = headers?.[CALL_HEADER]
     if (!id || !CALL.test(id)) return undefined
     const hold = validHold(headers[HOLD_HEADER])
-    return hold ? { id, hold } : { id }
+    const providerRequest = validProviderRequest(responseID) ?? validProviderRequest(headers[PROVIDER_REQUEST_HEADER])
+    return {
+      id,
+      ...(hold && { hold }),
+      ...(providerRequest && { providerRequest }),
+    }
   }
 }

backend/cli/src/provider/provider.ts

@@ -572,6 +572,7 @@ export namespace Provider {
     const linked = CallLink.responseHeaders(response.headers, {
       call,
       hold: options.managed ? CallLink.hold(response.headers) : undefined,
+      providerRequest: CallLink.providerRequest(response.headers),
     })
     // Response.error()/opaque responses use status 0, which the Response
     // constructor forbids. They do not expose a consumable network body, so
@@ -664,10 +665,6 @@ export namespace Provider {
     "gpt-5-6-luna",
     "gpt-5.5",
     "gpt-5-5",
-    "gpt-5.4",
-    "gpt-5-4",
-    "gpt-5.4-mini",
-    "gpt-5-4-mini",
   ])
 
   export function isCodexOAuthModel(modelID: string): boolean {
@@ -678,7 +675,7 @@ export namespace Provider {
     if (
       modelID !== "gpt-6-sol" &&
       modelID !== "gpt-6.1-sol" &&
-      !/^gpt-5[.-](?:4|5|6(?:-(?:sol|terra|luna))?)$/.test(modelID)
+      !/^gpt-5[.-](?:5|6(?:-(?:sol|terra|luna))?)$/.test(modelID)
     )
       return undefined
     return {

backend/cli/src/session/call-audit.ts

@@ -0,0 +1,77 @@
+import { Session } from "."
+import { SessionTraceStore } from "./trace-store"
+import { UsageLogging } from "./usage-logging"
+
+export namespace CallAudit {
+  export async function report(rootID: string) {
+    await Session.get(rootID)
+    const pending = [rootID]
+    const seen = new Set<string>()
+    const sessions: { sessionID: string; parentID?: string; telemetrySessionID: string }[] = []
+    const calls: {
+      sessionID: string
+      telemetrySessionID: string
+      messageID: string
+      stepID?: string
+      route: "managed" | "byok" | "chatgpt" | "subscription" | "local" | "custom"
+      provider: string
+      model: string
+      occurredAt: string
+      httpCall?: string
+      providerRequest?: string
+      hold?: string
+      recordedTokens?: { input: number; output: number; reasoning: number; cache: { read: number; write: number } }
+      recordedCostUSD?: number
+    }[] = []
+    while (pending.length) {
+      const sessionID = pending.shift()!
+      if (seen.has(sessionID)) continue
+      seen.add(sessionID)
+      const session = await Session.get(sessionID)
+      const telemetrySessionID = await UsageLogging.telemetrySessionID(sessionID)
+      sessions.push({ sessionID, ...(session.parentID && { parentID: session.parentID }), telemetrySessionID })
+      for (const child of await Session.children(sessionID)) pending.push(child.id)
+      const recorded = new Map<string, number>()
+      for (const call of (await SessionTraceStore.read(sessionID)).modelCalls) {
+        recorded.set(call.id, calls.length)
+        calls.push({
+          sessionID,
+          telemetrySessionID,
+          messageID: call.messageID,
+          route: call.route,
+          provider: call.provider,
+          model: call.model,
+          occurredAt: new Date(call.occurredAt).toISOString(),
+          httpCall: call.id,
+          ...(call.providerRequest && { providerRequest: call.providerRequest }),
+          ...(call.hold && { hold: call.hold }),
+        })
+      }
+      for (const message of await Session.messages({ sessionID })) {
+        for (const part of message.parts) {
+          if (part.type !== "step-finish" || !part.usage) continue
+          const call = {
+            sessionID,
+            telemetrySessionID,
+            messageID: part.messageID,
+            stepID: part.id,
+            route: part.usage.route,
+            provider: part.usage.provider,
+            model: part.usage.model,
+            occurredAt: new Date(part.usage.time).toISOString(),
+            ...(part.usage.call && { httpCall: part.usage.call }),
+            ...(part.usage.providerRequest && { providerRequest: part.usage.providerRequest }),
+            ...(part.usage.hold && { hold: part.usage.hold }),
+            recordedTokens: part.tokens,
+            recordedCostUSD: part.cost,
+          }
+          const index = part.usage.call ? recorded.get(part.usage.call) : undefined
+          if (index === undefined) calls.push(call)
+          else calls[index] = { ...calls[index], ...call }
+        }
+      }
+    }
+    calls.sort((a, b) => a.occurredAt.localeCompare(b.occurredAt) || (a.stepID ?? "").localeCompare(b.stepID ?? ""))
+    return { rootSessionID: rootID, sessions, calls }
+  }
+}

backend/cli/src/session/llm.ts

@@ -295,11 +295,12 @@ export namespace LLM {
     ]
     const logging = await UsageLogging.context().catch(() => undefined)
     const started = performance.now()
+    const traceMessageID = input.trace?.messageID ?? input.user.id
     const binding = logging
       ? {
           context: logging,
           sessionID: input.sessionID,
-          messageID: input.trace?.messageID ?? input.user.id,
+          messageID: traceMessageID,
           route: traceRoute,
           provider: routed.providerID,
           model: routed.api.id,
@@ -324,6 +325,27 @@ export namespace LLM {
         await capture("assistant.message", { ...partial, interrupted: true })
       },
       async onStepFinish(step) {
+        const call = CallLink.fromResponse(step.response?.headers, step.response?.id)
+        if (call) {
+          await SessionTraceStore.recordModelCall({
+            id: call.id,
+            sessionID: input.sessionID,
+            messageID: traceMessageID,
+            route:
+              traceRoute === "managed" ||
+              traceRoute === "byok" ||
+              traceRoute === "chatgpt" ||
+              traceRoute === "subscription" ||
+              traceRoute === "local"
+                ? traceRoute
+                : "custom",
+            provider: routed.providerID,
+            model: routed.api.id,
+            occurredAt: Date.now(),
+            ...(call.providerRequest && { providerRequest: call.providerRequest }),
+            ...(call.hold && { hold: call.hold }),
+          })
+        }
         if (!binding) return
         await UsageLogging.record({
           ...binding,
@@ -332,7 +354,7 @@ export namespace LLM {
           duration: performance.now() - started,
           content: { parts: step.content, toolResults: step.toolResults },
           finish: step.finishReason,
-          call: CallLink.fromResponse(step.response?.headers),
+          call,
         }).catch(() => l.warn("could not persist usage record"))
       },
       async onError(error) {

backend/cli/src/session/message-v2.ts

@@ -291,6 +291,8 @@ export namespace MessageV2 {
         call: z.string().optional(),
         /** Atlas hold that reserved this step's Ace charge (managed route only). */
         hold: z.string().optional(),
+        /** Provider-owned response or request identity, when the route exposes one. */
+        providerRequest: z.string().optional(),
       })
       .optional(),
     /** Delivered speed when the provider reports it; otherwise the selected speed. */

backend/cli/src/session/processor.ts

@@ -1234,7 +1234,7 @@ export namespace SessionProcessor {
                   // The call and hold travel on this step's own response: in
                   // a multi-step tool stream the SDK may already be fetching
                   // the next step, so shared request state would name it.
-                  const stepCall = CallLink.fromResponse(value.response?.headers)
+                  const stepCall = CallLink.fromResponse(value.response?.headers, value.response?.id)
                   const funded = requiresWalletBalance(credentialSource)
                   const usage = Session.getUsage({
                     model: resolved.model,
@@ -1270,6 +1270,7 @@ export namespace SessionProcessor {
                       ...(stepCall && {
                         call: stepCall.id,
                         ...(stepCall.hold && { hold: stepCall.hold }),
+                        ...(stepCall.providerRequest && { providerRequest: stepCall.providerRequest }),
                       }),
                     },
                     tier: usage.tier,

backend/cli/src/session/trace-store.ts

@@ -38,14 +38,28 @@ export namespace SessionTraceStore {
   export const Harness = SessionHarness.Entry
   export type Harness = z.infer<typeof Harness>
 
+  export const ModelCall = z.object({
+    id: z.string(),
+    sessionID: z.string(),
+    messageID: z.string(),
+    route: z.enum(["managed", "byok", "chatgpt", "subscription", "local", "custom"]),
+    provider: z.string(),
+    model: z.string(),
+    occurredAt: z.number(),
+    providerRequest: z.string().optional(),
+    hold: z.string().optional(),
+  })
+  export type ModelCall = z.infer<typeof ModelCall>
+
   const State = z.object({
     approvals: z.record(z.string(), Approval).default({}),
     retries: z.array(Retry).default([]),
     harness: z.array(Harness).default([]),
+    modelCalls: z.array(ModelCall).default([]),
   })
   export type State = z.infer<typeof State>
 
-  const empty = (): State => ({ approvals: {}, retries: [], harness: [] })
+  const empty = (): State => ({ approvals: {}, retries: [], harness: [], modelCalls: [] })
   const file = (sessionID: string) => path.join(Global.Path.data, "trace", `${encodeURIComponent(sessionID)}.json`)
 
   async function update(sessionID: string, fn: (state: State) => State) {
@@ -133,6 +147,16 @@ export namespace SessionTraceStore {
     return update(input.sessionID, (state) => ({ ...state, harness: [...state.harness, item] }))
   }
 
+  /** Durable, content-free identity for every completed provider call,
+   * including internal title/summary calls that do not create message parts. */
+  export function recordModelCall(input: ModelCall) {
+    const item = ModelCall.parse(input)
+    return update(input.sessionID, (state) => ({
+      ...state,
+      modelCalls: [...state.modelCalls.filter((call) => call.id !== item.id), item].slice(-2_048),
+    }))
+  }
+
   export async function remove(sessionID: string) {
     await fs.unlink(file(sessionID)).catch((error: NodeJS.ErrnoException) => {
       if (error.code === "ENOENT") return

backend/cli/src/session/usage-logging.ts

@@ -237,20 +237,35 @@ export namespace UsageLogging {
     }
   }
 
+  /** Opaque session identity used by the server. It lets a local audit name
+   * its consented trace without revealing the user's local session id. */
+  export async function telemetrySessionID(sessionID: string) {
+    // Persist the installation before deriving an identity. On a fresh data
+    // root, parsing defaults alone would mint a different installation for
+    // each call and none would match a later uploaded trace.
+    if (!(await Bun.file(filepath).exists())) await update(() => {})
+    const state = await read()
+    return digest(`${state.installation}:${sessionID}`)
+  }
+
   export async function record(
     input: Binding & {
       usage: LanguageModelUsage
       metadata?: ProviderMetadata
       duration: number
       content?: unknown
       finish?: string
-      call?: { id: string; hold?: string }
+      call?: { id: string; hold?: string; providerRequest?: string }
     },
   ) {
     return event(input, "model.response", {
       ...reported(input.usage, input.metadata),
       duration_ms: input.duration,
-      ...(input.call && { http_call_id: input.call.id, ...(input.call.hold && { hold: input.call.hold }) }),
+      ...(input.call && {
+        http_call_id: input.call.id,
+        ...(input.call.hold && { hold: input.call.hold }),
+        ...(input.call.providerRequest && { provider_request_id: input.call.providerRequest }),
+      }),
       ...(input.content === undefined ? {} : { content: input.content }),
       ...(input.finish === undefined ? {} : { finish: input.finish }),
     })