diff --git a/CHANGELOG.md b/CHANGELOG.md index b849e0623..6dcf60608 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ tagged release also ships native binaries for Linux, macOS, and Windows. ## Unreleased +- Stop advertising GPT-5.4 and GPT-5.4 Mini through ChatGPT/Codex subscriptions, whose endpoint rejects those models; they remain available through OpenAI API-key connections. + +- Model calls now retain provider request IDs across managed, API-key, and ChatGPT subscription routes, and `openscience debug audit ` reports content-free call identities for visible, internal, and worker calls. + - Managed model steps record the HTTP call and Ace hold that produced them, making each charge auditable. - Settings → Usage no longer fails when a saved session or message record has no id; such records are skipped. diff --git a/backend/cli/src/cli/cmd/debug/audit.ts b/backend/cli/src/cli/cmd/debug/audit.ts new file mode 100644 index 000000000..6ee89f1b6 --- /dev/null +++ b/backend/cli/src/cli/cmd/debug/audit.ts @@ -0,0 +1,21 @@ +import { EOL } from "os" +import { CallAudit } from "../../../session/call-audit" +import { bootstrap } from "../../bootstrap" +import { cmd } from "../cmd" + +export const AuditCommand = cmd({ + command: "audit ", + describe: "show model-call identities recorded for a session and its workers", + builder: (yargs) => + yargs.positional("session", { + type: "string", + demandOption: true, + description: "session id", + }), + async handler(args) { + await bootstrap(process.cwd(), async () => { + const report = await CallAudit.report(args.session as string) + process.stdout.write(JSON.stringify(report, null, 2) + EOL) + }) + }, +}) diff --git a/backend/cli/src/cli/cmd/debug/index.ts b/backend/cli/src/cli/cmd/debug/index.ts index d7537c3ec..d202df66e 100644 --- a/backend/cli/src/cli/cmd/debug/index.ts +++ b/backend/cli/src/cli/cmd/debug/index.ts @@ -10,6 +10,7 @@ import { SkillCommand } from "./skill" import { SnapshotCommand } from "./snapshot" import { AgentCommand } from "./agent" import { CapabilityCanaryCommand } from "./capability-canary" +import { AuditCommand } from "./audit" export const DebugCommand = cmd({ command: "debug", @@ -25,6 +26,7 @@ export const DebugCommand = cmd({ .command(SnapshotCommand) .command(AgentCommand) .command(CapabilityCanaryCommand) + .command(AuditCommand) .command(PathsCommand) .command( cmd({ diff --git a/backend/cli/src/provider/call-link.ts b/backend/cli/src/provider/call-link.ts index a387fe27d..41a065871 100644 --- a/backend/cli/src/provider/call-link.ts +++ b/backend/cli/src/provider/call-link.ts @@ -5,17 +5,44 @@ export namespace CallLink { export const CALL_HEADER = "x-openscience-call" export const HOLD_HEADER = "x-openscience-hold-id" + export const PROVIDER_REQUEST_HEADER = "x-openscience-provider-request-id" const HOLD = /^(hold|orgh)_[A-Za-z0-9_]{1,80}$/ const CALL = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + const PROVIDER_REQUEST = /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,199}$/ + const PROVIDER_HEADERS = [ + "x-generation-id", + "x-request-id", + "x-oai-request-id", + "request-id", + "openai-request-id", + "x-goog-request-id", + "x-amzn-requestid", + "x-amzn-request-id", + ] function validHold(value: string | null | undefined) { return value && HOLD.test(value) ? value : undefined } + function validProviderRequest(value: string | null | undefined) { + return value && PROVIDER_REQUEST.test(value) ? value : undefined + } + export function hold(headers: Headers) { return validHold(headers.get(HOLD_HEADER)) } + /** Provider-owned response identity. Body metadata wins because streaming + * APIs often expose their durable generation id only after response headers. */ + export function providerRequest(headers: Headers, responseID?: string) { + const body = validProviderRequest(responseID) + if (body) return body + for (const header of PROVIDER_HEADERS) { + const value = validProviderRequest(headers.get(header)) + if (value) return value + } + } + export function headers( base: HeadersInit | undefined, input: { call: string; sessionID: string; messageID: string; attempt: number }, @@ -30,20 +57,27 @@ export namespace CallLink { /** Response headers the wrapper returns: always this attempt's call id, and * the hold only when it came from the managed gateway in a valid shape. */ - export function responseHeaders(source: Headers, input: { call: string; hold?: string }) { + export function responseHeaders(source: Headers, input: { call: string; hold?: string; providerRequest?: string }) { const result = new Headers(source) result.set(CALL_HEADER, input.call) if (input.hold) result.set(HOLD_HEADER, input.hold) else result.delete(HOLD_HEADER) + if (input.providerRequest) result.set(PROVIDER_REQUEST_HEADER, input.providerRequest) + else result.delete(PROVIDER_REQUEST_HEADER) return result } /** The call identity of one step, from the response headers the AI SDK * attaches to that step (a lowercase record). */ - export function fromResponse(headers: Record | undefined) { + export function fromResponse(headers: Record | undefined, responseID?: string) { const id = headers?.[CALL_HEADER] if (!id || !CALL.test(id)) return undefined const hold = validHold(headers[HOLD_HEADER]) - return hold ? { id, hold } : { id } + const providerRequest = validProviderRequest(responseID) ?? validProviderRequest(headers[PROVIDER_REQUEST_HEADER]) + return { + id, + ...(hold && { hold }), + ...(providerRequest && { providerRequest }), + } } } diff --git a/backend/cli/src/provider/provider.ts b/backend/cli/src/provider/provider.ts index 5631376a8..f67270ec4 100644 --- a/backend/cli/src/provider/provider.ts +++ b/backend/cli/src/provider/provider.ts @@ -572,6 +572,7 @@ export namespace Provider { const linked = CallLink.responseHeaders(response.headers, { call, hold: options.managed ? CallLink.hold(response.headers) : undefined, + providerRequest: CallLink.providerRequest(response.headers), }) // Response.error()/opaque responses use status 0, which the Response // constructor forbids. They do not expose a consumable network body, so @@ -664,10 +665,6 @@ export namespace Provider { "gpt-5-6-luna", "gpt-5.5", "gpt-5-5", - "gpt-5.4", - "gpt-5-4", - "gpt-5.4-mini", - "gpt-5-4-mini", ]) export function isCodexOAuthModel(modelID: string): boolean { @@ -678,7 +675,7 @@ export namespace Provider { if ( modelID !== "gpt-6-sol" && modelID !== "gpt-6.1-sol" && - !/^gpt-5[.-](?:4|5|6(?:-(?:sol|terra|luna))?)$/.test(modelID) + !/^gpt-5[.-](?:5|6(?:-(?:sol|terra|luna))?)$/.test(modelID) ) return undefined return { diff --git a/backend/cli/src/session/call-audit.ts b/backend/cli/src/session/call-audit.ts new file mode 100644 index 000000000..303bc2fdb --- /dev/null +++ b/backend/cli/src/session/call-audit.ts @@ -0,0 +1,77 @@ +import { Session } from "." +import { SessionTraceStore } from "./trace-store" +import { UsageLogging } from "./usage-logging" + +export namespace CallAudit { + export async function report(rootID: string) { + await Session.get(rootID) + const pending = [rootID] + const seen = new Set() + const sessions: { sessionID: string; parentID?: string; telemetrySessionID: string }[] = [] + const calls: { + sessionID: string + telemetrySessionID: string + messageID: string + stepID?: string + route: "managed" | "byok" | "chatgpt" | "subscription" | "local" | "custom" + provider: string + model: string + occurredAt: string + httpCall?: string + providerRequest?: string + hold?: string + recordedTokens?: { input: number; output: number; reasoning: number; cache: { read: number; write: number } } + recordedCostUSD?: number + }[] = [] + while (pending.length) { + const sessionID = pending.shift()! + if (seen.has(sessionID)) continue + seen.add(sessionID) + const session = await Session.get(sessionID) + const telemetrySessionID = await UsageLogging.telemetrySessionID(sessionID) + sessions.push({ sessionID, ...(session.parentID && { parentID: session.parentID }), telemetrySessionID }) + for (const child of await Session.children(sessionID)) pending.push(child.id) + const recorded = new Map() + for (const call of (await SessionTraceStore.read(sessionID)).modelCalls) { + recorded.set(call.id, calls.length) + calls.push({ + sessionID, + telemetrySessionID, + messageID: call.messageID, + route: call.route, + provider: call.provider, + model: call.model, + occurredAt: new Date(call.occurredAt).toISOString(), + httpCall: call.id, + ...(call.providerRequest && { providerRequest: call.providerRequest }), + ...(call.hold && { hold: call.hold }), + }) + } + for (const message of await Session.messages({ sessionID })) { + for (const part of message.parts) { + if (part.type !== "step-finish" || !part.usage) continue + const call = { + sessionID, + telemetrySessionID, + messageID: part.messageID, + stepID: part.id, + route: part.usage.route, + provider: part.usage.provider, + model: part.usage.model, + occurredAt: new Date(part.usage.time).toISOString(), + ...(part.usage.call && { httpCall: part.usage.call }), + ...(part.usage.providerRequest && { providerRequest: part.usage.providerRequest }), + ...(part.usage.hold && { hold: part.usage.hold }), + recordedTokens: part.tokens, + recordedCostUSD: part.cost, + } + const index = part.usage.call ? recorded.get(part.usage.call) : undefined + if (index === undefined) calls.push(call) + else calls[index] = { ...calls[index], ...call } + } + } + } + calls.sort((a, b) => a.occurredAt.localeCompare(b.occurredAt) || (a.stepID ?? "").localeCompare(b.stepID ?? "")) + return { rootSessionID: rootID, sessions, calls } + } +} diff --git a/backend/cli/src/session/llm.ts b/backend/cli/src/session/llm.ts index ac0a5ed66..2f7cee43d 100644 --- a/backend/cli/src/session/llm.ts +++ b/backend/cli/src/session/llm.ts @@ -295,11 +295,12 @@ export namespace LLM { ] const logging = await UsageLogging.context().catch(() => undefined) const started = performance.now() + const traceMessageID = input.trace?.messageID ?? input.user.id const binding = logging ? { context: logging, sessionID: input.sessionID, - messageID: input.trace?.messageID ?? input.user.id, + messageID: traceMessageID, route: traceRoute, provider: routed.providerID, model: routed.api.id, @@ -324,6 +325,27 @@ export namespace LLM { await capture("assistant.message", { ...partial, interrupted: true }) }, async onStepFinish(step) { + const call = CallLink.fromResponse(step.response?.headers, step.response?.id) + if (call) { + await SessionTraceStore.recordModelCall({ + id: call.id, + sessionID: input.sessionID, + messageID: traceMessageID, + route: + traceRoute === "managed" || + traceRoute === "byok" || + traceRoute === "chatgpt" || + traceRoute === "subscription" || + traceRoute === "local" + ? traceRoute + : "custom", + provider: routed.providerID, + model: routed.api.id, + occurredAt: Date.now(), + ...(call.providerRequest && { providerRequest: call.providerRequest }), + ...(call.hold && { hold: call.hold }), + }) + } if (!binding) return await UsageLogging.record({ ...binding, @@ -332,7 +354,7 @@ export namespace LLM { duration: performance.now() - started, content: { parts: step.content, toolResults: step.toolResults }, finish: step.finishReason, - call: CallLink.fromResponse(step.response?.headers), + call, }).catch(() => l.warn("could not persist usage record")) }, async onError(error) { diff --git a/backend/cli/src/session/message-v2.ts b/backend/cli/src/session/message-v2.ts index 41ea03860..26b5409eb 100644 --- a/backend/cli/src/session/message-v2.ts +++ b/backend/cli/src/session/message-v2.ts @@ -291,6 +291,8 @@ export namespace MessageV2 { call: z.string().optional(), /** Atlas hold that reserved this step's Ace charge (managed route only). */ hold: z.string().optional(), + /** Provider-owned response or request identity, when the route exposes one. */ + providerRequest: z.string().optional(), }) .optional(), /** Delivered speed when the provider reports it; otherwise the selected speed. */ diff --git a/backend/cli/src/session/processor.ts b/backend/cli/src/session/processor.ts index b19eabc95..d7252ba74 100644 --- a/backend/cli/src/session/processor.ts +++ b/backend/cli/src/session/processor.ts @@ -1234,7 +1234,7 @@ export namespace SessionProcessor { // The call and hold travel on this step's own response: in // a multi-step tool stream the SDK may already be fetching // the next step, so shared request state would name it. - const stepCall = CallLink.fromResponse(value.response?.headers) + const stepCall = CallLink.fromResponse(value.response?.headers, value.response?.id) const funded = requiresWalletBalance(credentialSource) const usage = Session.getUsage({ model: resolved.model, @@ -1270,6 +1270,7 @@ export namespace SessionProcessor { ...(stepCall && { call: stepCall.id, ...(stepCall.hold && { hold: stepCall.hold }), + ...(stepCall.providerRequest && { providerRequest: stepCall.providerRequest }), }), }, tier: usage.tier, diff --git a/backend/cli/src/session/trace-store.ts b/backend/cli/src/session/trace-store.ts index 53141667c..6bf4c1e69 100644 --- a/backend/cli/src/session/trace-store.ts +++ b/backend/cli/src/session/trace-store.ts @@ -38,14 +38,28 @@ export namespace SessionTraceStore { export const Harness = SessionHarness.Entry export type Harness = z.infer + export const ModelCall = z.object({ + id: z.string(), + sessionID: z.string(), + messageID: z.string(), + route: z.enum(["managed", "byok", "chatgpt", "subscription", "local", "custom"]), + provider: z.string(), + model: z.string(), + occurredAt: z.number(), + providerRequest: z.string().optional(), + hold: z.string().optional(), + }) + export type ModelCall = z.infer + const State = z.object({ approvals: z.record(z.string(), Approval).default({}), retries: z.array(Retry).default([]), harness: z.array(Harness).default([]), + modelCalls: z.array(ModelCall).default([]), }) export type State = z.infer - const empty = (): State => ({ approvals: {}, retries: [], harness: [] }) + const empty = (): State => ({ approvals: {}, retries: [], harness: [], modelCalls: [] }) const file = (sessionID: string) => path.join(Global.Path.data, "trace", `${encodeURIComponent(sessionID)}.json`) async function update(sessionID: string, fn: (state: State) => State) { @@ -133,6 +147,16 @@ export namespace SessionTraceStore { return update(input.sessionID, (state) => ({ ...state, harness: [...state.harness, item] })) } + /** Durable, content-free identity for every completed provider call, + * including internal title/summary calls that do not create message parts. */ + export function recordModelCall(input: ModelCall) { + const item = ModelCall.parse(input) + return update(input.sessionID, (state) => ({ + ...state, + modelCalls: [...state.modelCalls.filter((call) => call.id !== item.id), item].slice(-2_048), + })) + } + export async function remove(sessionID: string) { await fs.unlink(file(sessionID)).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return diff --git a/backend/cli/src/session/usage-logging.ts b/backend/cli/src/session/usage-logging.ts index 78a4bfe92..f06745c5a 100644 --- a/backend/cli/src/session/usage-logging.ts +++ b/backend/cli/src/session/usage-logging.ts @@ -237,6 +237,17 @@ export namespace UsageLogging { } } + /** Opaque session identity used by the server. It lets a local audit name + * its consented trace without revealing the user's local session id. */ + export async function telemetrySessionID(sessionID: string) { + // Persist the installation before deriving an identity. On a fresh data + // root, parsing defaults alone would mint a different installation for + // each call and none would match a later uploaded trace. + if (!(await Bun.file(filepath).exists())) await update(() => {}) + const state = await read() + return digest(`${state.installation}:${sessionID}`) + } + export async function record( input: Binding & { usage: LanguageModelUsage @@ -244,13 +255,17 @@ export namespace UsageLogging { duration: number content?: unknown finish?: string - call?: { id: string; hold?: string } + call?: { id: string; hold?: string; providerRequest?: string } }, ) { return event(input, "model.response", { ...reported(input.usage, input.metadata), duration_ms: input.duration, - ...(input.call && { http_call_id: input.call.id, ...(input.call.hold && { hold: input.call.hold }) }), + ...(input.call && { + http_call_id: input.call.id, + ...(input.call.hold && { hold: input.call.hold }), + ...(input.call.providerRequest && { provider_request_id: input.call.providerRequest }), + }), ...(input.content === undefined ? {} : { content: input.content }), ...(input.finish === undefined ? {} : { finish: input.finish }), }) diff --git a/backend/cli/test/provider/call-link.test.ts b/backend/cli/test/provider/call-link.test.ts index 96000779e..7c5156532 100644 --- a/backend/cli/test/provider/call-link.test.ts +++ b/backend/cli/test/provider/call-link.test.ts @@ -72,6 +72,19 @@ describe("call link", () => { expect(result.response.headers.get("x-openscience-hold-id")).toBeNull() }) + test("non-managed responses keep a bounded provider request id without trusting an internal header", async () => { + const result = await send({ + managed: false, + ctx: context(), + response: new Headers({ + "x-oai-request-id": "req_byok_0123456789", + "x-openscience-provider-request-id": "spoofed", + }), + }) + expect(result.link).toMatchObject({ providerRequest: "req_byok_0123456789" }) + expect(result.response.headers.get("x-openscience-provider-request-id")).toBe("req_byok_0123456789") + }) + test("an upstream cannot choose the call id", async () => { const result = await send({ managed: true, @@ -143,11 +156,24 @@ describe("call link", () => { test("fromResponse() reads a step's own response headers", () => { const id = "7f3c1e9a-0000-4000-8000-000000000001" - expect(CallLink.fromResponse({ "x-openscience-call": id, "x-openscience-hold-id": "hold_abc" })).toEqual({ - id, - hold: "hold_abc", - }) + expect( + CallLink.fromResponse( + { + "x-openscience-call": id, + "x-openscience-hold-id": "hold_abc", + "x-openscience-provider-request-id": "req_header", + }, + "resp_body", + ), + ).toEqual({ id, hold: "hold_abc", providerRequest: "resp_body" }) expect(CallLink.fromResponse({ "x-openscience-call": id, "x-openscience-hold-id": "nope" })).toEqual({ id }) + expect( + CallLink.fromResponse({ + "x-openscience-call": id, + "x-openscience-provider-request-id": "Bearer secret", + }), + ).toEqual({ id }) + expect(CallLink.fromResponse({ "x-openscience-call": id }, "x".repeat(201))).toEqual({ id }) expect(CallLink.fromResponse({ "x-openscience-call": "not a uuid" })).toBeUndefined() expect(CallLink.fromResponse({})).toBeUndefined() expect(CallLink.fromResponse(undefined)).toBeUndefined() diff --git a/backend/cli/test/provider/provider.test.ts b/backend/cli/test/provider/provider.test.ts index def5a2f18..26996d814 100644 --- a/backend/cli/test/provider/provider.test.ts +++ b/backend/cli/test/provider/provider.test.ts @@ -165,6 +165,12 @@ test("Codex OAuth includes the exact GPT-6.1 Sol model without inventing adjacen } }) +test("Codex OAuth excludes API-only GPT-5.4 models", () => { + for (const id of ["gpt-5.4", "gpt-5-4", "gpt-5.4-mini", "gpt-5-4-mini"]) { + expect(Provider.isCodexOAuthModel(id)).toBe(false) + } +}) + test("synthesized Codex OAuth models use Codex variants and preserve model-specific context", async () => { const previous = await Auth.get("openai-codex") await using tmp = await tmpdir({ @@ -225,13 +231,10 @@ test("synthesized Codex OAuth models use Codex variants and preserve model-speci expect(sol61.modes?.fast.cost).toBeUndefined() expect(providers.openai?.models[sol61.id].cost.input).toBe(2) - const codex54 = codex.models["gpt-5.4"] - expect(codex54.limit.context).toBe(1_050_000) - expect(Object.keys(codex54.variants ?? {})).toEqual(["low", "medium", "high", "xhigh"]) - expect(Object.keys(codex54.modes ?? {})).toEqual(["fast"]) - const mini = codex.models["gpt-5.4-mini"] - expect(mini.limit.context).toBe(400_000) - expect(mini.modes).toBeUndefined() + expect(codex.models["gpt-5.4"]).toBeUndefined() + expect(codex.models["gpt-5.4-mini"]).toBeUndefined() + expect(providers.openai?.models["gpt-5.4"]).toBeDefined() + expect(providers.openai?.models["gpt-5.4-mini"]).toBeDefined() expect(codex.name).toBe("OpenAI (Codex subscription)") const publicSol = providers.openai?.models["gpt-5.6-sol"] diff --git a/backend/cli/test/provider/transform-reasoning.test.ts b/backend/cli/test/provider/transform-reasoning.test.ts index dbd939e11..faeb8ae58 100644 --- a/backend/cli/test/provider/transform-reasoning.test.ts +++ b/backend/cli/test/provider/transform-reasoning.test.ts @@ -264,7 +264,7 @@ describe("new model reasoning effort contracts", () => { expect( ProviderTransform.options({ model: codex("gpt-5.6-terra"), sessionID, providerOptions: {} }).reasoningEffort, ).toBe("medium") - expect(ProviderTransform.options({ model: codex("gpt-5.4"), sessionID, providerOptions: {} }).reasoningEffort).toBe( + expect(ProviderTransform.options({ model: codex("gpt-5.5"), sessionID, providerOptions: {} }).reasoningEffort).toBe( "medium", ) }) @@ -289,7 +289,7 @@ describe("new model reasoning effort contracts", () => { }) test("dash-normalized versioned GPT-5 Codex ids use a supported small-call effort", () => { - for (const id of ["gpt-5-6-sol", "gpt-5-6-terra", "gpt-5-6-luna", "gpt-5-5", "gpt-5-4", "gpt-5-4-mini"]) { + for (const id of ["gpt-5-6-sol", "gpt-5-6-terra", "gpt-5-6-luna", "gpt-5-5"]) { const codex = model({ id, providerID: "openai-codex", diff --git a/backend/cli/test/session/call-audit.test.ts b/backend/cli/test/session/call-audit.test.ts new file mode 100644 index 000000000..517e26350 --- /dev/null +++ b/backend/cli/test/session/call-audit.test.ts @@ -0,0 +1,91 @@ +import { expect, test } from "bun:test" +import { CallAudit } from "../../src/session/call-audit" +import { Instance } from "../../src/project/instance" +import { Session } from "../../src/session" +import { UsageLogging } from "../../src/session/usage-logging" +import { SessionTraceStore } from "../../src/session/trace-store" +import { tmpdir } from "../fixture/fixture" + +async function step(sessionID: string, messageID: string, input: { route: "byok" | "chatgpt"; call: string }) { + await Session.updateMessage({ + id: messageID, + sessionID, + role: "assistant", + parentID: "msg_user", + mode: "research", + agent: "research", + path: { cwd: Instance.directory, root: Instance.worktree }, + cost: 0, + tokens: { input: 10, output: 2, reasoning: 1, cache: { read: 3, write: 0 } }, + modelID: "fixture-model", + providerID: input.route === "chatgpt" ? "openai-codex" : "openai", + time: { created: 1, completed: 2 }, + }) + await Session.updatePart({ + id: `prt_${input.call}`, + sessionID, + messageID, + type: "step-finish", + reason: "stop", + cost: 0, + tokens: { input: 10, output: 2, reasoning: 1, cache: { read: 3, write: 0 } }, + usage: { + route: input.route, + provider: input.route === "chatgpt" ? "openai-codex" : "openai", + model: "fixture-model", + time: 1_750_000_000_000, + call: input.call, + providerRequest: `resp_${input.call}`, + }, + }) +} + +test("a session audit includes BYOK and ChatGPT calls from child sessions without content", async () => { + await using tmp = await tmpdir() + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const lead = await Session.create({ title: "Private research title" }) + const worker = await Session.create({ parentID: lead.id, title: "Private worker title" }) + await SessionTraceStore.recordModelCall({ + id: "call-summary", + sessionID: lead.id, + messageID: "summary:msg_byok", + route: "chatgpt", + provider: "openai-codex", + model: "fixture-model", + occurredAt: 1_749_999_999_000, + providerRequest: "resp_call-summary", + }) + await SessionTraceStore.recordModelCall({ + id: "call-byok", + sessionID: lead.id, + messageID: "msg_byok", + route: "byok", + provider: "openai", + model: "fixture-model", + occurredAt: 1_750_000_000_000, + providerRequest: "resp_call-byok", + }) + await step(lead.id, "msg_byok", { route: "byok", call: "call-byok" }) + await step(worker.id, "msg_chatgpt", { route: "chatgpt", call: "call-chatgpt" }) + + const report = await CallAudit.report(lead.id) + + expect(report.sessions).toHaveLength(2) + expect(report.sessions.every((session) => /^[0-9a-f]{64}$/.test(session.telemetrySessionID))).toBe(true) + expect(report.sessions[0].telemetrySessionID).toBe(await UsageLogging.telemetrySessionID(lead.id)) + expect(report.sessions[1].telemetrySessionID).toBe(await UsageLogging.telemetrySessionID(worker.id)) + expect(report.calls.map((call) => call.route)).toEqual(["chatgpt", "byok", "chatgpt"]) + expect(report.calls.map((call) => call.providerRequest)).toEqual([ + "resp_call-summary", + "resp_call-byok", + "resp_call-chatgpt", + ]) + expect(report.calls.find((call) => call.httpCall === "call-summary")?.stepID).toBeUndefined() + expect(report.calls.filter((call) => call.httpCall === "call-byok")).toHaveLength(1) + expect(JSON.stringify(report)).not.toContain("Private research title") + expect(JSON.stringify(report)).not.toContain("Private worker title") + }, + }) +}) diff --git a/backend/cli/test/session/call-link-step.test.ts b/backend/cli/test/session/call-link-step.test.ts index cd94d3fb0..fd5c960f5 100644 --- a/backend/cli/test/session/call-link-step.test.ts +++ b/backend/cli/test/session/call-link-step.test.ts @@ -5,9 +5,9 @@ import z from "zod" import { CallLink } from "../../src/provider/call-link" import { Provider } from "../../src/provider/provider" -function chunk(delta: Record, finish: string | null) { +function chunk(id: string, delta: Record, finish: string | null) { return `data: ${JSON.stringify({ - id: "chatcmpl-link", + id, object: "chat.completion.chunk", created: 1, model: "openai/test", @@ -16,17 +16,19 @@ function chunk(delta: Record, finish: string | null) { })}\n\n` } -const toolStep = () => +const toolStep = (id: string) => chunk( + id, { role: "assistant", tool_calls: [{ index: 0, id: "call_tool_1", type: "function", function: { name: "lookup", arguments: "{}" } }], }, null, ) + - chunk({}, "tool_calls") + + chunk(id, {}, "tool_calls") + "data: [DONE]\n\n" -const textStep = () => chunk({ role: "assistant", content: "done" }, null) + chunk({}, "stop") + "data: [DONE]\n\n" +const textStep = (id: string) => + chunk(id, { role: "assistant", content: "done" }, null) + chunk(id, {}, "stop") + "data: [DONE]\n\n" async function until(check: () => boolean, timeoutMs: number) { const deadline = Date.now() + timeoutMs @@ -44,8 +46,9 @@ test("each step reads the call and hold of its own response, even when the next port: 0, fetch(request) { const hold = `hold_step${sent.length + 1}` + const providerRequest = `chatcmpl-step${sent.length + 1}` sent.push({ call: request.headers.get("x-openscience-call")!, hold }) - return new Response(sent.length === 1 ? toolStep() : textStep(), { + return new Response(sent.length === 1 ? toolStep(providerRequest) : textStep(providerRequest), { headers: { "content-type": "text/event-stream", "x-openscience-hold-id": hold }, }) }, @@ -60,7 +63,7 @@ test("each step reads the call and hold of its own response, even when the next managed: true, })) as typeof fetch, }) - const onStep: ({ id: string; hold?: string } | undefined)[] = [] + const onStep: ({ id: string; hold?: string; providerRequest?: string } | undefined)[] = [] const context: Provider.RequestContext = { sessionID: "ses_step", messageID: "msg_step", attempt: 1 } const finished = await Provider.withRequestContext(context, async () => { const result = streamText({ @@ -71,10 +74,10 @@ test("each step reads the call and hold of its own response, even when the next maxRetries: 0, abortSignal: AbortSignal.timeout(5_000), onStepFinish(step) { - onStep.push(CallLink.fromResponse(step.response?.headers)) + onStep.push(CallLink.fromResponse(step.response?.headers, step.response?.id)) }, }) - const steps: ({ id: string; hold?: string } | undefined)[] = [] + const steps: ({ id: string; hold?: string; providerRequest?: string } | undefined)[] = [] for await (const part of Provider.withRequestContextIterable(context, result.fullStream)) { if (part.type !== "finish-step") continue // Lag like a busy consumer: let the SDK send the next step first. @@ -83,14 +86,18 @@ test("each step reads the call and hold of its own response, even when the next // The next step was already sent while this step was unread. expect(sent).toHaveLength(2) } - steps.push(CallLink.fromResponse(part.response?.headers)) + steps.push(CallLink.fromResponse(part.response?.headers, part.response?.id)) } return steps }) expect(sent).toHaveLength(2) expect(sent[0]!.call).not.toBe(sent[1]!.call) - const expected = sent.map((item) => ({ id: item.call, hold: item.hold })) + const expected = sent.map((item, index) => ({ + id: item.call, + hold: item.hold, + providerRequest: `chatcmpl-step${index + 1}`, + })) expect(finished).toEqual(expected) expect(onStep).toEqual(expected) }) diff --git a/backend/cli/test/session/llm.test.ts b/backend/cli/test/session/llm.test.ts index 733060b3f..1d7a35f0b 100644 --- a/backend/cli/test/session/llm.test.ts +++ b/backend/cli/test/session/llm.test.ts @@ -224,7 +224,7 @@ describe("session.llm.responseStructure", () => { fn: async () => { const research = await Agent.get("research") if (!research) throw new Error("missing Research agent") - const model = await Provider.getModel("openai-codex", "gpt-5.4") + const model = await Provider.getModel("openai-codex", "gpt-5.6-sol") const expected = SystemPrompt.header(model) const provider = await Provider.getProvider(model.providerID) // Fail before starting a stream if the fixture transport was not loaded. diff --git a/backend/cli/test/session/trace.test.ts b/backend/cli/test/session/trace.test.ts index 4f8d1944a..bd59936ab 100644 --- a/backend/cli/test/session/trace.test.ts +++ b/backend/cli/test/session/trace.test.ts @@ -446,7 +446,12 @@ test("builds one local observable harness trace without reasoning or copied outp expect(SessionTrace.Info.parse(trace)).toEqual(trace) await Session.remove(session.id) - expect(await SessionTraceStore.read(session.id)).toEqual({ approvals: {}, retries: [], harness: [] }) + expect(await SessionTraceStore.read(session.id)).toEqual({ + approvals: {}, + retries: [], + harness: [], + modelCalls: [], + }) expect(await SessionResearch.read(session.id)).toBeUndefined() }, }) diff --git a/backend/cli/test/session/usage-logging.test.ts b/backend/cli/test/session/usage-logging.test.ts index 12d978e97..637229308 100644 --- a/backend/cli/test/session/usage-logging.test.ts +++ b/backend/cli/test/session/usage-logging.test.ts @@ -13,6 +13,7 @@ import { Identifier } from "../../src/id/id" import { tmpdir, trustProject } from "../fixture/fixture" import { stressProviderConfig } from "../fixture/stress-provider" import { tracePayload } from "../../src/session/trace-payload" +import { SessionTraceStore } from "../../src/session/trace-store" const store = path.join(Global.Path.data, "usage-logging.json") const session = path.join(Global.Path.data, "openscience-session.json") @@ -421,8 +422,12 @@ describe("usage delivery", () => { const events = remote.batches.flatMap((batch) => batch.events) const request = events.find((event) => event.event_type === "model.request")! const response = events.find((event) => event.event_type === "model.response")! + const httpCallID = response.payload.http_call_id + if (typeof httpCallID !== "string") throw new Error("model response is missing its HTTP call id") expect(events).toHaveLength(2) expect(response.payload).toMatchObject({ + http_call_id: expect.stringMatching(/^[0-9a-f-]{36}$/), + provider_request_id: "chatcmpl-usage", usage: { input_tokens: 917, output_tokens: 83, @@ -437,6 +442,17 @@ describe("usage delivery", () => { expect(JSON.stringify(request.payload)).toContain("PRIVATE_PROMPT") expect(JSON.stringify(response.payload)).toContain("PRIVATE_COMPLETION") expect(response.parent_span_id).toBe(request.span_id) + const calls = (await SessionTraceStore.read(session.id)).modelCalls + expect(calls).toEqual([ + expect.objectContaining({ + sessionID: session.id, + route: "local", + provider: "stress", + model: "fixture-model", + providerRequest: "chatcmpl-usage", + }), + ]) + expect(calls[0].id).toBe(httpCallID) await Instance.dispose() }, }) @@ -747,7 +763,11 @@ describe("usage delivery", () => { model: "anthropic/claude-sonnet-4", usage: measured, duration: 10, - call: { id: "7f3c1e9a-0000-4000-8000-000000000001", hold: "orgh_0123456789abcdef" }, + call: { + id: "7f3c1e9a-0000-4000-8000-000000000001", + hold: "orgh_0123456789abcdef", + providerRequest: "gen-provider-012345", + }, }) await UsageLogging.flush() const payload = remote.batches[0].events[0].payload @@ -756,6 +776,7 @@ describe("usage delivery", () => { expect(payload).toMatchObject({ http_call_id: "7f3c1e9a-0000-4000-8000-000000000001", hold: "orgh_0123456789abcdef", + provider_request_id: "gen-provider-012345", }) expect(payload).not.toHaveProperty("call_id") expect(JSON.stringify(remote.batches)).not.toContain("private-conversation") diff --git a/frontend/docs/src/content/openscience/commands.mdx b/frontend/docs/src/content/openscience/commands.mdx index 1c3d85db7..0ea512a22 100644 --- a/frontend/docs/src/content/openscience/commands.mdx +++ b/frontend/docs/src/content/openscience/commands.mdx @@ -113,6 +113,7 @@ See [Sessions](/openscience/sessions) for meanings and [Automation](/openscience | `openscience upgrade [version]` | Update the CLI. On a copy that came with the desktop app, it points at the app's own updater instead. | | `openscience uninstall --dry-run` | Preview removal. | | `openscience uninstall` | Uninstall while keeping work and settings by default. | +| `openscience debug audit ` | Show content-free identities for its visible, internal, and worker model calls. | | `openscience debug paths` | Show resolved application directories. | | `openscience completion` | Generate the completion script for the detected shell. | diff --git a/frontend/docs/src/content/openscience/privacy.mdx b/frontend/docs/src/content/openscience/privacy.mdx index 79a082b18..8ee5447e4 100644 --- a/frontend/docs/src/content/openscience/privacy.mdx +++ b/frontend/docs/src/content/openscience/privacy.mdx @@ -24,6 +24,8 @@ Open **Customize → General → Data & privacy → Share session traces** to tu The **Delivery** row distinguishes queued, acknowledged, and rejected records. Temporary failures can leave records waiting for retry. Large text may be truncated and binary attachments represented by metadata. Known credentials are redacted, but private research can still appear in prompts and tool output. +For support investigations, `openscience debug audit ` prints a local, content-free report for that conversation, its internal title and summary calls, and its worker sessions. It includes local and opaque telemetry session IDs, routes, models, timestamps, call IDs, provider request IDs, recorded token counts when a visible step has them, and any managed hold IDs, but not prompts, responses, tool payloads, or session titles. The command does not upload anything or bypass trace-sharing preferences. + The [privacy policy](https://openscience.sh/privacy) describes retention and account deletion controls. Disabling future uploads and deleting already stored account data are different actions. Review the scope of the deletion control before confirming it; telemetry deletion does not remove your local project, revoke external credentials, or reverse billing records. ## Control file and tool access diff --git a/frontend/docs/src/content/openscience/usage.mdx b/frontend/docs/src/content/openscience/usage.mdx index d0a0ec0b1..8bb69ff3a 100644 --- a/frontend/docs/src/content/openscience/usage.mdx +++ b/frontend/docs/src/content/openscience/usage.mdx @@ -48,6 +48,6 @@ Workspace usage respects member permissions. Owners and members with usage-view Use **Wallet** for settled managed charges and your provider's billing for direct usage. Card payments add purchased funds and can include a separately disclosed processing fee; they are not the same thing as model spend. Temporary reservations, unsettled requests, different date boundaries, and different account scopes can also explain a mismatch. -For a single conversation, open its context indicator or run `openscience stats` for local session summaries. These views describe saved research activity and may have a different scope from account reports. If a discrepancy remains, retain the date range, funding workspace, model, request identifier, and relevant CSV rows for support. +For a single conversation, open its context indicator or run `openscience stats` for local session summaries. `openscience debug audit ` adds content-free call identities for the conversation and its workers, including provider request IDs when the provider reports them. These views describe saved research activity and may have a different scope from account reports. If a discrepancy remains, retain the date range, funding workspace, model, request identifier, and relevant CSV rows for support. See [Pricing and usage](/openscience/pricing) for rates, reservations, and spending controls, and [Privacy and data](/openscience/privacy) for trace sharing. diff --git a/tooling/sdk/js/src/v2/gen/types.gen.ts b/tooling/sdk/js/src/v2/gen/types.gen.ts index fcb0d6007..6d182b6d5 100644 --- a/tooling/sdk/js/src/v2/gen/types.gen.ts +++ b/tooling/sdk/js/src/v2/gen/types.gen.ts @@ -566,6 +566,7 @@ export type StepFinishPart = { time: number call?: string hold?: string + providerRequest?: string } tier?: string reason: string diff --git a/tooling/sdk/openapi.json b/tooling/sdk/openapi.json index 1a65f9604..28ebee668 100644 --- a/tooling/sdk/openapi.json +++ b/tooling/sdk/openapi.json @@ -57958,6 +57958,9 @@ }, "hold": { "type": "string" + }, + "providerRequest": { + "type": "string" } }, "required": [