diff --git a/packages/ai/src/integration_v7.ts b/packages/ai/src/integration_v7.ts index b721910..c7f0f1f 100644 --- a/packages/ai/src/integration_v7.ts +++ b/packages/ai/src/integration_v7.ts @@ -404,7 +404,7 @@ interface CallState { * onEnd/onError/onAbort). The emitted span/metric shape intentionally matches the v6 path so * ingest and the UI see a single format. */ -export function createV7Hooks( +function createV7Hooks( config: ResolvedConfig, emitter: GenerationEmitter, ): TelemetryDevIntegration { diff --git a/packages/ai/tests/integration.test.ts b/packages/ai/tests/integration.test.ts index 39b2691..e88a3e4 100644 --- a/packages/ai/tests/integration.test.ts +++ b/packages/ai/tests/integration.test.ts @@ -10,7 +10,7 @@ import { TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; import { sessionSpanContext } from "@telemetry-dev/otel"; -import { expect, onTestFinished, test, vi } from "vitest"; +import { expect, test, vi } from "vitest"; import { telemetryDev } from "../src/v6.ts"; @@ -278,11 +278,13 @@ test("a failed tool call records ERROR status and an exception event", async () const callA = { toolCallId: "a", toolName: "t1", input: { n: 1 } }; const callB = { toolCallId: "b", toolName: "t2", input: { n: 2 } }; + const callC = { toolCallId: "c", toolName: "t3", input: { n: 3 } }; integ.onStart?.({ model, prompt: "use tools" }); integ.onStepStart?.({ stepNumber: 0, model }); integ.onToolCallStart?.({ stepNumber: 0, toolCall: callA }); integ.onToolCallStart?.({ stepNumber: 0, toolCall: callB }); + integ.onToolCallStart?.({ stepNumber: 0, toolCall: callC }); integ.onToolCallFinish?.({ stepNumber: 0, toolCall: callA, @@ -297,6 +299,13 @@ test("a failed tool call records ERROR status and an exception event", async () success: false, error: new Error("kaboom"), }); + integ.onToolCallFinish?.({ + stepNumber: 0, + toolCall: callC, + durationMs: 5, + success: false, + error: { code: "tool_failed", secret: "private" }, + }); integ.onStepFinish?.({ stepNumber: 0, model, @@ -310,9 +319,9 @@ test("a failed tool call records ERROR status and an exception event", async () const spans = spanBatches[0]!; const tools = byOperation(spans, "execute_tool"); const step = spans.find((s) => s.kind === SpanKind.CLIENT)!; - expect(tools).toHaveLength(2); + expect(tools).toHaveLength(3); - // Both tools parent to their step. + // Every tool parents to its step. for (const t of tools) { expect(t.parentSpanContext?.spanId).toBe(spanId(step)); } @@ -324,6 +333,15 @@ test("a failed tool call records ERROR status and an exception event", async () const exc = event(failed, "exception"); expect(exc?.attributes?.["exception.message"]).toBe("kaboom"); expect(exc?.attributes?.["log.severity_number"]).toBe(17); + + // A structured non-Error payload is never exported, only its string form. + const structured = tools.find((t) => t.attributes["gen_ai.tool.name"] === "t3")!; + expect(structured.status.code).toBe(SpanStatusCode.ERROR); + expect(structured.attributes["error.type"]).toBe("tool_error"); + expect(event(structured, "exception")?.attributes?.["exception.message"]).toBe("[object Object]"); + expect( + JSON.stringify(spans.map((s) => ({ attributes: s.attributes, events: s.events }))), + ).not.toContain("private"); }); test("the mask rewrites every exported payload", async () => { @@ -453,33 +471,6 @@ test("gateway provider is relabeled to Vercel AI Gateway on every span", async ( expect(step.attributes["gen_ai.request.model"]).toBe("openai/gpt-4o"); }); -test("non-gateway providers pass through unchanged", async () => { - const { spanBatches, overrides } = makeCapture(); - - const integ = telemetryDev( - { apiKey: "td_live_test", environment: "test", serviceName: "svc" }, - overrides, - ); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "hello", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "hello", finishReason: "stop" }); - - const spans = spanBatches[0]!; - const root = spans.find((s) => s.kind === SpanKind.INTERNAL)!; - const step = spans.find((s) => s.kind === SpanKind.CLIENT)!; - expect(root.attributes["gen_ai.provider.name"]).toBe("openai"); - expect(step.attributes["gen_ai.provider.name"]).toBe("openai"); -}); - test("model warnings become model.warning events at severity 13", async () => { const { spanBatches, overrides } = makeCapture(); @@ -512,26 +503,36 @@ test("model warnings become model.warning events at severity 13", async () => { test("no apiKey is a complete no-op (transport never invoked)", async () => { // The integration falls back to this variable, which a developer's shell can export. vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); - onTestFinished(() => { - vi.unstubAllEnvs(); - }); - const { spanBatches, metrics, overrides } = makeCapture(); - const integ = telemetryDev({ environment: "test", serviceName: "svc" }, overrides); - integ.onStart?.({ model, prompt: "x" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "y", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: {}, - }); - await integ.onFinish?.({ text: "y", finishReason: "stop" }); + try { + let calls = 0; + + const integ = telemetryDev({ + environment: "test", + serviceName: "svc", + fetch: async () => { + calls += 1; - expect(spanBatches).toHaveLength(0); - expect(metrics).toHaveLength(0); + return new Response(null, { status: 200 }); + }, + }); + + integ.onStart?.({ model, prompt: "x" }); + integ.onStepStart?.({ stepNumber: 0, model }); + integ.onStepFinish?.({ + stepNumber: 0, + model, + text: "y", + finishReason: "stop", + response: { id: "r", modelId: "gpt-4o" }, + usage: {}, + }); + await integ.onFinish?.({ text: "y", finishReason: "stop" }); + + expect(calls).toBe(0); + } finally { + vi.unstubAllEnvs(); + } }); test("a step with missing usage records no token metric and no summary usage attrs", async () => { @@ -612,8 +613,9 @@ test("export failure reaches onError even on the waitUntil path", async () => { expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe("trace export boom"); }); -test("spans use each emitter's own transport (no cross-talk on a shared cached core)", async () => { +test("spans and metrics use each emitter's own transport (no cross-talk on a shared cached core)", async () => { const traceCalls: string[] = []; + const metricCalls: string[] = []; const aErrors: unknown[] = []; const bErrors: unknown[] = []; @@ -622,6 +624,8 @@ test("spans use each emitter's own transport (no cross-talk on a shared cached c async (url: string | Request | URL, _init?: RequestInit) => { if (requestUrl(url).endsWith("/v1/traces")) traceCalls.push(tag); + if (requestUrl(url).endsWith("/v1/metrics")) metricCalls.push(tag); + return new Response(null, { status: 200 }); }; @@ -643,8 +647,8 @@ test("spans use each emitter's own transport (no cross-talk on a shared cached c onError: (e) => bErrors.push(e), }); - // Run a generation on the FIRST (older) emitter after the second was created: its spans - // must still go through ITS OWN fetch (A), proving span transport is per-call, not shared. + // Run a generation on the FIRST (older) emitter after the second was created: its spans and + // metrics must still go through ITS OWN fetch (A), proving transport is per-call, not shared. integA.onStart?.({ model, prompt: "hi" }); integA.onStepStart?.({ stepNumber: 0, model }); integA.onStepFinish?.({ @@ -659,95 +663,43 @@ test("spans use each emitter's own transport (no cross-talk on a shared cached c expect(traceCalls.length).toBeGreaterThan(0); expect(traceCalls.every((c) => c === "A")).toBe(true); + expect(metricCalls.length).toBeGreaterThan(0); + expect(metricCalls.every((c) => c === "A")).toBe(true); expect(aErrors).toHaveLength(0); expect(bErrors).toHaveLength(0); }); -test("trace export retries a transient ingest failure", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_trace_retry", - environment: "test", - serviceName: "trace-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; +const networkError = new Error("socket closed"); - return new Response(null, { status: traceCalls === 1 ? 503 : 200 }); - } - - return new Response(null, { status: 200 }); +test.each([ + { + name: "network failures", + respond: (): Response => { + throw networkError; }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(2); - expect(errors).toHaveLength(0); -}); - -test("trace export retries a transient network error", async () => { + check: (error: unknown) => expect(error).toBe(networkError), + }, + { + name: "retryable 503 responses", + respond: () => new Response(null, { status: 503 }), + check: (error: unknown) => + expect(error instanceof Error ? error.message : undefined).toBe( + "telemetry.dev trace ingest failed: 503", + ), + }, +])("trace export reports one error after exhausting $name", async ({ respond, check }) => { const errors: unknown[] = []; let traceCalls = 0; const integ = telemetryDev({ - apiKey: "td_live_trace_network_retry", + apiKey: "td_live_trace_exhausted", environment: "test", - serviceName: "trace-network-retry", + serviceName: "trace-exhausted", fetch: async (url: string | Request | URL, _init?: RequestInit) => { if (requestUrl(url).endsWith("/v1/traces")) { traceCalls += 1; - if (traceCalls === 1) throw new Error("socket closed"); - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(2); - expect(errors).toHaveLength(0); -}); - -test("trace export reports one error after exhausting network failures", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - const networkError = new Error("socket closed"); - - const integ = telemetryDev({ - apiKey: "td_live_trace_network_exhausted", - environment: "test", - serviceName: "trace-network-exhausted", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; - throw networkError; + return respond(); } return new Response(null, { status: 200 }); @@ -769,85 +721,7 @@ test("trace export reports one error after exhausting network failures", async ( expect(traceCalls).toBe(3); expect(errors).toHaveLength(1); - expect(errors[0]).toBe(networkError); -}); - -test("trace export does not retry non-retryable ingest failures", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_trace_no_retry", - environment: "test", - serviceName: "trace-no-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; - - return new Response(null, { status: 400 }); - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(1); - expect(errors).toHaveLength(1); - expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe( - "telemetry.dev trace ingest failed: 400", - ); -}); - -test("trace export reports one error after exhausting retryable failures", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_trace_retry_exhausted", - environment: "test", - serviceName: "trace-retry-exhausted", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; - - return new Response(null, { status: 503 }); - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(3); - expect(errors).toHaveLength(1); - expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe( - "telemetry.dev trace ingest failed: 503", - ); + check(errors[0]); }); test("step metrics attribute tokens and duration to each step's own model, not the root", async () => { @@ -915,93 +789,6 @@ test("step metrics attribute tokens and duration to each step's own model, not t expect(openaiOutput?.value).toBe(5); }); -test("metrics use each emitter's own transport (no cross-talk on a shared cached core)", async () => { - const metricCalls: string[] = []; - const aErrors: unknown[] = []; - const bErrors: unknown[] = []; - - const makeFetch = - (tag: string): typeof fetch => - async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/metrics")) metricCalls.push(tag); - - return new Response(null, { status: 200 }); - }; - - // First emitter caches a core under identity "dupm" with fetch A (real metrics pipeline). - const integA = telemetryDev({ - apiKey: "td_live_dupm", - environment: "test", - serviceName: "dupm", - fetch: makeFetch("A"), - onError: (e) => aErrors.push(e), - }); - - // Second emitter reuses that cached core but supplies fetch B. - telemetryDev({ - apiKey: "td_live_dupm", - environment: "test", - serviceName: "dupm", - fetch: makeFetch("B"), - onError: (e) => bErrors.push(e), - }); - - // Run a generation on the FIRST (older) emitter after the second was created: its metrics must - // flush through ITS OWN fetch (A). Before the fix the shared core transport was last-writer (B). - integA.onStart?.({ model, prompt: "hi" }); - integA.onStepStart?.({ stepNumber: 0, model }); - integA.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integA.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(metricCalls.length).toBeGreaterThan(0); - expect(metricCalls.every((c) => c === "A")).toBe(true); - expect(aErrors).toHaveLength(0); - expect(bErrors).toHaveLength(0); -}); - -test("metric export retries a transient ingest failure", async () => { - const errors: unknown[] = []; - let metricCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_metric_retry", - environment: "test", - serviceName: "metric-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/metrics")) { - metricCalls += 1; - - return new Response(null, { status: metricCalls === 1 ? 503 : 200 }); - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(metricCalls).toBe(2); - expect(errors).toHaveLength(0); -}); - test("a non-OK /v1/metrics response is reported as a failed export, not a success", async () => { const errors: unknown[] = []; const exportErrors: unknown[] = []; @@ -1037,10 +824,11 @@ test("a non-OK /v1/metrics response is reported as a failed export, not a succes }); await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - // Our own callback sees the 503 once the retries are exhausted. - expect( - errors.some((e) => e instanceof Error && /metric ingest failed: 503/.test(e.message)), - ).toBe(true); + // Our own callback sees the 503 once, after the retries are exhausted. + expect(errors).toHaveLength(1); + expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe( + "telemetry.dev metric ingest failed: 503", + ); expect(metricCalls).toBe(3); // The export is marked FAILED, so OTel rethrows it to the global handler — this is the bit the // fix changes (code 0 → code 1). diff --git a/packages/ai/tests/integration_v7.test.ts b/packages/ai/tests/integration_v7.test.ts index b39b2da..0a4079f 100644 --- a/packages/ai/tests/integration_v7.test.ts +++ b/packages/ai/tests/integration_v7.test.ts @@ -9,14 +9,9 @@ import { TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; import { sessionSpanContext } from "@telemetry-dev/otel"; -import { expect, test } from "vitest"; +import { expect, test, vi } from "vitest"; import { telemetryDev } from "../src/index.ts"; -import { unknownErrorMessage } from "../src/shared.ts"; - -test("unknown errors do not expose structured payloads", () => { - expect(unknownErrorMessage({ code: "tool_failed", secret: "private" })).toBe("[object Object]"); -}); interface MetricRecord { metric: "duration" | "tokens"; @@ -299,40 +294,30 @@ test("missing or empty runtimeContext emits no identity or metadata attributes", } }); -test("onStepFinish is not implemented, so ai@7's step-end fan-out cannot double spans", async () => { +test("ai@7 step-end fan-out records each step once", async () => { + const { generateText } = await import("ai"); + const { MockLanguageModelV4 } = await import("ai/test"); const { spanBatches, metrics, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); - const callId = "call-dedupe"; - const stepResult = { - callId, - stepNumber: 0, - model, - text: "once", - finishReason: "stop", - response: { id: "r1", modelId: "gpt-4o" }, - usage: { inputTokens: 4, outputTokens: 2 }, - performance: { responseTimeMs: 1000 }, - }; - - integ.onStart?.({ - callId, - operationId: "ai.generateText", - provider: "openai", - modelId: "gpt-4o", - messages: [{ role: "user", content: "x" }], - }); - integ.onStepStart?.({ callId, stepNumber: 0 }); - integ.onStepEnd?.(stepResult); - // ai@7 fans step-end out to BOTH onStepEnd and the deprecated onStepFinish, calling each - // implemented hook once. The integration must not declare onStepFinish, or every step would be - // recorded twice. - expect(Object.keys(integ)).not.toContain("onStepFinish"); - await integ.onEnd?.({ - callId, - text: "once", - finishReason: "stop", - usage: stepResult.usage, + // ai@7 fans step-end out to both onStepEnd and the deprecated onStepFinish, so an integration + // that implemented both would record every step twice. + await generateText({ + model: new MockLanguageModelV4({ + provider: "openai", + modelId: "gpt-4o", + doGenerate: { + content: [{ type: "text", text: "once" }], + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 4, noCache: 4, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 2, text: 2, reasoning: undefined }, + }, + warnings: [], + }, + }), + prompt: "x", + telemetry: { integrations: [integ] }, }); expect(spanBatches).toHaveLength(1); @@ -342,6 +327,53 @@ test("onStepFinish is not implemented, so ai@7's step-end fan-out cannot double expect(metrics.filter((m) => m.metric === "duration")).toHaveLength(1); }); +test("no apiKey is a complete no-op (transport never invoked)", async () => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + + try { + let calls = 0; + + const integ = telemetryDev({ + environment: "test", + serviceName: "svc", + fetch: async () => { + calls += 1; + + return new Response(null, { status: 200 }); + }, + }); + + const callId = "no-key"; + + integ.onStart?.({ + callId, + operationId: "ai.generateText", + provider: "openai", + modelId: "gpt-4o", + messages: [{ role: "user", content: "x" }], + }); + integ.onStepStart?.({ callId, stepNumber: 0 }); + integ.onStepEnd?.({ + callId, + stepNumber: 0, + model, + text: "y", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + await integ.onEnd?.({ + callId, + text: "y", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + + expect(calls).toBe(0); + } finally { + vi.unstubAllEnvs(); + } +}); + test("two interleaved callIds on one integration produce disjoint traces per flush", async () => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); @@ -1147,70 +1179,25 @@ test("provider results recover the latest matching input from a later generation ); }); -test("provider results recover inputs from prepareStep message replacements", async () => { - const { spanBatches, overrides } = makeCapture(); - const integ = telemetryDev(baseOptions, overrides); - const callId = "call-provider-prepare-step-history"; - - const providerCall = { - type: "tool-call", - toolCallId: "deferred-tool", - toolName: "web_search", - input: { query: "initial" }, - providerExecuted: true, - }; - - const messages = [{ role: "assistant", content: [providerCall] }]; - - const providerResult = { - type: "tool-result", - toolCallId: "deferred-tool", - toolName: "web_search", - output: { result: "prepared result" }, - providerExecuted: true, - }; - - integ.onStart?.({ - callId, - operationId: "ai.generateText", - provider: "openai", - modelId: "gpt-4o", - messages, - }); - providerCall.input = { query: "prepared" }; - integ.onStepStart?.({ callId, stepNumber: 0, messages }); - integ.onLanguageModelCallEnd?.({ callId, content: [providerResult] }); - integ.onStepEnd?.({ - callId, - stepNumber: 0, - model, - content: [providerResult], - text: "done", - finishReason: "stop", - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onEnd?.({ - callId, - text: "done", - finishReason: "stop", - usage: { inputTokens: 1, outputTokens: 1 }, - }); - - const tools = byOperation(spanBatches[0]!, "execute_tool"); - - expect(tools).toHaveLength(1); - expect(tools[0]!.attributes["gen_ai.tool.call.arguments"]).toBe( - JSON.stringify({ query: "prepared" }), - ); - expect(tools[0]!.attributes["gen_ai.tool.call.result"]).toBe( - JSON.stringify({ result: "prepared result" }), - ); -}); - -test("provider results recover inputs from nested history mutations", async () => { +test.each([ + { + name: "prepareStep message replacements", + mutate: (call: { input: { query: string } }) => { + call.input = { query: "prepared" }; + }, + query: "prepared", + }, + { + name: "nested history mutations", + mutate: (call: { input: { query: string } }) => { + call.input.query = "mutated"; + }, + query: "mutated", + }, +])("provider results recover inputs from $name", async ({ mutate, query }) => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); - const callId = "call-provider-history-nested-mutation"; + const callId = `call-provider-history-${query}`; const providerCall = { type: "tool-call", @@ -1226,7 +1213,7 @@ test("provider results recover inputs from nested history mutations", async () = type: "tool-result", toolCallId: "deferred-tool", toolName: "web_search", - output: { result: "mutated result" }, + output: { result: `${query} result` }, providerExecuted: true, }; @@ -1237,7 +1224,7 @@ test("provider results recover inputs from nested history mutations", async () = modelId: "gpt-4o", messages, }); - providerCall.input.query = "mutated"; + mutate(providerCall); integ.onStepStart?.({ callId, stepNumber: 0, messages }); integ.onLanguageModelCallEnd?.({ callId, content: [providerResult] }); integ.onStepEnd?.({ @@ -1259,11 +1246,9 @@ test("provider results recover inputs from nested history mutations", async () = const tools = byOperation(spanBatches[0]!, "execute_tool"); expect(tools).toHaveLength(1); - expect(tools[0]!.attributes["gen_ai.tool.call.arguments"]).toBe( - JSON.stringify({ query: "mutated" }), - ); + expect(tools[0]!.attributes["gen_ai.tool.call.arguments"]).toBe(JSON.stringify({ query })); expect(tools[0]!.attributes["gen_ai.tool.call.result"]).toBe( - JSON.stringify({ result: "mutated result" }), + JSON.stringify({ result: `${query} result` }), ); }); @@ -3840,6 +3825,13 @@ test.each([ : undefined, ); } + + const exported = JSON.stringify( + spanBatches[0]!.map((span) => ({ attributes: span.attributes, events: span.events })), + ); + + expect(/private-action|private-policy/.test(exported)).toBe(recordInputs); + expect(exported.includes("probability")).toBe(recordOutputs); }, ); @@ -3983,43 +3975,6 @@ test("evaluation hooks emit correlated root and model spans without duplicate to ]); }); -test("evaluation capture flags redact payloads while preserving identity and usage", async () => { - const { spanBatches, overrides } = makeCapture(); - const integ = telemetryDev(baseOptions, overrides); - const callId = "evaluation-redacted"; - - integ.experimental_onEvaluateStart?.({ - callId, - provider: "openai", - modelId: "judge", - state: { secret: "input" }, - questions: { secret: { type: "score", question: "private" } }, - recordInputs: false, - recordOutputs: false, - }); - integ.experimental_onEvaluationModelCallStart?.({ callId }); - integ.experimental_onEvaluationModelCallEnd?.({ - callId, - answers: { secret: { type: "score", score: 1 } }, - usage: { inputTokens: 4, outputTokens: 2 }, - }); - await integ.experimental_onEvaluateEnd?.({ - callId, - provider: "openai", - modelId: "judge", - answers: { secret: { type: "score", score: 1 } }, - usage: { inputTokens: 4, outputTokens: 2 }, - }); - - const spans = spanBatches[0]!; - expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("secret"); - expect(spans.every((span) => span.attributes["gen_ai.input.messages"] === undefined)).toBe(true); - expect(spans.every((span) => span.attributes["gen_ai.output.messages"] === undefined)).toBe(true); - expect( - spans.find((span) => span.kind === SpanKind.CLIENT)?.attributes["gen_ai.usage.input_tokens"], - ).toBe(4); -}); - test("overlapping evaluations stay isolated and errors clean up their call state", async () => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); diff --git a/packages/ai/tests/type_conformance.test.ts b/packages/ai/tests/type_conformance.test.ts index d69a383..a502e33 100644 --- a/packages/ai/tests/type_conformance.test.ts +++ b/packages/ai/tests/type_conformance.test.ts @@ -1,23 +1,14 @@ import type { Telemetry } from "ai"; import type { TelemetryIntegration } from "ai-v6"; -import { expect, test } from "vitest"; +import { expectTypeOf, test } from "vitest"; import { telemetryDev } from "../src/index.ts"; import { telemetryDev as telemetryDevV6 } from "../src/v6.ts"; -// Compile-time contract: the emitted types never import from `ai`, so each entry's integration -// object must stay structurally assignable to its major's interface (v6 `TelemetryIntegration` -// was removed in v7; v7 `Telemetry` does not exist in v6). A break here fails typecheck, not -// runtime. -const integ = telemetryDev({ apiKey: "td_live_t" }); - -void (integ satisfies Telemetry); - -const integV6 = telemetryDevV6({ apiKey: "td_live_t" }); - -void (integV6 satisfies TelemetryIntegration); - -test("each entry's telemetryDev satisfies its ai major's integration type", () => { - expect(integ).toBeDefined(); - expect(integV6).toBeDefined(); +// The emitted types never import from `ai`, so each entry's integration object must stay +// structurally assignable to its major's interface (v6 `TelemetryIntegration` was removed in v7; +// v7 `Telemetry` does not exist in v6). A break fails `vp check`, not the test run. +test("each entry's telemetryDev is assignable to its ai major's integration type (compile-time)", () => { + expectTypeOf(telemetryDev).returns.toExtend(); + expectTypeOf(telemetryDevV6).returns.toExtend(); }); diff --git a/packages/ai/tests/wrappers.test.ts b/packages/ai/tests/wrappers.test.ts index f5cedea..7510ec4 100644 --- a/packages/ai/tests/wrappers.test.ts +++ b/packages/ai/tests/wrappers.test.ts @@ -44,7 +44,8 @@ test("media wrappers preserve structural function types and record metadata with .images[0]!.uint8Array, ).toBe(bytes); expect(spans).toHaveLength(1); - expect(spans[0]!.attributes).toMatchObject({ + // An exact map: an attribute that carries the bytes in any encoding fails here. + expect(spans[0]!.attributes).toEqual({ "gen_ai.operation.name": "generate_content", "gen_ai.output.type": "image", "gen_ai.provider.name": "openai", @@ -54,7 +55,6 @@ test("media wrappers preserve structural function types and record metadata with "gen_ai.usage.image.output_tokens": 2, "td.ai.output.image_count": 1, }); - expect(JSON.stringify(spans[0]!.attributes)).not.toContain("1,2,3"); }); test("media wrappers preserve rejection and synchronous throw identity", async () => { @@ -143,6 +143,11 @@ test("throwing telemetry overrides do not replace provider results or errors", a await Promise.resolve(); expect(durationAttempts).toBe(1); expect(spans).toHaveLength(1); + expect(telemetryErrors.map((error) => (error as Error).message)).toEqual([ + "tokens failed", + "duration failed", + ]); + telemetryErrors.length = 0; const providerError = new Error("provider failed"); expect(() => @@ -154,7 +159,8 @@ test("throwing telemetry overrides do not replace provider results or errors", a overrides, )(), ).toThrow(providerError); - expect(telemetryErrors.length).toBeGreaterThan(0); + expect(durationAttempts).toBe(2); + expect(telemetryErrors.map((error) => (error as Error).message)).toEqual(["duration failed"]); }); test("throwing telemetry setup and one-shot input getters preserve exact provider outcomes", () => { diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index bc1cd00..cc10ede 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -551,40 +551,6 @@ test("messages.create streaming preserves events and records aggregated text usa ).toBe(true); }); -test("stream capture is bounded without changing delivered events", async () => { - const spans = setupSpans(); - - const events: JsonRecord[] = [ - { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, - ...Array.from({ length: 80 }, () => ({ - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "x".repeat(1024) }, - })), - { type: "message_stop" }, - ]; - - const fake = createFakeFetch(namedSseResponse(events)); - - const stream = await clientWith(fake.fetch).messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [], - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); - - const output = jsonAttr }>>( - span, - "gen_ai.output.messages", - ); - - expect(output[0]!.content[0]!.text.length).toBeLessThan(48 * 1024); -}); - test("stream capture retains no output when output capture is disabled", async () => { const spans = setupSpans(undefined, false); const capture = vi.spyOn(telemetrySdk, "boundedCaptureDetails"); @@ -642,44 +608,6 @@ test("wrapAnthropic observes a CommonJS client subclass stream", async () => { ]); }); -test("messages.create streaming observes duck-typed stream responses", async () => { - const spans = setupSpans(); - - class AlternateStream implements AsyncIterable { - private abortController = new AbortController(); - controller = { - abort: (reason?: unknown) => this.abortController.abort(reason), - signal: this.abortController.signal, - }; - - async *[Symbol.asyncIterator](): AsyncIterator { - for (const event of streamEvents()) yield event; - } - } - - const client = wrapAnthropic({ - messages: { - create: (_params: JsonValue) => Promise.resolve(new AlternateStream()), - }, - }); - - const stream = await client.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Say hello" }], - stream: true, - }); - - await collectStream(stream); - - const span = await exportedSpan(spans); - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: [{ type: "text", text: "Hello world" }] }, - ]); - expect(span.attributes["gen_ai.usage.input_tokens"]).toBe(9); - expect(span.attributes["gen_ai.usage.output_tokens"]).toBe(2); -}); - test("message timestamps precede telemetry mapping", async () => { const spans = setupSpans(); let now = 0; @@ -1026,48 +954,6 @@ test("messages.create streaming keeps telemetry on a core span without a capture ]); }); -test("messages.create streaming retains output when the core span lacks chunk recording", async () => { - const spans = setupSpans(); - const startSpan = telemetrySdk.startSpan; - vi.spyOn(telemetrySdk, "startSpan").mockImplementation((...args) => { - const legacySpan = { ...startSpan(...args) }; - Reflect.deleteProperty(legacySpan, "recordOutputChunk"); - - return legacySpan; - }); - - const events = [ - { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "delivered" }, - }, - { type: "message_stop" }, - ]; - - const source = { - controller: new AbortController(), - async *[Symbol.asyncIterator]() { - yield* events; - }, - }; - - const client = wrapAnthropic({ messages: { create: async (_params: unknown) => source } }); - - const stream = await client.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [], - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - expect(jsonAttr(await exportedSpan(spans), "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: [{ type: "text", text: "delivered" }] }, - ]); -}); - test("messages.create streaming leaves the configured attribute limit to the core", async () => { const spans = setupSpans(undefined, true, undefined, { maxAttributeLength: 128 }); @@ -1211,30 +1097,13 @@ test("uninstrumentAnthropic preserves a method installed after its own wrapper", } }); -test("wrapAnthropic keeps a client instrumented after global instrumentation is removed", async () => { - const spans = setupSpans(); - instrumentAnthropic(); - const fake = createFakeFetch(jsonResponse(messagePayload({ id: "msg_instance_wrapped" }))); - const client = wrapAnthropic(new Anthropic({ apiKey: "test", fetch: fake.fetch, maxRetries: 0 })); - uninstrumentAnthropic(); - - await client.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Hi wrapped" }], - }); - - const span = await exportedSpan(spans); - expect(fake.requests).toHaveLength(1); - expect(span.attributes["gen_ai.request.model"]).toBe("claude-sonnet-4-6"); -}); - test("instrumentAnthropic and wrapAnthropic together record one span per call", async () => { const spans = setupSpans(); instrumentAnthropic(); const fake = createFakeFetch( jsonResponse(messagePayload({ id: "msg_global_wrapped" })), + jsonResponse(messagePayload({ id: "msg_beta_both" })), jsonResponse(messagePayload({ id: "msg_instance_wrapped_after_global" })), ); @@ -1245,10 +1114,18 @@ test("instrumentAnthropic and wrapAnthropic together record one span per call", max_tokens: 64, messages: [{ role: "user", content: "Hi wrapped" }], }); + await client.beta.messages.create({ + model: "claude-sonnet-4-6", + max_tokens: 64, + messages: [{ role: "user", content: "Hi beta" }], + }); - const activeSpans = await finishedSpans(spans, 1); - expect(fake.requests).toHaveLength(1); - expect(activeSpans[0]!.attributes["gen_ai.request.model"]).toBe("claude-sonnet-4-6"); + const activeSpans = await finishedSpans(spans, 2); + expect(fake.requests).toHaveLength(2); + expect(activeSpans.map((span) => span.attributes["gen_ai.response.id"])).toEqual([ + "msg_global_wrapped", + "msg_beta_both", + ]); uninstrumentAnthropic(); await client.messages.create({ @@ -1257,9 +1134,9 @@ test("instrumentAnthropic and wrapAnthropic together record one span per call", messages: [{ role: "user", content: "Hi after uninstrument" }], }); - const allSpans = await finishedSpans(spans, 2); - expect(fake.requests).toHaveLength(2); - expect(allSpans[1]!.attributes["gen_ai.request.model"]).toBe("claude-sonnet-4-6"); + const allSpans = await finishedSpans(spans, 3); + expect(fake.requests).toHaveLength(3); + expect(allSpans[2]!.attributes["gen_ai.response.id"]).toBe("msg_instance_wrapped_after_global"); }); const providerClients = { @@ -1298,25 +1175,6 @@ test.each<{ client: keyof typeof providerClients; subclass: boolean; expected: s }, ); -test("wrapAnthropic records Bedrock provider names from client constructor names", async () => { - const spans = setupSpans(); - - class AnthropicBedrock { - messages = { - create: async (_params: JsonValue) => messagePayload({ id: "msg_bedrock" }), - }; - } - - await wrapAnthropic(new AnthropicBedrock()).messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Hi" }], - }); - - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.provider.name"]).toBe("aws.bedrock"); -}); - test("beta.messages.create records a generation span for the beta endpoint", async () => { const spans = setupSpans(); const messages = [{ role: "user" as const, content: "Say hello" }]; @@ -1461,22 +1319,6 @@ test("instrumentAnthropic covers beta.messages and uninstrumentAnthropic restore expect(spans.getFinishedSpans()).toHaveLength(1); }); -test("instrumentAnthropic and wrapAnthropic together record one beta span per call", async () => { - const spans = setupSpans(); - instrumentAnthropic(); - const fake = createFakeFetch(jsonResponse(messagePayload({ id: "msg_beta_both" }))); - const client = wrapAnthropic(new Anthropic({ apiKey: "test", fetch: fake.fetch, maxRetries: 0 })); - - await client.beta.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Hi" }], - }); - - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.id"]).toBe("msg_beta_both"); -}); - function betaStreamEvents(...blocks: [JsonRecord, JsonRecord[]][]): JsonRecord[] { const events: JsonRecord[] = [ { @@ -1543,33 +1385,28 @@ function fallbackStream(iterations: JsonRecord[]): JsonRecord[] { return events; } -test.each([false, true])( - "stream output over the encoded limit keeps a bounded prefix (masked: %s)", - async (masked) => { - const spans = setupSpans(undefined, true, masked ? (value) => value : undefined); - - const events = betaStreamEvents( - [{ type: "text", text: "" }, [{ type: "text_delta", text: "first" }]], - [{ type: "text", text: "" }, [{ type: "text_delta", text: '"'.repeat(30_000) }]], - ); - - const span = await streamedBetaSpan(events, spans); +test("a stream delta over the encoded limit is dropped and earlier output is kept", async () => { + const spans = setupSpans(); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); + const events = betaStreamEvents( + [{ type: "text", text: "" }, [{ type: "text_delta", text: "first" }]], + [{ type: "text", text: "" }, [{ type: "text_delta", text: '"'.repeat(30_000) }]], + ); - if (masked) expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); - else { - const output = jsonAttr }>>( - span, - "gen_ai.output.messages", - ); + const span = await streamedBetaSpan(events, spans); - expect(String(span.attributes["gen_ai.output.messages"])).not.toContain("[truncated]"); - expect(output[0]?.content[0]).toEqual({ type: "text", text: "first" }); - expect('"'.repeat(30_000).startsWith(output[0]?.content[1]?.text ?? "")).toBe(true); - } - }, -); + expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); + expect(String(span.attributes["gen_ai.output.messages"])).not.toContain("[truncated]"); + expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ + { + role: "assistant", + content: [ + { type: "text", text: "first" }, + { type: "text", text: "" }, + ], + }, + ]); +}); test.each([ [400, true], @@ -1595,7 +1432,7 @@ test.each([ }, ); -test("deltas that fill the stream limit keep the text that was accepted", async () => { +test("deltas that fill the stream limit keep the accepted text and deliver every event", async () => { const spans = setupSpans(); const events = betaStreamEvents([ @@ -1606,7 +1443,17 @@ test("deltas that fill the stream limit keep the text that was accepted", async ], ]); - const span = await streamedBetaSpan(events, spans); + const stream = await clientWith( + createFakeFetch(namedSseResponse(events)).fetch, + ).beta.messages.create({ + model: "claude-opus-5", + max_tokens: 64, + messages: [{ role: "user", content: "Hi" }], + stream: true, + }); + + expect(await collectStream(stream)).toEqual(events); + const span = await exportedSpan(spans); const output = jsonAttr }>>( span, @@ -1698,18 +1545,6 @@ const servedIteration = { output_tokens: 2, }; -test("beta streams record the fallback model when a fallback_message iteration served it", async () => { - const span = await streamedBetaSpan(fallbackStream([declinedIteration, servedIteration])); - - expect(span.attributes["gen_ai.response.model"]).toBe("claude-opus-4-8"); -}); - -test("beta streams keep the requested model when the fallback request failed", async () => { - const span = await streamedBetaSpan(fallbackStream([declinedIteration])); - - expect(span.attributes["gen_ai.response.model"]).toBe("claude-opus-5"); -}); - const advisorIteration = { ...declinedIteration, type: "advisor_message", @@ -1798,6 +1633,12 @@ test("fallback metadata overflow preserves complete masked output", async () => }); test.each<{ name: string; iterations: JsonRecord[]; expected: string }>([ + { + name: "fallback served", + iterations: [declinedIteration, servedIteration], + expected: "claude-opus-4-8", + }, + { name: "fallback failed", iterations: [declinedIteration], expected: "claude-opus-5" }, { name: "advisor without fallback", iterations: [declinedIteration, advisorIteration], @@ -1818,7 +1659,7 @@ test.each<{ name: string; iterations: JsonRecord[]; expected: string }>([ expected: "claude-sonnet-5", }, ])( - "beta streams ignore advisor iterations for the served model: $name", + "beta streams take the served model from the last fallback_message iteration: $name", async ({ iterations, expected }) => { const span = await streamedBetaSpan(fallbackStream(iterations)); @@ -2299,6 +2140,7 @@ async function usageSpan( }); const { cache_creation: _split, ...deltaUsage } = finalUsage; + const abortController = new AbortController(); const events = [ { @@ -2310,8 +2152,12 @@ async function usageSpan( { type: "message_stop" }, ]; + // A plain { abort, signal } controller: stream detection must not require an AbortController. const source = { - controller: new AbortController(), + controller: { + abort: (reason?: unknown) => abortController.abort(reason), + signal: abortController.signal, + }, async *[Symbol.asyncIterator]() { yield* events; }, diff --git a/packages/bedrock/src/internal.ts b/packages/bedrock/src/internal.ts index 5a231fc..104f595 100644 --- a/packages/bedrock/src/internal.ts +++ b/packages/bedrock/src/internal.ts @@ -351,7 +351,7 @@ export function parseJson(value: T): JsonValue { } // Retention uses fixed bounds; the core applies maxAttributeLength after the mask. -export const STREAM_CAPTURE_MAX_BYTES = 48 * 1024; +const STREAM_CAPTURE_MAX_BYTES = 48 * 1024; export const STREAM_CAPTURE_MAX_ITEMS = 1_000; const CAPTURE_TRUNCATED_ATTRIBUTE = "telemetry.dev.capture.truncated"; diff --git a/packages/bedrock/tests/agents.test.ts b/packages/bedrock/tests/agents.test.ts index 07bcc4d..de39030 100644 --- a/packages/bedrock/tests/agents.test.ts +++ b/packages/bedrock/tests/agents.test.ts @@ -17,7 +17,16 @@ import { uninstrumentBedrockAgents, wrapBedrockAgents, } from "../src/agents.ts"; -import { FakeClient, bytes, collect, jsonAttr, setup, streamOf, teardown } from "./helpers.ts"; +import { + FakeClient, + bytes, + collect, + jsonAttr, + jsonRequestHandler, + setup, + streamOf, + teardown, +} from "./helpers.ts"; afterEach(async () => { await teardown(); @@ -1021,50 +1030,63 @@ test("modeled stream errors from Agent, RAG, and Flow mark spans as errors", asy expect(jsonAttr(flowSpan, "gen_ai.output.messages")).toEqual([{ document: { value: 1 } }]); }); -test("global agents instrumentation is idempotent", async () => { +test("global agents instrumentation patches the real client once and restores its inherited send", async () => { const spans = setup(); - - const originalDescriptor = Object.getOwnPropertyDescriptor( - BedrockAgentRuntimeClient.prototype, - "send", - ); - - const fakeSend = async () => ({ - completion: streamOf([{ chunk: { bytes: bytes("global") } }]), - $metadata: { requestId: "agent-global-1" }, + const proto = BedrockAgentRuntimeClient.prototype; + const command = new RetrieveCommand({ knowledgeBaseId: "kb-1", retrievalQuery: { text: "q" } }); + + const client = new BedrockAgentRuntimeClient({ + region: "us-east-1", + credentials: { accessKeyId: "test", secretAccessKey: "test" }, + maxAttempts: 1, + requestHandler: jsonRequestHandler("agent-global-1", { + retrievalResults: [{ content: { text: "doc" } }], + }), }); - Object.defineProperty(BedrockAgentRuntimeClient.prototype, "send", { - configurable: true, - writable: true, - value: fakeSend, - }); + // AWS clients inherit send from the Smithy base class. + expect(Object.hasOwn(proto, "send")).toBe(false); try { instrumentBedrockAgents(); instrumentBedrockAgents(); - const client = new BedrockAgentRuntimeClient({ - region: "us-east-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - }); + await client.send(command); + expect(spans.getFinishedSpans()).toHaveLength(1); + expect(spans.getFinishedSpans()[0]!.attributes["gen_ai.response.id"]).toBe("agent-global-1"); - const response = (await client.send( - new InvokeAgentCommand({ agentId: "agent-1", agentAliasId: "alias", sessionId: "sess" }), - )) as { completion: AsyncIterable }; + const telemetryWrapper = Object.getOwnPropertyDescriptor(proto, "send")?.value; + const laterWrapper = async () => ({ retrievalResults: [] }); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: laterWrapper, + }); + uninstrumentBedrockAgents(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(laterWrapper); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: telemetryWrapper, + }); + uninstrumentBedrockAgents(); + expect(Object.hasOwn(proto, "send")).toBe(false); - await collect(response.completion); + await client.send(command); expect(spans.getFinishedSpans()).toHaveLength(1); - expect(spans.getFinishedSpans()[0]!.attributes["gen_ai.response.id"]).toBe("agent-global-1"); + + const priorOwner = async () => ({ retrievalResults: [] }); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: priorOwner, + }); + instrumentBedrockAgents(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).not.toBe(priorOwner); uninstrumentBedrockAgents(); - expect( - Object.getOwnPropertyDescriptor(BedrockAgentRuntimeClient.prototype, "send")?.value, - ).toBe(fakeSend); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(priorOwner); } finally { uninstrumentBedrockAgents(); - - if (originalDescriptor) { - Object.defineProperty(BedrockAgentRuntimeClient.prototype, "send", originalDescriptor); - } + Reflect.deleteProperty(proto, "send"); } }); diff --git a/packages/bedrock/tests/bedrock.test.ts b/packages/bedrock/tests/bedrock.test.ts index 40b217d..8e94b8f 100644 --- a/packages/bedrock/tests/bedrock.test.ts +++ b/packages/bedrock/tests/bedrock.test.ts @@ -12,12 +12,21 @@ import { afterEach, expect, test, vi } from "vitest"; import type { MaskFn } from "@telemetry-dev/sdk"; import { instrumentBedrock, uninstrumentBedrock, wrapBedrock } from "../src/index.ts"; -import { STREAM_CAPTURE_MAX_BYTES } from "../src/internal.ts"; -import { FakeClient, bytes, collect, jsonAttr, setup, streamOf, teardown } from "./helpers.ts"; +import { + FakeClient, + bytes, + collect, + jsonAttr, + jsonRequestHandler, + setup, + streamOf, + teardown, +} from "./helpers.ts"; afterEach(async () => { uninstrumentBedrock(); await teardown(); + vi.restoreAllMocks(); }); test("Converse captures normalized messages, usage, metadata, request id, and sampling", async () => { @@ -267,22 +276,15 @@ test("masked ConverseStream early break withholds partial output", async () => { expect(seen.some((value) => value.includes("partial secret"))).toBe(false); }); -test.each([ - [false, 0], - [false, 100], - [false, 1024 * 1024], - [true, 0], - [true, 100], - [true, 1024 * 1024], -])( - "ConverseStream past the fixed bound is flagged (masked %s, maxAttributeLength %i)", - async (masked, maxAttributeLength) => { +test.each([false, true])( + "ConverseStream past the fixed bound is flagged even when the cap is larger (masked %s)", + async (masked) => { const seen: string[] = []; - const options: NonNullable[0]> = { maxAttributeLength }; - if (masked) options.mask = recordingMask(seen); - - const spans = setup(options); + const spans = setup({ + maxAttributeLength: 1024 * 1024, + ...(masked ? { mask: recordingMask(seen) } : {}), + }); await streamedConverse(textStream("x".repeat(100), 1_100)); @@ -293,15 +295,11 @@ test.each([ if (masked) { expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); expect(seen.some((value) => value.includes("x".repeat(100)))).toBe(false); - } else if (maxAttributeLength > STREAM_CAPTURE_MAX_BYTES) { + } else { const retained = jsonAttr(span, "gen_ai.output.messages")[0].parts[0].content as string; expect(retained.length).toBeGreaterThan(0); - expect(retained.length).toBeLessThan(1_100 * 100); + expect(retained.length).toBeLessThanOrEqual(48 * 1024); expect(retained).toBe("x".repeat(retained.length)); - } else { - expect(String(span.attributes["gen_ai.output.messages"] ?? "").length).toBeLessThanOrEqual( - maxAttributeLength, - ); } }, ); @@ -462,6 +460,17 @@ test("InvokeModel captures provider-native bodies and embedding spans", async () ); const [chat, embedding] = spans.getFinishedSpans(); + expect(jsonAttr(chat, "gen_ai.input.messages")).toEqual({ + messages: [], + max_tokens: 10, + top_k: 3, + }); + expect(jsonAttr(chat, "gen_ai.output.messages")).toEqual({ + content: [{ type: "text", text: "answer" }], + usage: { input_tokens: 9, output_tokens: 4, cache_read_input_tokens: 1 }, + stop_reason: "end_turn", + }); + expect(chat.attributes["gen_ai.usage.input_tokens"]).toBe(9 + 1); expect(chat.attributes["gen_ai.request.top_k"]).toBe(3); expect(chat.attributes["gen_ai.usage.cache_read.input_tokens"]).toBe(1); expect(chat.attributes["gen_ai.response.id"]).toBe("invoke-1"); @@ -833,7 +842,6 @@ test("InvokeModelWithResponseStream parses each payload once and delivers malfor expect(await collect(response.body)).toEqual(events); expect(parse.mock.calls.length - before).toBe(4); - parse.mockRestore(); const [span] = spans.getFinishedSpans(); expect(span).toMatchObject({ [Symbol.for("telemetry.dev.outputChunkHistogram")]: { count: 2 }, @@ -996,7 +1004,7 @@ test("ApplyGuardrail captures action metadata", async () => { expect(span.attributes["td.metadata.guardrail_action_reason"]).toBe("blocked"); }); -test("errors, fail-open, idempotency, and prototype restore behave", async () => { +test("send errors end one error span, double wrapping traces once, and shutdown passes through", async () => { const spans = setup(); const error = Object.assign(new Error("throttled"), { @@ -1017,11 +1025,10 @@ test("errors, fail-open, idempotency, and prototype restore behave", async () => await teardown(); const disabled = wrapBedrock(new FakeClient([{ output: { message: { content: [] } } }])); - await disabled.send(new ConverseCommand({ modelId: "m", messages: [] })); + await expect(disabled.send(new ConverseCommand({ modelId: "m", messages: [] }))).resolves.toEqual( + { output: { message: { content: [] } } }, + ); expect(disabled.calls).toHaveLength(1); - - instrumentBedrock(); - uninstrumentBedrock(); }); test("callback-style send creates a span while returning undefined", () => { @@ -1066,67 +1073,66 @@ test("callback-style send creates a span while returning undefined", () => { expect(jsonAttr(span, "gen_ai.output.messages")[0].parts[0].content).toBe("ok"); }); -test("global instrumentation is idempotent", async () => { +test("global instrumentation patches the real client once and restores its inherited send", async () => { const spans = setup(); - - const originalDescriptor = Object.getOwnPropertyDescriptor( - BedrockRuntimeClient.prototype, - "send", - ); - - const fakeSend = async () => ({ - output: { message: { content: [] } }, - $metadata: { requestId: "global-1" }, + const proto = BedrockRuntimeClient.prototype; + const command = new ConverseCommand({ modelId: "m", messages: [] }); + + const client = new BedrockRuntimeClient({ + region: "us-east-1", + credentials: { accessKeyId: "test", secretAccessKey: "test" }, + maxAttempts: 1, + requestHandler: jsonRequestHandler("global-1", { + output: { message: { role: "assistant", content: [{ text: "ok" }] } }, + stopReason: "end_turn", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: { latencyMs: 1 }, + }), }); - Object.defineProperty(BedrockRuntimeClient.prototype, "send", { - configurable: true, - writable: true, - value: fakeSend, - }); + // AWS clients inherit send from the Smithy base class. + expect(Object.hasOwn(proto, "send")).toBe(false); try { instrumentBedrock(); instrumentBedrock(); - const client = new BedrockRuntimeClient({ - region: "us-east-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - }); - - await client.send(new ConverseCommand({ modelId: "m", messages: [] })); + await client.send(command); expect(spans.getFinishedSpans()).toHaveLength(1); expect(spans.getFinishedSpans()[0]!.attributes["gen_ai.response.id"]).toBe("global-1"); - const telemetryWrapper = Object.getOwnPropertyDescriptor( - BedrockRuntimeClient.prototype, - "send", - )?.value; - + const telemetryWrapper = Object.getOwnPropertyDescriptor(proto, "send")?.value; const laterWrapper = async () => ({ output: { message: { content: [] } } }); - Object.defineProperty(BedrockRuntimeClient.prototype, "send", { + Object.defineProperty(proto, "send", { configurable: true, writable: true, value: laterWrapper, }); uninstrumentBedrock(); - expect(Object.getOwnPropertyDescriptor(BedrockRuntimeClient.prototype, "send")?.value).toBe( - laterWrapper, - ); - Object.defineProperty(BedrockRuntimeClient.prototype, "send", { + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(laterWrapper); + Object.defineProperty(proto, "send", { configurable: true, writable: true, value: telemetryWrapper, }); uninstrumentBedrock(); - expect(Object.getOwnPropertyDescriptor(BedrockRuntimeClient.prototype, "send")?.value).toBe( - fakeSend, - ); + expect(Object.hasOwn(proto, "send")).toBe(false); + + await client.send(command); + expect(spans.getFinishedSpans()).toHaveLength(1); + + const priorOwner = async () => ({ output: { message: { content: [] } } }); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: priorOwner, + }); + instrumentBedrock(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).not.toBe(priorOwner); + uninstrumentBedrock(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(priorOwner); } finally { uninstrumentBedrock(); - - if (originalDescriptor) { - Object.defineProperty(BedrockRuntimeClient.prototype, "send", originalDescriptor); - } + Reflect.deleteProperty(proto, "send"); } }); diff --git a/packages/bedrock/tests/helpers.ts b/packages/bedrock/tests/helpers.ts index 6ebc2d0..5d8ed86 100644 --- a/packages/bedrock/tests/helpers.ts +++ b/packages/bedrock/tests/helpers.ts @@ -4,6 +4,7 @@ import { type ReadableSpan, type Sampler, } from "@opentelemetry/sdk-trace-base"; +import { HttpResponse } from "@smithy/protocol-http"; import { init, shutdown, type MaskFn } from "@telemetry-dev/sdk"; @@ -62,6 +63,19 @@ export class FakeClient { } } +/** A Smithy request handler that answers every request with one JSON body, so a real client runs its own send. */ +export function jsonRequestHandler(requestId: string, body: unknown) { + return { + handle: async () => ({ + response: new HttpResponse({ + statusCode: 200, + headers: { "content-type": "application/json", "x-amzn-requestid": requestId }, + body: new TextEncoder().encode(JSON.stringify(body)), + }), + }), + }; +} + export async function collect(iterable: AsyncIterable, limit?: number): Promise { const out: T[] = []; diff --git a/packages/cursor/tests/cursor.test.ts b/packages/cursor/tests/cursor.test.ts index bc243f6..080df60 100644 --- a/packages/cursor/tests/cursor.test.ts +++ b/packages/cursor/tests/cursor.test.ts @@ -162,18 +162,6 @@ describe("turn lifecycle", () => { expect(agent!.attributes["error.type"]).toBe("AgentError"); }); - test("a second beforeSubmitPrompt ends the stale turn as incomplete", async () => { - const { spans, telemetry } = makeHarness(); - telemetry.handle(event("beforeSubmitPrompt", { prompt: "first" })); - telemetry.handle(event("beforeSubmitPrompt", { prompt: "second", generation_id: "gen-2" })); - telemetry.handle(event("stop", { status: "completed", generation_id: "gen-2" })); - await flush(); - - const agents = agentSpans(spans, "cursor"); - expect(agents).toHaveLength(2); - expect(agents[0]!.attributes["gen_ai.response.finish_reasons"]).toEqual(["incomplete"]); - }); - test("generation_id drift within a turn does not split it", async () => { const { spans, telemetry } = makeHarness(); telemetry.handle(event("beforeSubmitPrompt", { prompt: "one turn" })); @@ -836,21 +824,6 @@ describe("titles", () => { } }); - test("a missing title falls back to the prompt as the span name", async () => { - const chats = mkdtempSync(join(tmpdir(), "cursor-chats-")); - const { spans, telemetry } = makeHarness({ chatsDir: chats }); - - try { - telemetry.handle(event("beforeSubmitPrompt", { prompt: "x" })); - telemetry.handle(event("stop", { status: "completed" })); - await flush(); - - expect(finished(spans).map((s) => s.name)).toContain("x"); - } finally { - rmSync(chats, { recursive: true, force: true }); - } - }); - test.each([ { captureInput: true, @@ -999,24 +972,34 @@ describe("settle", () => { }); test("malformed events never throw", () => { - const { telemetry } = makeHarness(); + const onError = vi.fn(); + const { telemetry } = makeHarness({ onError }); + const failure = new Error("unreadable hook payload"); const eventRecord = { hook_event_name: "unknownEvent", nested: { arbitrary: new Date() }, }; + const unreadable = { + get hook_event_name(): string { + throw failure; + }, + }; + expectTypeOf().parameter(0).toEqualTypeOf>(); telemetry.handle({}); telemetry.handle({ hook_event_name: "postToolUse" }); telemetry.handle({ hook_event_name: "unknownEvent", conversation_id: "conv-1" }); telemetry.handle(eventRecord); + expect(() => telemetry.handle(unreadable)).not.toThrow(); + expect(onError).toHaveBeenCalledWith(failure); expect(telemetry.open()).toBe(false); }); }); describe("fileConfig", () => { - test("a missing file yields defaults; an unreadable or non-object file disables telemetry", () => { + test("a missing file yields defaults and a non-string value is ignored", () => { const home = mkdtempSync(join(tmpdir(), "cursor-cfg-")); const restoreHome = setHome(home); const file = join(home, ".cursor", "telemetry-dev.json"); @@ -1025,11 +1008,6 @@ describe("fileConfig", () => { expect(fileConfig()).toEqual({}); mkdirSync(join(home, ".cursor")); - for (const content of ["{not json", '["captureInput", false]', '"text"', "null"]) { - writeFileSync(file, content); - expect(fileConfig(), content).toEqual({ enabled: false }); - } - // A non-string value is ignored rather than coerced, which could throw. vi.stubEnv("OTEL_SERVICE_NAME", undefined); vi.stubEnv("TELEMETRY_DEV_ENVIRONMENT", undefined); @@ -1050,6 +1028,8 @@ describe("fileConfig", () => { { content: '{"enabled":false}', exported: false }, { content: "{not json", exported: false }, { content: '["captureOutput", false]', exported: false }, + { content: '"text"', exported: false }, + { content: "null", exported: false }, // Creating symlinks on Windows needs extra privileges. ...(process.platform === "win32" ? [] : [{ content: "dangling symlink", exported: false }]), ])( diff --git a/packages/eve/tests/eve.test.ts b/packages/eve/tests/eve.test.ts index b97fc5e..d91ed42 100644 --- a/packages/eve/tests/eve.test.ts +++ b/packages/eve/tests/eve.test.ts @@ -309,6 +309,7 @@ function onlyLog(exporters: Exporters) { } afterEach(async () => { + vi.useRealTimers(); vi.unstubAllGlobals(); vi.restoreAllMocks(); @@ -406,11 +407,20 @@ test("instrumentation puts every ai.eve.turn of a session in the session trace", } }); -test("instrumentation captureInput:false maps to eve recordInputs:false", () => { - const definition = telemetryDevInstrumentation(telemetryOptions({ captureInput: false })); +test.each([ + { capture: { captureInput: false }, recordInputs: false, recordOutputs: true }, + { capture: { captureOutput: false }, recordInputs: true, recordOutputs: false }, +])( + "instrumentation maps $capture to eve recordInputs and recordOutputs", + ({ capture, recordInputs, recordOutputs }) => { + const definition = telemetryDevInstrumentation(telemetryOptions(capture)); - expect(definition.recordInputs).toBe(false); -}); + expect([definition.recordInputs, definition.recordOutputs]).toEqual([ + recordInputs, + recordOutputs, + ]); + }, +); test("provider integration forwards content capture flags", () => { const integration = telemetryDevOtelIntegration({ @@ -952,17 +962,31 @@ test.each([ test("wrapped eve client puts every turn of one session in the session trace", async () => { const exporters = makeExporters(); - stubFetchSequence([ + + const responses = [ jsonResponse({ sessionId: "session-1" }), ndjsonResponse(successfulTurnEvents()), jsonResponse({ sessionId: "session-1" }), ndjsonResponse(successfulTurnEvents()), - ]); + ]; + + // Each request takes a minute on a frozen clock: a span that started when its + // POST resolved would start a minute after its turn was sent. + vi.useFakeTimers({ now: Date.parse("2026-01-02T03:04:05.000Z"), toFake: ["Date"] }); + vi.stubGlobal( + "fetch", + vi.fn(async () => { + vi.setSystemTime(Date.now() + 60_000); + + return responses.shift()!; + }), + ); const client = wrapClient(exporters); - const before = Date.now(); + const sentAt = [Date.now()]; const { response, session } = await client.sessions.create({ message: "first" }); await response.result(); + sentAt.push(Date.now()); await (await session.send("second")).result(); await flush(); @@ -976,10 +1000,9 @@ test("wrapped eve client puts every turn of one session in the session trace", a expect(span.spanContext().traceId).toBe(expected.traceId); expect(span.parentSpanContext?.spanId).toBe(expected.spanId); expect(span.attributes["gen_ai.conversation.id"]).toBe("session-1"); - // The span keeps the pre-send start time even though it is created after the response. - expect(ms(span.startTime)).toBeGreaterThanOrEqual(before); - expect(ms(span.startTime)).toBeLessThanOrEqual(ms(span.endTime)); } + + expect(spans.map((span) => ms(span.startTime))).toEqual(sentAt); }); test("wrapped eve client marks streams without terminal events as errored spans", async () => { diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index 1f5d07b..d1d291a 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -1820,6 +1820,7 @@ test.each([ const span = await exportedSpan(spans); expect(span.attributes["google_genai.response.block_reason"]).toBe("SAFETY"); + expect(span.status.code).toBe(SPAN_STATUS_UNSET); if (capture === "on") { expect(span.attributes["google_genai.response.block_reason_message"]).toBe( @@ -2516,32 +2517,6 @@ test("non-Error stream setup failures end spans and preserve the thrown values", ).toEqual(["stream setup rejected", "[object Object]"]); }); -test("blocked prompt records block attrs without output", async () => { - const spans = setupSpans(); - - const fake = createFakeFetch( - jsonResponse({ - promptFeedback: { - blockReason: "SAFETY", - blockReasonMessage: "blocked for safety", - safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", probability: "HIGH" }], - }, - }), - ); - - vi.stubGlobal("fetch", fake.fetch); - const client = clientWith(); - await client.models.generateContent({ model: "gemini-2.5-flash", contents: "blocked" }); - const span = await exportedSpan(spans); - expect(span.status.code).toBe(SPAN_STATUS_UNSET); - expect(span.attributes["google_genai.response.block_reason"]).toBe("SAFETY"); - expect(span.attributes["google_genai.response.block_reason_message"]).toBe("blocked for safety"); - expect(jsonAttr(span, "google_genai.response.prompt_safety_ratings")).toEqual([ - { category: "HARM_CATEGORY_DANGEROUS_CONTENT", probability: "HIGH" }, - ]); - expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); -}); - test("embedContent maps embedding span fields without output", async () => { const spans = setupSpans(); @@ -2704,7 +2679,7 @@ test("wrapped calls fail open when telemetry is not initialized", async () => { expect(response.text).toBe("ok"); }); -test("usage modality details map text, image, and audio without video attributes", async () => { +test("usage modality details map text, image, and audio tokens and ignore other modalities", async () => { const spans = setupSpans(); const client = wrapGoogleGenAI({ @@ -2737,9 +2712,6 @@ test("usage modality details map text, image, and audio without video attributes expect(span.attributes["gen_ai.usage.text.cache_read.input_tokens"]).toBe(5); expect(span.attributes["gen_ai.usage.image.cache_read.input_tokens"]).toBe(6); expect(span.attributes["gen_ai.usage.audio.cache_read.input_tokens"]).toBe(7); - expect( - Object.keys(span.attributes).some((key) => key.includes("video") && key.includes("usage")), - ).toBe(false); }); test.each(["generateImages", "editImage", "upscaleImage"] as const)( @@ -2775,7 +2747,6 @@ test.each(["generateImages", "editImage", "upscaleImage"] as const)( expect(messagesAttr(span, "gen_ai.output.messages")).toEqual([ { type: "image", uri: "gs://bucket/output.png" }, ]); - expect(span.attributes["google_genai.response.image_uris"]).toBeUndefined(); expect(JSON.stringify(span.attributes)).not.toContain(binary); }, ); @@ -2857,7 +2828,11 @@ test("generateVideos traces submit and tracked polling idempotently", async () = expect(messagesAttr(exported[1]!, "gen_ai.output.messages")).toEqual([ { type: "video", uri: "gs://bucket/video.mp4" }, ]); - expect(exported[1]!.attributes["google_genai.response.video_uris"]).toBeUndefined(); + expect( + Object.keys(exported[1]!.attributes).filter((key) => + String(exported[1]!.attributes[key]).includes("gs://bucket/video.mp4"), + ), + ).toEqual(["gen_ai.output.messages"]); expect(exported.every((span) => span.attributes["gen_ai.output.type"] === "video")).toBe(true); }); @@ -3094,30 +3069,6 @@ test.each(["get", "getVideosOperation"] as const)( }, ); -test("media methods preserve rejected and synchronously thrown failures", async () => { - const spans = setupSpans(); - const rejected = new Error("image rejected"); - const thrown = new Error("video threw"); - - const client = wrapGoogleGenAI({ - models: { - generateImages(_params: unknown) { - return Promise.reject(rejected); - }, - generateVideos(_params: unknown) { - throw thrown; - }, - }, - }); - - await expect(client.models.generateImages({ model: "imagen", prompt: "fail" })).rejects.toBe( - rejected, - ); - expect(() => client.models.generateVideos({ model: "veo", prompt: "fail" })).toThrow(thrown); - const exported = await finishedSpans(spans, 2); - expect(exported.every((span) => span.status.code === SPAN_STATUS_ERROR)).toBe(true); -}); - test("capture traversal is bounded, cycle-aware, binary-safe, and preserves provider outcomes", async () => { const spans = setupSpans(); const shared = { text: "shared" }; @@ -3157,6 +3108,8 @@ test("capture traversal is bounded, cycle-aware, binary-safe, and preserves prov const span = await exportedSpan(spans); expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); expect(String(span.attributes["gen_ai.input.messages"])).not.toContain("SECRET_BINARY"); + expect(String(span.attributes["gen_ai.input.messages"])).toContain('"shared"'); + expect(String(span.attributes["gen_ai.output.messages"])).toContain('"shared"'); expect(String(span.attributes["gen_ai.input.messages"]).length).toBeLessThan(65_536); expect(String(span.attributes["gen_ai.output.messages"]).length).toBeLessThan(65_536); }); diff --git a/packages/mcp/tests/mcp.test.ts b/packages/mcp/tests/mcp.test.ts index 40757ed..bd0e590 100644 --- a/packages/mcp/tests/mcp.test.ts +++ b/packages/mcp/tests/mcp.test.ts @@ -251,7 +251,12 @@ test("completes responses before a per-request HTTP transport closes", async () await handler.close(); }); -test("applies asymmetric JSON-RPC error status rules", async () => { +test.each([ + { code: -32602, serverFault: false }, + { code: -32021, serverFault: false }, + { code: -32022, serverFault: false }, + { code: -32603, serverFault: true }, +])("applies asymmetric JSON-RPC error status rules to $code", async ({ code, serverFault }) => { const spans = setup(); const [client, server] = InMemoryTransport.createLinkedPair().map((transport) => @@ -260,60 +265,24 @@ test("applies asymmetric JSON-RPC error status rules", async () => { client.onmessage = () => undefined; server.onmessage = () => { - void server.send({ - jsonrpc: "2.0", - id: 7, - error: { code: -32602, message: "Invalid params" }, - }); + void server.send({ jsonrpc: "2.0", id: 7, error: { code, message: "JSON-RPC error" } }); }; await client.start(); await server.start(); - await client.send({ jsonrpc: "2.0", id: 7, method: "prompts/get", params: { name: "review" } }); + await client.send({ jsonrpc: "2.0", id: 7, method: "ping" }); const finished = await exported(spans, 2); const clientSpan = finished.find((span) => span.kind === SpanKind.CLIENT)!; const serverSpan = finished.find((span) => span.kind === SpanKind.SERVER)!; - expect(clientSpan.attributes["rpc.response.status_code"]).toBe("-32602"); - expect(clientSpan.attributes["error.type"]).toBe("-32602"); + expect(clientSpan.attributes["rpc.response.status_code"]).toBe(String(code)); + expect(serverSpan.attributes["rpc.response.status_code"]).toBe(String(code)); + expect(clientSpan.attributes["error.type"]).toBe(String(code)); expect(clientSpan.status.code).toBe(SpanStatusCode.ERROR); - expect(serverSpan.attributes["rpc.response.status_code"]).toBe("-32602"); - expect(serverSpan.attributes["error.type"]).toBeUndefined(); - expect(serverSpan.status.code).toBe(SpanStatusCode.UNSET); + expect(serverSpan.attributes["error.type"]).toBe(serverFault ? String(code) : undefined); + expect(serverSpan.status.code).toBe(serverFault ? SpanStatusCode.ERROR : SpanStatusCode.UNSET); }); -test.each([-32021, -32022])( - "treats MCP v2 caller error %i as a client failure only", - async (code) => { - const spans = setup(); - - const [client, server] = InMemoryTransport.createLinkedPair().map((transport) => - instrumentMcpTransport(transport), - ); - - client.onmessage = () => undefined; - server.onmessage = () => { - void server.send({ - jsonrpc: "2.0", - id: 8, - error: { code, message: "Client compatibility error" }, - }); - }; - - await client.start(); - await server.start(); - await client.send({ jsonrpc: "2.0", id: 8, method: "ping" }); - - const finished = await exported(spans, 2); - const clientSpan = finished.find((span) => span.kind === SpanKind.CLIENT)!; - const serverSpan = finished.find((span) => span.kind === SpanKind.SERVER)!; - expect(clientSpan.attributes["error.type"]).toBe(String(code)); - expect(clientSpan.status.code).toBe(SpanStatusCode.ERROR); - expect(serverSpan.attributes["error.type"]).toBeUndefined(); - expect(serverSpan.status.code).toBe(SpanStatusCode.UNSET); - }, -); - test("marks CallToolResult errors and closes pending requests", async () => { const spans = setup(); @@ -348,12 +317,14 @@ test("marks CallToolResult errors and closes pending requests", async () => { }; const transport = instrumentMcpTransport(rawTransport); - transport.onclose = () => undefined; + const hostClose = vi.fn(); + transport.onclose = hostClose; await transport.send({ jsonrpc: "2.0", id: 10, method: "ping" } as never); transport.onclose?.(); const [closed] = await exported(closeSpans, 1); expect(closed?.attributes["error.type"]).toBe("connection_error"); expect(closed?.status.code).toBe(SpanStatusCode.ERROR); + expect(hostClose).toHaveBeenCalledOnce(); }); test("ends an outgoing request span when send fails", async () => { diff --git a/packages/omp/tests/omp.test.ts b/packages/omp/tests/omp.test.ts index c2ecc94..efddf8d 100644 --- a/packages/omp/tests/omp.test.ts +++ b/packages/omp/tests/omp.test.ts @@ -393,6 +393,7 @@ test("tool execution produces an execute_tool span with arguments and result", a "gen_ai.operation.name": "execute_tool", "gen_ai.tool.call.arguments": JSON.stringify({ command: "ls" }), "gen_ai.tool.call.id": "call-1", + "gen_ai.tool.call.result": JSON.stringify({ content: [{ type: "text", text: "README.md" }] }), "gen_ai.tool.name": "bash", }); expect(span.status.code).not.toBe(SpanStatusCode.ERROR); @@ -542,14 +543,17 @@ test("malformed events never throw and report through onError", async () => { expect(harness.exporters.spans.getFinishedSpans().length).toBeGreaterThan(0); }); -test("events for a different session carry that session's conversation id", async () => { +test("each event reads the current session id", async () => { const harness = makeHarness(); - const other = makeContext("session-2"); - await harness.emit({ type: "session_start" }, other); + await harness.emit({ type: "session_start" }, makeContext("session-2")); + await harness.emit({ type: "session_shutdown" }); await flush(); - const record = harness.exporters.logs.getFinishedLogRecords()[0]; - expect(record?.attributes["gen_ai.conversation.id"]).toBe("session-2"); + const records = harness.exporters.logs.getFinishedLogRecords(); + expect(records.map((record) => record.attributes["gen_ai.conversation.id"])).toEqual([ + "session-2", + "session-1", + ]); }); test("tool spans nest under the chat span that issued the tool call", async () => { diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 81fdd87..4467aa3 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -35,11 +35,15 @@ test("runs the compatibility suite against the selected OpenAI major", () => { expect(VERSION).toBe(process.env.OPENAI_SDK_VERSION === "6" ? "6.45.0" : "7.18.0"); }); -test("package exports resolve to built files inside the package", () => { - expect(packageJson.exports["."].default).toBe("./dist/index.mjs"); - expect(packageJson.publishConfig.exports["."].types).toBe("./dist/index.d.mts"); - expect(packageJson.publishConfig.exports["."].import).toBe("./dist/index.mjs"); - expect(packageJson.publishConfig.exports["."].default).toBe("./dist/index.mjs"); +test("published exports resolve types first and point only at shipped dist files", () => { + const published = packageJson.publishConfig.exports["."]; + + expect(packageJson.files).toContain("dist"); + expect(Object.keys(published)).toEqual(["types", "import", "default"]); + expect(published.types).toMatch(/\.d\.mts$/); + + for (const target of [packageJson.exports["."].default, ...Object.values(published)]) + expect(target).toMatch(/^\.\/dist\/[^/]+$/); }); test.each(["", "8"])("rejects unsupported OpenAI SDK selector %j", async (version) => { @@ -601,18 +605,18 @@ test("chat completions preserve tool-call messages in output", async () => { expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["tool_calls"]); }); -test("chat completions capture every returned choice", async () => { +test("chat completions capture every returned choice with its finish reason", async () => { const spans = setupSpans(); const fake = createFakeFetch( jsonResponse({ - id: "chatcmpl_two", + id: "chatcmpl_multi_finish", object: "chat.completion", created: 1, model: "gpt-4o-2024-11-20", choices: [ { index: 0, message: { role: "assistant", content: "First" }, finish_reason: "stop" }, - { index: 1, message: { role: "assistant", content: "Second" }, finish_reason: "stop" }, + { index: 1, message: { role: "assistant", content: "Second" }, finish_reason: "length" }, ], usage: { prompt_tokens: 10, completion_tokens: 8, total_tokens: 18 }, }), @@ -620,69 +624,16 @@ test("chat completions capture every returned choice", async () => { await clientWith(fake.fetch).chat.completions.create({ model: "gpt-4o", - messages: [{ role: "user", content: "Give two options" }], + messages: [{ role: "user", content: "Two answers" }], n: 2, }); const span = await exportedSpan(spans); - expect(jsonAttr(span, "gen_ai.output.messages")).toHaveLength(2); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ { role: "assistant", content: "First" }, { role: "assistant", content: "Second" }, ]); -}); - -test("chat completions record finish reasons for single and multi-choice responses", async () => { - const spans = setupSpans(); - - const fake = createFakeFetch( - jsonResponse({ - id: "chatcmpl_single_finish", - object: "chat.completion", - created: 1, - model: "gpt-4o-2024-11-20", - choices: [ - { index: 0, message: { role: "assistant", content: "Single" }, finish_reason: "stop" }, - ], - usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 }, - }), - jsonResponse({ - id: "chatcmpl_multi_finish", - object: "chat.completion", - created: 1, - model: "gpt-4o-2024-11-20", - choices: [ - { index: 0, message: { role: "assistant", content: "First" }, finish_reason: "stop" }, - { index: 1, message: { role: "assistant", content: "Second" }, finish_reason: "length" }, - ], - usage: { prompt_tokens: 10, completion_tokens: 8, total_tokens: 18 }, - }), - ); - - const client = clientWith(fake.fetch); - - await client.chat.completions.create({ - model: "gpt-4o", - messages: [{ role: "user", content: "One answer" }], - }); - await client.chat.completions.create({ - model: "gpt-4o", - messages: [{ role: "user", content: "Two answers" }], - n: 2, - }); - - const finished = await finishedSpans(spans, 2); - - const single = finished.find( - (span) => span.attributes["gen_ai.response.id"] === "chatcmpl_single_finish", - ); - - const multi = finished.find( - (span) => span.attributes["gen_ai.response.id"] === "chatcmpl_multi_finish", - ); - - expect(single?.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); - expect(multi?.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); + expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); }); test("chunk timing preserves delivery", async () => { @@ -907,7 +858,7 @@ test("chat streams record finish reasons for every choice", async () => { expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); }); -test("chat streams leave the request unchanged and capture no usage by default", async () => { +test("chat streams send the request unchanged by default", async () => { const spans = setupSpans(); const fake = createFakeFetch( @@ -946,7 +897,6 @@ test("chat streams leave the request unchanged and capture no usage by default", expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ { role: "assistant", content: "Hello" }, ]); - expect(span.attributes["gen_ai.usage.total_tokens"]).toBeUndefined(); }); test("chat streams bound retained content without changing events or usage", async () => { @@ -1090,42 +1040,6 @@ test.each(["chat", "responses", "transcription"] as const)( }, ); -test("chat streams retain output when the core span lacks chunk recording", async () => { - const spans = setupSpans(); - const startSpan = telemetrySdk.startSpan; - vi.spyOn(telemetrySdk, "startSpan").mockImplementation((...args) => { - const legacySpan = { ...startSpan(...args) }; - Reflect.deleteProperty(legacySpan, "recordOutputChunk"); - - return legacySpan; - }); - - const events = [ - { choices: [{ index: 0, delta: { content: "delivered" }, finish_reason: "stop" }] }, - ]; - - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create: async (_params: unknown) => source } }, - responses: { create() {} }, - embeddings: { create() {} }, - }); - - const stream = await client.chat.completions.create({ - model: "gpt-4o", - messages: [], - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - expect(jsonAttr(await exportedSpan(spans), "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: "delivered" }, - ]); -}); - test("chat streams bound retained tool arguments", async () => { const spans = setupSpans(); const states = observeChatChoiceStates(); @@ -2517,7 +2431,6 @@ test("chat streams reclaim finish-reason items when replacing a choice", async ( test("chat streams bound retained choice states", async () => { const spans = setupSpans(); - const states = observeChatChoiceStates(); const event = { choices: Array.from({ length: 500 }, (_, index) => ({ @@ -2544,7 +2457,6 @@ test("chat streams bound retained choice states", async () => { const iterator = stream[Symbol.asyncIterator](); expect((await iterator.next()).value).toBe(event); - expect(states().size).toBe(499); expect(await iterator.next()).toEqual({ done: true, value: undefined }); const span = await exportedSpan(spans); const output = jsonAttr(span, "gen_ai.output.messages"); @@ -2785,83 +2697,6 @@ test("chat streams inject include_usage when opted in and hide only the syntheti ).toBe("number"); }); -test.each([true, false])( - "nonterminal chat stream close marks capture incomplete with output capture=%s", - async (captureOutput) => { - const spans = setupSpans(undefined, { captureOutput }); - - const event = { - choices: [{ index: 0, delta: { role: "assistant", content: "partial" } }], - }; - - const source = new Stream(async function* () { - yield event; - yield { choices: [{ index: 0, delta: { content: "unread" }, finish_reason: "stop" }] }; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create: async (_params: unknown) => source } }, - responses: { create() {} }, - embeddings: { create() {} }, - }); - - const stream = await client.chat.completions.create({ - model: "gpt-4o", - messages: [], - stream: true, - }); - - for await (const chunk of stream) { - expect(chunk).toBe(event); - break; - } - - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.finish_reasons"]).toBeUndefined(); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); - - if (captureOutput) - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: "partial" }, - ]); - else expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); - }, -); - -test("nonterminal chat stream errors omit finish reasons and mark capture incomplete", async () => { - const spans = setupSpans(); - const failure = new Error("stream failed before terminal chunk"); - - const event = { - choices: [{ index: 0, delta: { role: "assistant", content: "partial" } }], - }; - - const source = new Stream(async function* () { - yield event; - throw failure; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create: async (_params: unknown) => source } }, - responses: { create() {} }, - embeddings: { create() {} }, - }); - - const stream = await client.chat.completions.create({ - model: "gpt-4o", - messages: [], - stream: true, - }); - - await expect(collectStream(stream)).rejects.toBe(failure); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.finish_reasons"]).toBeUndefined(); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: "partial" }, - ]); -}); - test.each([false, true])( "a post-finish content filter chunk keeps a complete chat stream complete (masked: %s)", async (masked) => { @@ -3325,55 +3160,6 @@ test.each([ ]); }); -test("responses streams end from response.completed terminal event", async () => { - const spans = setupSpans(); - - const completed = { - id: "resp_stream", - object: "response", - status: "completed", - model: "gpt-4.1-2025-04-14", - output: [ - { - id: "msg_stream", - type: "message", - role: "assistant", - content: [{ type: "output_text", text: "Done" }], - }, - ], - usage: { input_tokens: 8, output_tokens: 2, total_tokens: 10 }, - }; - - const fake = createFakeFetch( - sseResponse([ - { type: "response.created", response: { id: "resp_stream", status: "in_progress" } }, - { type: "response.completed", response: completed }, - ]), - ); - - const stream = await clientWith(fake.fetch).responses.create({ - model: "gpt-4.1", - input: "Finish", - stream: true, - }); - - const events = await collectStream(stream); - - expect(events).toHaveLength(2); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.id"]).toBe("resp_stream"); - expect(span.attributes["gen_ai.response.model"]).toBe("gpt-4.1-2025-04-14"); - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual(completed.output); - expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); - expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(10); - expect( - Number(span.attributes["gen_ai.response.time_to_first_chunk"]) === - span.attributes["gen_ai.response.time_to_first_chunk"] - ? "number" - : "other", - ).toBe("number"); -}); - test("responses streams retain the last fitting output when the terminal snapshot truncates", async () => { const spans = setupSpans(); @@ -3893,6 +3679,8 @@ test("responses streams end from terminal event before reading the stream sentin const span = await exportedSpan(spans); expect(span.attributes["gen_ai.response.id"]).toBe("resp_stream_terminal"); + expect(span.attributes["gen_ai.response.model"]).toBe("gpt-4.1-2025-04-14"); + expect(span.attributes["gen_ai.response.time_to_first_chunk"]).toEqual(expect.any(Number)); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual(completed.output); expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(10); @@ -4087,21 +3875,12 @@ test("global instrumentation records responses and embeddings with wrapOpenAI pa expect(responseSpan?.attributes["gen_ai.provider.name"]).toBe("openai"); expect(responseSpan?.attributes["gen_ai.operation.name"]).toBe("chat"); expect(responseSpan?.attributes["gen_ai.system_instructions"]).toBe("You must never tell jokes"); - expect(responseSpan?.attributes["gen_ai.input.messages"]).toBe("Tell me a joke"); expect(jsonAttr(responseSpan!, "gen_ai.output.messages")).toEqual(output); - expect(responseSpan?.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); - expect(responseSpan?.attributes["gen_ai.usage.cache_read.input_tokens"]).toBe(4); - expect(responseSpan?.attributes["gen_ai.usage.reasoning.output_tokens"]).toBe(1); - expect(responseSpan?.attributes["gen_ai.request.max_tokens"]).toBe(50); const embeddingSpan = finished.find((span) => span.name === "embeddings text-embedding-3-small"); expect(embeddingSpan?.attributes["gen_ai.provider.name"]).toBe("openai"); expect(embeddingSpan?.attributes["gen_ai.operation.name"]).toBe("embeddings"); - expect(embeddingSpan?.attributes["gen_ai.request.model"]).toBe("text-embedding-3-small"); - expect(embeddingSpan?.attributes["gen_ai.response.model"]).toBe("text-embedding-3-small"); - expect(embeddingSpan?.attributes["gen_ai.input.messages"]).toBe("OpenTelemetry traces"); expect(embeddingSpan?.attributes["gen_ai.usage.input_tokens"]).toBe(6); - expect(embeddingSpan?.attributes["gen_ai.usage.total_tokens"]).toBe(6); expect(embeddingSpan?.attributes["gen_ai.output.messages"]).toBeUndefined(); }); @@ -4675,7 +4454,10 @@ test("images generate maps modality usage without capturing image bytes", async expect(span.attributes["gen_ai.usage.image.cache_read.input_tokens"]).toBe(1); expect(span.attributes["gen_ai.usage.audio.cache_read.input_tokens"]).toBe(0); expect(span.attributes["gen_ai.usage.image.output_tokens"]).toBe(12); - expect(String(span.attributes["gen_ai.output.messages"])).not.toContain("secret-image-bytes"); + expect(jsonAttr(span, "gen_ai.output.messages")).toEqual({ + count: 1, + revised_prompts: ["A revised prompt"], + }); }); test.each([true, false])( @@ -4783,32 +4565,6 @@ test("streaming transcription bounds captured output while yielding every event" expect(span.attributes["gen_ai.usage.text.output_tokens"]).toBe(5); }); -test("streaming transcription retains no more than the transcript ceiling", async () => { - const spans = setupSpans(); - const events = [{ type: "transcript.text.delta", delta: "x".repeat(70_000) }]; - - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); - - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("x".repeat(65_534)); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); -}); - test("streaming transcription omits a bounded prefix that a mask cannot evaluate safely", async () => { const spans = setupSpans(undefined, { mask: (value) => (String(value).includes("SECRET") ? "[REDACTED]" : value), @@ -4950,94 +4706,73 @@ test.each([ }, ); -test("a complete masked transcription stream keeps its output and is not flagged", async () => { - const spans = setupSpans(undefined, { mask: (value) => value }); - - const events = [ - { type: "transcript.text.delta", delta: "partial" }, - { type: "transcript.text.done", text: "complete transcript" }, - ]; - - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); - - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("complete transcript"); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBeUndefined(); -}); - -test("streaming transcription replaces truncated deltas with complete terminal text", async () => { - const spans = setupSpans(); +test.each([false, true])( + "streaming transcription replaces truncated deltas with complete terminal text (masked: %s)", + async (masked) => { + const spans = setupSpans(undefined, masked ? { mask: (value) => value } : {}); - const events = [ - { type: "transcript.text.delta", delta: "x".repeat(70_000) }, - { type: "transcript.text.done", text: "complete transcript" }, - ]; + const events = [ + { type: "transcript.text.delta", delta: "x".repeat(70_000) }, + { type: "transcript.text.done", text: "complete transcript" }, + ]; - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); + const source = new Stream(async function* () { + yield* events; + }, new AbortController()); - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); + const client = wrapOpenAI({ + chat: { completions: { create() {} } }, + responses: { create() {} }, + embeddings: { create() {} }, + audio: { transcriptions: { create: async (_params: unknown) => source } }, + }); - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); + const stream = await client.audio!.transcriptions!.create({ + model: "gpt-4o-transcribe", + stream: true, + }); - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("complete transcript"); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBeUndefined(); -}); + expect(await collectStream(stream)).toEqual(events); + const span = await exportedSpan(spans); + expect(span.attributes["gen_ai.output.messages"]).toBe("complete transcript"); + expect(span.attributes["telemetry.dev.capture.truncated"]).toBeUndefined(); + }, +); -test("streaming transcription captures many small deltas in linear work", async () => { - const spans = setupSpans(); +test.each([ + ["one oversized delta", 1, 70_000], + ["70,000 one-character deltas", 70_000, 1], +])( + "streaming transcription retains at most the transcript ceiling from %s", + async (_case, count, size) => { + const spans = setupSpans(); - const events = Array.from({ length: 70_000 }, () => ({ - type: "transcript.text.delta", - delta: "x", - })); + const events = Array.from({ length: count }, () => ({ + type: "transcript.text.delta", + delta: "x".repeat(size), + })); - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); + const source = new Stream(async function* () { + yield* events; + }, new AbortController()); - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); + const client = wrapOpenAI({ + chat: { completions: { create() {} } }, + responses: { create() {} }, + embeddings: { create() {} }, + audio: { transcriptions: { create: async (_params: unknown) => source } }, + }); - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); + const stream = await client.audio!.transcriptions!.create({ + model: "gpt-4o-transcribe", + stream: true, + }); - expect(await collectStream(stream)).toHaveLength(events.length); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("x".repeat(65_534)); -}); + expect(await collectStream(stream)).toHaveLength(events.length); + const span = await exportedSpan(spans); + expect(span.attributes["gen_ai.output.messages"]).toBe("x".repeat(65_534)); + }, +); test.each([ ["enabled", true, "hello"], @@ -5656,36 +5391,27 @@ test("realtime wrapper correlates reversed manual responses and ignores automati expect(withMetadataFailure.find((span) => span.status.code === SPAN_STATUS_ERROR)).toBeDefined(); }); -test("realtime wrapper ends active spans on transport errors and close", async () => { +test("realtime close fails pending spans and stops tracing later sends", async () => { const spans = setupSpans(); - const listeners = new Map void>>(); + const sent: unknown[] = []; const connection = { - send(_event?: unknown) {}, - close() {}, - on(event: string, listener: (...args: unknown[]) => void) { - listeners.set(event, [...(listeners.get(event) ?? []), listener]); - }, - off(event: string, listener: (...args: unknown[]) => void) { - listeners.set( - event, - (listeners.get(event) ?? []).filter((candidate) => candidate !== listener), - ); + send(event: unknown) { + sent.push(event); }, + close() {}, + on() {}, + off() {}, }; const wrapped = wrapOpenAIRealtime(connection); - const emit = (event: string, value: unknown) => { - for (const listener of listeners.get(event) ?? []) listener(value); - }; - wrapped.send({ type: "response.create" }); - emit("error", "socket failed"); wrapped.send({ type: "response.create" }); wrapped.close(); wrapped.send({ type: "response.create" }); + expect(sent[2]).toEqual({ type: "response.create" }); const finished = await finishedSpans(spans, 2); expect(finished.every((span) => span.status.code === SPAN_STATUS_ERROR)).toBe(true); expect(spans.getFinishedSpans()).toHaveLength(2); diff --git a/packages/opencode/tests/opencode.test.ts b/packages/opencode/tests/opencode.test.ts index 35bdad0..69851b4 100644 --- a/packages/opencode/tests/opencode.test.ts +++ b/packages/opencode/tests/opencode.test.ts @@ -264,34 +264,6 @@ test.each([ expect(span.endTime[0]).toBe(1_700_000_021); }); -test("terminal session.error marks the agent span when the session settles", async () => { - const harness = await makeHarness(); - await chatMessage(harness.hooks); - - await event(harness.hooks, "session.error", { - sessionID: "session-1", - error: { name: "ApiError", data: { message: "provider overloaded" } }, - }); - await event(harness.hooks, "session.idle", { sessionID: "session-1" }); - await flush(); - - const span = spanByName(harness.exporters, "invoke_agent"); - expect(span.status.code).toBe(SpanStatusCode.ERROR); - expect(span.attributes["error.type"]).toBe("ApiError"); - - const records = harness.exporters.logs.getFinishedLogRecords(); - expect(records).toHaveLength(1); - expect(records[0]?.body).toBe("Session error"); - expect(records[0]?.eventName).toBe("session.error"); - expect(records[0]?.severityText).toBe("ERROR"); - expect(records[0]?.attributes).toMatchObject({ - "gen_ai.agent.name": "opencode", - "gen_ai.conversation.id": "session-1", - "opencode.error.message": "provider overloaded", - "opencode.error.name": "ApiError", - }); -}); - test("dispose ends every dangling agent span as incomplete and flushes", async () => { const harness = await makeHarness(); await chatMessage(harness.hooks, "session-1"); @@ -307,23 +279,6 @@ test("dispose ends every dangling agent span as incomplete and flushes", async ( } }); -test("dispose applies a deferred session.error instead of ending the span as incomplete", async () => { - const harness = await makeHarness(); - await chatMessage(harness.hooks); - await event(harness.hooks, "session.error", { - sessionID: "session-1", - error: { name: "ApiError", data: { message: "provider overloaded" } }, - }); - - // The plugin is disposed before the session ever goes idle. - await harness.hooks.dispose?.(); - - const span = spanByName(harness.exporters, "invoke_agent"); - expect(span.status.code).toBe(SpanStatusCode.ERROR); - expect(span.attributes["error.type"]).toBe("ApiError"); - expect(span.attributes["gen_ai.response.finish_reasons"]).toBeUndefined(); -}); - describe.each([ { name: "default capture", @@ -382,10 +337,19 @@ describe.each([ const stack = String(exception?.["exception.stacktrace"]); expect(stack.startsWith(`UnknownError: ${message}\n`), name).toBe(true); expect(stack.includes("private-attachment-listing"), name).toBe(message !== "session error"); + expect(span.attributes["gen_ai.response.finish_reasons"], name).toBeUndefined(); } const records = harness.exporters.logs.getFinishedLogRecords(); - expect(records[0]?.attributes["opencode.error.name"]).toBe("UnknownError"); + expect(records).toHaveLength(1); + expect(records[0]?.body).toBe("Session error"); + expect(records[0]?.eventName).toBe("session.error"); + expect(records[0]?.severityText).toBe("ERROR"); + expect(records[0]?.attributes).toMatchObject({ + "gen_ai.agent.name": "opencode", + "gen_ai.conversation.id": "session-1", + "opencode.error.name": "UnknownError", + }); expect(records[0]?.attributes["opencode.error.message"]).toBe(logMessage); }); }); diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index dcfa465..1296de0 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -2081,47 +2081,38 @@ test("multi-choice chat streams reconstruct every choice", async () => { expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); }); -test("non-2xx chat responses end one error span and propagate the failure", async () => { +test.each([ + { + failure: "non-2xx responses", + fetcher: (): Fetcher => + createFakeFetcher( + new Response(JSON.stringify({ error: { code: 500, message: "provider exploded" } }), { + status: 500, + headers: { "content-type": "application/json" }, + }), + ).fetcher, + message: "provider exploded", + }, + { + failure: "rejected transport requests", + fetcher: (): Fetcher => () => Promise.reject(new Error("network down")), + message: "network down", + }, +])("chat $failure end one error span and propagate the failure", async ({ fetcher, message }) => { const spans = setupSpans(); - const fake = createFakeFetcher( - new Response(JSON.stringify({ error: { code: 500, message: "provider exploded" } }), { - status: 500, - headers: { "content-type": "application/json" }, - }), - ); - await expect( - clientWith(fake.fetcher).chat.send({ + clientWith(fetcher()).chat.send({ chatRequest: { model: "openai/gpt-4o", messages: [{ role: "user", content: "Boom" }], }, }), - ).rejects.toThrow(); - - const span = await exportedSpan(spans); - expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.events[0]?.name).toBe("exception"); -}); - -test("rejected transport requests end one error span and rethrow", async () => { - const spans = setupSpans(); - const failure = new Error("network down"); - const fetcher: Fetcher = () => Promise.reject(failure); - - await expect( - clientWith(fetcher).chat.send({ - chatRequest: { - model: "openai/gpt-4o", - messages: [{ role: "user", content: "Offline" }], - }, - }), - ).rejects.toThrow("network down"); + ).rejects.toThrow(message); const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.events[0]?.attributes?.["exception.message"]).toContain("network down"); + expect(span.events[0]?.attributes?.["exception.message"]).toContain(message); }); test("mid-stream body failures end one error span with partial output", async () => { diff --git a/packages/otel/src/transport.ts b/packages/otel/src/transport.ts index 4c684ed..2fde519 100644 --- a/packages/otel/src/transport.ts +++ b/packages/otel/src/transport.ts @@ -250,7 +250,6 @@ async function postSerialized( target: OtlpTarget, transport: Transport, label: string, - post: typeof postOtlp = postOtlp, signal?: AbortSignal, ): Promise { const { body: finalBody, contentEncoding } = await maybeGzip(body); @@ -259,7 +258,7 @@ async function postSerialized( ? { ...target.headers, "content-encoding": contentEncoding } : target.headers; - const res = await post({ + const res = await postOtlp({ fetchImpl: transport.fetchImpl, url: target.url, headers, @@ -387,7 +386,7 @@ function createOtlpBatchSender( return; } - await postSerialized(body, target, transport, label, postOtlp, signal); + await postSerialized(body, target, transport, label, signal); }; return send; @@ -446,7 +445,7 @@ export function createMetricExporter(target: OtlpTarget, transport: Transport): : undefined; if (body !== undefined && body.byteLength > 0) { - await postSerialized(body, target, transport, "metric", postOtlp, signal); + await postSerialized(body, target, transport, "metric", signal); } }, resultCallback); }, diff --git a/packages/otel/tests/otel-byo.test.ts b/packages/otel/tests/otel-byo.test.ts index 814d1b6..346d581 100644 --- a/packages/otel/tests/otel-byo.test.ts +++ b/packages/otel/tests/otel-byo.test.ts @@ -7,6 +7,7 @@ import { trace, TraceFlags, } from "@opentelemetry/api"; +import { ExportResultCode } from "@opentelemetry/core"; import { AlwaysOffSampler, AlwaysOnSampler, @@ -99,57 +100,54 @@ test("spanFilter narrows what gets exported", async () => { }); test("without an api key or exporter override the processor is a silent no-op", async () => { - const previous = process.env.TELEMETRY_DEV_API_KEY; - delete process.env.TELEMETRY_DEV_API_KEY; + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + const urls: string[] = []; + + const fetchImpl: typeof fetch = (input) => { + urls.push(input instanceof URL ? input.href : input instanceof Request ? input.url : input); + + return Promise.resolve(new Response(null, { status: 200 })); + }; + + for (const apiKey of [undefined, "td_live_x"]) { + const processor = new TelemetrySpanProcessor({ + apiKey, + baseUrl: "https://ingest.example", + fetch: fetchImpl, + }); - try { - const processor = new TelemetrySpanProcessor(); const provider = new BasicTracerProvider({ spanProcessors: [processor] }); - provider.getTracer("x").startSpan("lost").end(); + provider.getTracer("x").startSpan("span").end(); await processor.forceFlush(); await processor.shutdown(); - } finally { - if (previous !== undefined) process.env.TELEMETRY_DEV_API_KEY = previous; + expect(urls).toEqual(apiKey ? ["https://ingest.example/v1/traces"] : []); } }); -test("forceFlush and shutdown resolve", async () => { - const { processor } = byoSetup(); - await expect(processor.forceFlush()).resolves.toBeUndefined(); - await expect(processor.shutdown()).resolves.toBeUndefined(); -}); - test("createTelemetrySpanExporter is a no-op without a key and real otherwise", async () => { - const previous = process.env.TELEMETRY_DEV_API_KEY; - delete process.env.TELEMETRY_DEV_API_KEY; - - try { - const noop = createTelemetrySpanExporter(); - - const code = await new Promise((resolve) => - noop.export([], (result) => resolve(result.code)), - ); + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + const calls: Array<{ url: string; method?: string }> = []; - expect(code).toBe(0); - await noop.shutdown(); - - const calls: Array<{ url: string; method?: string }> = []; + const fetchImpl: typeof fetch = (input, init) => { + calls.push({ + url: input instanceof URL ? input.href : input instanceof Request ? input.url : input, + method: init?.method, + }); - const fetchImpl: typeof fetch = (input, init) => { - calls.push({ - url: input instanceof URL ? input.href : input instanceof Request ? input.url : input, - method: init?.method, - }); - - return Promise.resolve(new Response(null, { status: 200 })); - }; + return Promise.resolve(new Response(null, { status: 200 })); + }; + for (const apiKey of [undefined, "td_live_x"]) { const exporter = createTelemetrySpanExporter({ - apiKey: "td_live_x", + apiKey, baseUrl: "https://ingest.example/", fetch: fetchImpl, }); + expect( + await new Promise((resolve) => exporter.export([], (result) => resolve(result.code))), + ).toBe(ExportResultCode.SUCCESS); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)], }); @@ -157,12 +155,12 @@ test("createTelemetrySpanExporter is a no-op without a key and real otherwise", try { provider.getTracer("public-exporter").startSpan("exported").end(); await provider.forceFlush(); - expect(calls).toEqual([{ url: "https://ingest.example/v1/traces", method: "POST" }]); + expect(calls).toEqual( + apiKey ? [{ url: "https://ingest.example/v1/traces", method: "POST" }] : [], + ); } finally { await provider.shutdown(); } - } finally { - if (previous !== undefined) process.env.TELEMETRY_DEV_API_KEY = previous; } }); @@ -245,19 +243,6 @@ test("metrics are skipped without an api key even when enabled", async () => { } }); -test("global tracer interop: registered provider receives raw OTel API spans", async () => { - const { exporter, processor, provider } = byoSetup(); - trace.setGlobalTracerProvider(provider); - - try { - trace.getTracer("third-party").startSpan("via-global").end(); - await processor.forceFlush(); - expect(exporter.getFinishedSpans().map((s) => s.name)).toEqual(["via-global"]); - } finally { - trace.disable(); - } -}); - test("sha256 matches the FIPS vector", () => { const hex = Array.from(sha256(new TextEncoder().encode("abc")), (b) => b.toString(16).padStart(2, "0"), diff --git a/packages/otel/tests/transport.test.ts b/packages/otel/tests/transport.test.ts index 0e07496..a93e1ee 100644 --- a/packages/otel/tests/transport.test.ts +++ b/packages/otel/tests/transport.test.ts @@ -27,9 +27,7 @@ import { createMetricExporter, createLogExporter, createTraceExporter, - maybeGzip, otlpHeaders, - postOtlp, } from "../src/transport.ts"; interface Call { @@ -76,6 +74,8 @@ function makeSpans(count: number, attrBytes = 0): ReadableSpan[] { return exporter.getFinishedSpans(); } +const target = { url: "https://ingest.example/v1/telemetry", headers: otlpHeaders("td_live_x") }; + const exportSpans = (exporter: SpanExporter, spans: ReadableSpan[]): Promise => new Promise((resolve) => exporter.export(spans, (result) => resolve(result.code))); @@ -152,20 +152,6 @@ test("does not retry non-retryable statuses and reports the failure", async () = expect(errors).toHaveLength(1); }); -test("gives up after exhausting retries", async () => { - const { calls, fetchImpl } = makeFetch([503, 503, 503]); - - const res = await postOtlp({ - fetchImpl, - url: "https://ingest.example/v1/traces", - headers: {}, - body: new Uint8Array([1]), - }); - - expect(res.status).toBe(503); - expect(calls).toHaveLength(3); -}); - test("exhausted retries surface as a failed export with onError", async () => { const errors: unknown[] = []; const { calls, fetchImpl } = makeFetch([503, 503, 503]); @@ -204,47 +190,37 @@ describe("retry-after", () => { vi.useRealTimers(); }); - const startPost = (statuses: number[], headers: Record) => { + const startExport = (statuses: number[], headers: Record) => { const { calls, fetchImpl } = makeFetch(statuses, headers); - const pending = postOtlp({ - fetchImpl, - url: "https://ingest.example/v1/traces", - headers: {}, - body: new Uint8Array([1]), - }); - - return { calls, pending }; + return { + calls, + pending: exportSpans(createTraceExporter(target, { fetchImpl }), makeSpans(1)), + }; }; test("sets the delay before the next attempt", async () => { - const { calls, pending } = startPost([503, 200], { "retry-after": "1" }); + const { calls, pending } = startExport([503, 200], { "retry-after": "1" }); await vi.advanceTimersByTimeAsync(999); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test("waits when retry-after fits under the cap", async () => { - const { calls, pending } = startPost([503, 200], { "retry-after": "2" }); + const { calls, pending } = startExport([503, 200], { "retry-after": "2" }); await vi.advanceTimersByTimeAsync(1999); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); - }); - - test("gives up at once when retry-after exceeds the cap", async () => { - const { calls, pending } = startPost([429, 200], { "retry-after": "60" }); - expect((await pending).status).toBe(429); - expect(calls).toHaveLength(1); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test("an HTTP date sets the delay relative to now", async () => { vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); - const { calls, pending } = startPost([503, 200], { + const { calls, pending } = startExport([503, 200], { "retry-after": new Date(Date.now() + 1000).toUTCString(), }); @@ -252,27 +228,27 @@ describe("retry-after", () => { expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test.each(["Thursday, 01-Jan-26 00:00:01 GMT", "Thu Jan 1 00:00:01 2026"])( "accepts obsolete HTTP date %s", async (retryAfter) => { vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); - const { calls, pending } = startPost([503, 200], { "retry-after": retryAfter }); + const { calls, pending } = startExport([503, 200], { "retry-after": retryAfter }); await vi.advanceTimersByTimeAsync(999); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }, ); test("falls back to the fixed schedule when unparseable", async () => { - const { calls, pending } = startPost([503, 200], { "retry-after": "soon" }); + const { calls, pending } = startExport([503, 200], { "retry-after": "soon" }); await vi.advanceTimersByTimeAsync(100); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test.each([ @@ -288,35 +264,17 @@ describe("retry-after", () => { "January 1, 2026 00:00:01 GMT", ])("falls back to the fixed schedule for invalid value %s", async (retryAfter) => { vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); - const { calls, pending } = startPost([503, 200], { "retry-after": retryAfter }); + const { calls, pending } = startExport([503, 200], { "retry-after": retryAfter }); await vi.advanceTimersByTimeAsync(99); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); }); -test("a missing CompressionStream ships large bodies uncompressed", async () => { - vi.stubGlobal("CompressionStream", undefined); - - try { - const original = new TextEncoder().encode("a".repeat(5000)); - const { body, contentEncoding } = await maybeGzip(original); - expect(contentEncoding).toBeUndefined(); - expect(body).toBe(original); - } finally { - vi.unstubAllGlobals(); - } -}); - -test("bodies above the threshold are gzipped and round-trip", async () => { - const original = new TextEncoder().encode("a".repeat(5000)); - const { body, contentEncoding } = await maybeGzip(original); - expect(contentEncoding).toBe("gzip"); - expect(body.byteLength).toBeLessThan(original.byteLength); - - const decompressed = new Uint8Array( +const gunzip = async (body: Uint8Array) => + new Uint8Array( await new Response( new Blob([body as Uint8Array]) .stream() @@ -324,27 +282,37 @@ test("bodies above the threshold are gzipped and round-trip", async () => { ).arrayBuffer(), ); - expect(decompressed).toEqual(original); -}); - -test("small bodies are not compressed", async () => { - const original = new TextEncoder().encode("tiny"); - const { body, contentEncoding } = await maybeGzip(original); - expect(contentEncoding).toBeUndefined(); - expect(body).toBe(original); -}); - -test("exported batches send a content-encoding gzip header for large payloads", async () => { - const { calls, fetchImpl } = makeFetch([200]); - - const exporter = createTraceExporter( - { url: "https://ingest.example/v1/traces", headers: otlpHeaders("td_live_x") }, - { fetchImpl }, - ); +test.each([ + { name: "large", attrBytes: 10_000, compressionStream: true, encoding: "gzip" }, + { name: "small", attrBytes: 0, compressionStream: true, encoding: undefined }, + { + name: "large without CompressionStream", + attrBytes: 10_000, + compressionStream: false, + encoding: undefined, + }, +])( + "$name batches post the serialized request with content-encoding $encoding", + async ({ attrBytes, compressionStream, encoding }) => { + if (!compressionStream) vi.stubGlobal("CompressionStream", undefined); - await exportSpans(exporter, makeSpans(1, 10_000)); - expect(calls[0]!.headers["content-encoding"]).toBe("gzip"); -}); + try { + const { calls, fetchImpl } = makeFetch([200]); + const spans = makeSpans(1, attrBytes); + const serialized = ProtobufTraceSerializer.serializeRequest(spans)!; + expect(await exportSpans(createTraceExporter(target, { fetchImpl }), spans)).toBe( + ExportResultCode.SUCCESS, + ); + expect(calls).toHaveLength(1); + const { headers, body } = calls[0]!; + expect(headers["content-encoding"]).toBe(encoding); + expect(body.byteLength < serialized.byteLength).toBe(encoding === "gzip"); + expect(encoding === "gzip" ? await gunzip(body) : body).toEqual(serialized); + } finally { + vi.unstubAllGlobals(); + } + }, +); test("oversized batches split into multiple POSTs", async () => { const { calls, fetchImpl } = makeFetch([200]); @@ -423,8 +391,6 @@ test("metric exporter exhausts retries for a network failure", async () => { expect(String(errors[0])).toContain("connection reset"); }); -const target = { url: "https://ingest.example/v1/telemetry", headers: otlpHeaders("td_live_x") }; - const lifecycleCases = [ { name: "trace", diff --git a/packages/pi/tests/pi.test.ts b/packages/pi/tests/pi.test.ts index 88f4803..b957b0f 100644 --- a/packages/pi/tests/pi.test.ts +++ b/packages/pi/tests/pi.test.ts @@ -580,7 +580,7 @@ test("failed assistant message marks the chat and agent spans as errors", async expect(agent.attributes["error.type"]).toBe("error"); }); -test("tool executions produce successful and failed execute_tool spans", async () => { +test("a successful tool execution produces an execute_tool span with arguments and result", async () => { const harness = makeHarness(); await harness.emit({ type: "agent_start" }); await harness.emit({ @@ -596,38 +596,22 @@ test("tool executions produce successful and failed execute_tool spans", async ( result: { content: [{ type: "text", text: "README.md" }] }, isError: false, }); - await harness.emit({ - type: "tool_execution_start", - toolCallId: "call-2", - toolName: "bash", - args: { command: "false" }, - }); - await harness.emit({ - type: "tool_execution_end", - toolCallId: "call-2", - toolName: "bash", - result: { content: [{ type: "text", text: "command failed" }] }, - isError: true, - }); await harness.emit({ type: "agent_end", messages: [] }); await harness.emit({ type: "agent_settled" }); await flush(); - const spans = harness.exporters.spans - .getFinishedSpans() - .filter((span) => span.name === "execute_tool bash"); - - expect(spans).toHaveLength(2); - expect(spans[0]?.attributes).toMatchObject({ + const span = spanByName(harness.exporters, "execute_tool bash"); + const agent = spanByName(harness.exporters, "invoke_agent"); + expect(span.parentSpanContext?.spanId).toBe(agent.spanContext().spanId); + expect(span.attributes).toMatchObject({ "gen_ai.conversation.id": "session-1", "gen_ai.operation.name": "execute_tool", "gen_ai.tool.call.arguments": JSON.stringify({ command: "ls" }), "gen_ai.tool.call.id": "call-1", + "gen_ai.tool.call.result": JSON.stringify({ content: [{ type: "text", text: "README.md" }] }), "gen_ai.tool.name": "bash", }); - expect(spans[0]?.status.code).not.toBe(SpanStatusCode.ERROR); - expect(spans[1]?.status.code).toBe(SpanStatusCode.ERROR); - expect(spans[1]?.attributes["error.type"]).toBe("ToolExecutionError"); + expect(span.status.code).not.toBe(SpanStatusCode.ERROR); }); test.each([ diff --git a/packages/sdk/tests/client.test.ts b/packages/sdk/tests/client.test.ts index 1b0e75b..095b4b8 100644 --- a/packages/sdk/tests/client.test.ts +++ b/packages/sdk/tests/client.test.ts @@ -36,7 +36,7 @@ test("enabled:false yields a no-op client even with a key", () => { const spans = new InMemorySpanExporter(); const client = init( - { apiKey: "td_live_test", enabled: false, logLevel: "silent" }, + { apiKey: "td_live_test", enabled: false, exportMode: "immediate", logLevel: "silent" }, { spanExporter: spans }, ); diff --git a/packages/sdk/tests/metrics.test.ts b/packages/sdk/tests/metrics.test.ts index d2266a2..7d70f48 100644 --- a/packages/sdk/tests/metrics.test.ts +++ b/packages/sdk/tests/metrics.test.ts @@ -1,8 +1,4 @@ -import { - AggregationTemporality, - DataPointType, - type HistogramMetricData, -} from "@opentelemetry/sdk-metrics"; +import { DataPointType, type HistogramMetricData } from "@opentelemetry/sdk-metrics"; import { afterEach, expect, test } from "vitest"; import { flush, shutdown, startSpan } from "../src/index.ts"; @@ -203,7 +199,7 @@ test("plain function spans record no metrics", async () => { expect(histogramPoints(batches, "gen_ai.client.token.usage")).toHaveLength(0); }); -test("histograms use DELTA temporality and the GenAI bucket boundaries", async () => { +test("histograms use the GenAI bucket boundaries", async () => { const { batches, exporter } = makeMetricCapture(); setup({}, { metricExporter: exporter }); startSpan("gen", { type: "generation", usage: { inputTokens: 10 } }).end(); @@ -214,7 +210,6 @@ test("histograms use DELTA temporality and the GenAI bucket boundaries", async ( .flatMap((sm) => sm.metrics) .find((m) => m.descriptor.name === "gen_ai.client.token.usage")!; - expect(metric.aggregationTemporality).toBe(AggregationTemporality.DELTA); expect(metric.dataPointType).toBe(DataPointType.HISTOGRAM); const point = metric.dataPoints[0]!; expect( @@ -246,15 +241,24 @@ test("filter-rejected spans record no metrics", async () => { expect(histogramPoints(batches, "gen_ai.client.operation.time_to_first_chunk")).toHaveLength(0); }); -test("quiet intervals collect zero data points", async () => { - const { batches, exporter } = makeMetricCapture(); - setup({}, { metricExporter: exporter }); - await flush(); +test("the metric exporter sends DELTA intervals: idle flushes post nothing and points are not resent", async () => { + const urls: string[] = []; - const pointCount = batches - .flatMap((rm) => rm.scopeMetrics) - .flatMap((sm) => sm.metrics) - .reduce((total, metric) => total + metric.dataPoints.length, 0); + setup({ + baseUrl: "https://ingest.test", + fetch: async (input) => { + urls.push(new Request(input).url); + + return new Response(null, { status: 200 }); + }, + }); - expect(pointCount).toBe(0); + await flush(); + expect(urls).toEqual([]); + startSpan("gen", { type: "generation", usage: { inputTokens: 10 } }).end(); + await flush(); + expect(urls).toEqual(["https://ingest.test/v1/metrics"]); + // A cumulative exporter would resend the duration and token series on every collection. + await flush(); + expect(urls).toEqual(["https://ingest.test/v1/metrics"]); }); diff --git a/packages/sdk/tests/process-session.test.ts b/packages/sdk/tests/process-session.test.ts index 398812e..4b2d562 100644 --- a/packages/sdk/tests/process-session.test.ts +++ b/packages/sdk/tests/process-session.test.ts @@ -59,6 +59,8 @@ test("explicit sessions win and remain consistent on nested spans and logs", asy }, ); await flush(); + expect(spans.getFinishedSpans()).toHaveLength(6); + expect(logs.getFinishedLogRecords()).toHaveLength(3); for (const span of spans.getFinishedSpans()) { expect(span.attributes["gen_ai.conversation.id"]).toBe(span.name.split("-")[0]); diff --git a/packages/sdk/tests/span.test.ts b/packages/sdk/tests/span.test.ts index 76b2e1f..b569db5 100644 --- a/packages/sdk/tests/span.test.ts +++ b/packages/sdk/tests/span.test.ts @@ -387,16 +387,6 @@ test("updateActiveSpan maps fields using the active span's type", async () => { expect(span.attributes["gen_ai.usage.output_tokens"]).toBe(9); }); -test("spanLimits backstop truncates raw attributes set outside the capture funnel", async () => { - const { spans } = setup(); - const handle = startSpan("raw"); - handle.span.setAttribute("raw.dump", "x".repeat(70000)); - handle.end(); - await flush(); - const value = String(spans.getFinishedSpans()[0]!.attributes["raw.dump"]); - expect(value.length).toBe(65536); -}); - test("per-call captureInput:false drops input but keeps output", async () => { const { spans } = setup(); startSpan("private", { diff --git a/packages/tanstack-ai/tests/evaluate.test.ts b/packages/tanstack-ai/tests/evaluate.test.ts index 589b297..5982e05 100644 --- a/packages/tanstack-ai/tests/evaluate.test.ts +++ b/packages/tanstack-ai/tests/evaluate.test.ts @@ -67,50 +67,38 @@ test("decide emits isolated evaluation spans and one set of usage metrics for co const spans = spanBatches.flat(); expect(spans).toHaveLength(2); expect(spans.every((span) => span.kind === SpanKind.CLIENT)).toBe(true); - expect( - spans - .map((span) => String(span.attributes["gen_ai.provider.name"])) - .sort((a, b) => a.localeCompare(b)), - ).toEqual(["first", "second"]); - expect( - spans - .map((span) => String(span.attributes["gen_ai.request.model"])) - .sort((a, b) => a.localeCompare(b)), - ).toEqual(["first-model", "second-model"]); - expect( - spans.map((span) => Number(span.attributes["gen_ai.usage.input_tokens"])).sort((a, b) => a - b), - ).toEqual([7, 11]); - expect(spans.every((span) => span.attributes["gen_ai.output.messages"] === undefined)).toBe(true); - expect(metrics.filter((metric) => metric.type === "duration")).toHaveLength(2); - expect(metrics.filter((metric) => metric.type === "input")).toHaveLength(2); - expect(metrics.filter((metric) => metric.type === "output")).toHaveLength(2); -}); -test.each([ - { provider: "anthropic", input: 10 + 3 + 2 }, - { provider: "openrouter", input: 10 }, -])( - "decide from $provider records input tokens with cache reads and writes once", - async ({ provider, input }) => { - const { middleware, metrics, spanBatches } = capture(); - - const judge = adapter(provider, async () => ({ - ...answer("judge-model", 10, 2), - usage: { - promptTokens: 10, - completionTokens: 2, - totalTokens: 12, - promptTokensDetails: { cachedTokens: 3, cacheWriteTokens: 2 }, - }, - })); + const byProvider = (provider: string) => + spans.find((span) => span.attributes["gen_ai.provider.name"] === provider)!.attributes; - await decide({ adapter: judge, state: "one", questions, middleware: [middleware] }); + expect(byProvider("first")).toMatchObject({ + "gen_ai.request.model": "first-model", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 3, + }); + expect(byProvider("second")).toMatchObject({ + "gen_ai.request.model": "second-model", + "gen_ai.usage.input_tokens": 7, + "gen_ai.usage.output_tokens": 2, + }); + expect(spans.every((span) => span.attributes["gen_ai.output.messages"] === undefined)).toBe(true); - const [span] = spanBatches.flat(); - expect(span?.attributes["gen_ai.usage.input_tokens"]).toBe(input); - expect(metrics.find((metric) => metric.type === "input")?.value).toBe(input); - }, -); + const tokens = (provider: string) => + metrics + .filter((metric) => metric.attributes["gen_ai.provider.name"] === provider) + .map((metric) => [metric.type, metric.value]); + + expect(tokens("first")).toEqual([ + ["duration", expect.any(Number)], + ["input", 11], + ["output", 3], + ]); + expect(tokens("second")).toEqual([ + ["duration", expect.any(Number)], + ["input", 7], + ["output", 2], + ]); +}); test.each([ { error: new TypeError("provider failed"), type: "TypeError", message: "provider failed" }, diff --git a/packages/tanstack-ai/tests/generation.test.ts b/packages/tanstack-ai/tests/generation.test.ts index b5b4ebd..4da2238 100644 --- a/packages/tanstack-ai/tests/generation.test.ts +++ b/packages/tanstack-ai/tests/generation.test.ts @@ -65,7 +65,7 @@ test.each([ id: "e1", model: "text-embedding-3-small", embeddings: [{ index: 0, vector: [0.1, 0.2] }], - usage: usage(8, 0), + usage: usage(8, 0, { promptTokensDetails: { cachedTokens: 4, cacheWriteTokens: 2 } }), }), }), input: "hello", @@ -77,8 +77,10 @@ test.each([ operation: "embeddings", outputType: undefined, responseModel: undefined, + // OpenAI reports promptTokens with cache reads and writes; the span total adds nothing. input: 8, output: 0, + modality: {}, }, { activity: "image", @@ -106,6 +108,11 @@ test.each([ responseModel: "gpt-image-1-2025", input: 12, output: 40, + modality: { + "gen_ai.usage.text.input_tokens": 10, + "gen_ai.usage.image.input_tokens": 2, + "gen_ai.usage.image.output_tokens": 40, + }, }, { activity: "tts", @@ -131,6 +138,7 @@ test.each([ responseModel: "eleven-v3", input: 5, output: 30, + modality: { "gen_ai.usage.audio.output_tokens": 30 }, }, { activity: "transcription", @@ -155,6 +163,7 @@ test.each([ responseModel: "whisper-1", input: 20, output: 4, + modality: { "gen_ai.usage.audio.input_tokens": 20 }, }, { activity: "summarize", @@ -165,7 +174,9 @@ test.each([ id: "m1", model: "claude-resolved", summary: "secret summary", - usage: usage(100, 10, { promptTokensDetails: { cachedTokens: 30 } }), + usage: usage(100, 10, { + promptTokensDetails: { cachedTokens: 30, cacheWriteTokens: 5 }, + }), }), }), text: "long secret article", @@ -177,9 +188,10 @@ test.each([ operation: "chat", outputType: "text", responseModel: "claude-resolved", - // Anthropic reports promptTokens without cache reads; the span total includes them. - input: 130, + // Anthropic reports promptTokens without cache reads and writes; the span total includes them. + input: 135, output: 10, + modality: {}, }, ])( "$activity emits one $operation span with provider, model, and usage", @@ -193,6 +205,7 @@ test.each([ responseModel, input, output, + modality, }) => { const { middleware, spans, metrics, errors } = capture(); @@ -212,6 +225,13 @@ test.each([ expect(span!.attributes["gen_ai.response.model"]).toBe(responseModel); expect(span!.attributes["gen_ai.usage.input_tokens"]).toBe(input); expect(span!.attributes["gen_ai.usage.output_tokens"]).toBe(output); + expect( + Object.fromEntries( + Object.entries(span!.attributes).filter(([key]) => + /^gen_ai\.usage\.(text|image|audio)\./.test(key), + ), + ), + ).toEqual(modality); // Inputs and outputs (prompts, text, audio, image bytes) are never captured. const values = JSON.stringify(Object.values(span!.attributes)); @@ -229,33 +249,6 @@ test.each([ }, ); -test("media usage records provider-reported modality token breakdowns", async () => { - const { middleware, spans } = capture(); - - await generateImage({ - adapter: adapter("image", "openai", { - generateImages: async () => ({ - id: "i1", - model: "gpt-image-1", - images: [{ url: "https://example.com/i.png" }], - usage: usage(12, 40, { - promptTokensDetails: { textTokens: 10, imageTokens: 2 }, - completionTokensDetails: { imageTokens: 40 }, - }), - }), - }), - prompt: "p", - middleware: [middleware], - }); - - expect(spans[0]!.attributes).toMatchObject({ - "gen_ai.usage.text.input_tokens": 10, - "gen_ai.usage.image.input_tokens": 2, - "gen_ai.usage.image.output_tokens": 40, - }); - expect(spans[0]!.attributes["gen_ai.usage.audio.input_tokens"]).toBeUndefined(); -}); - test("a generation without reported usage omits token attributes and metrics", async () => { const { middleware, spans, metrics } = capture(); diff --git a/packages/tanstack-ai/tests/middleware.test.ts b/packages/tanstack-ai/tests/middleware.test.ts index a8ef6bf..20e9a99 100644 --- a/packages/tanstack-ai/tests/middleware.test.ts +++ b/packages/tanstack-ai/tests/middleware.test.ts @@ -8,9 +8,9 @@ import { SamplingDecision, TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; -import { type ChatMiddlewareContext } from "@tanstack/ai"; +import { chat, type ChatMiddlewareContext, embed, type EmbeddingAdapter } from "@tanstack/ai"; import { sessionSpanContext } from "@telemetry-dev/otel"; -import { expect, onTestFinished, test, vi } from "vitest"; +import { expect, test, vi } from "vitest"; import { telemetryDev } from "../src/index.ts"; @@ -313,48 +313,34 @@ test.each([ } }); -test("bigint metadata IDs populate user and conversation attributes", async () => { - const { spanBatches, overrides } = makeCapture(); - const mw = middleware(overrides); - const ctx = makeCtx({ options: { metadata: { userId: 2n, sessionId: 9n } } }); - - await mw.onStart?.(ctx); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); - - const spans = spanBatches[0]!; - - for (const s of spans) { - expect(s.attributes["gen_ai.conversation.id"]).toBe("9"); - } - - const root = spans.find((s) => s.kind === SpanKind.INTERNAL)!; - expect(root.attributes["user.id"]).toBe("2"); - // Reserved keys never leak into td.metadata.*. - expect(root.attributes["td.metadata.userId"]).toBeUndefined(); - expect(root.attributes["td.metadata.sessionId"]).toBeUndefined(); -}); - -test("string metadata IDs override threadId and populate user attributes", async () => { - const { spanBatches, overrides } = makeCapture(); - const mw = middleware(overrides); - const ctx = makeCtx({ options: { metadata: { userId: "u2", sessionId: "sess_9" } } }); +test.each([ + { kind: "bigint", userId: 2n, sessionId: 9n, user: "2", conversation: "9" }, + { kind: "string", userId: "u2", sessionId: "sess_9", user: "u2", conversation: "sess_9" }, +])( + "$kind metadata IDs override threadId and populate user and conversation attributes", + async ({ userId, sessionId, user, conversation }) => { + const { spanBatches, overrides } = makeCapture(); + const mw = middleware(overrides); + const ctx = makeCtx({ options: { metadata: { userId, sessionId } } }); - await mw.onStart?.(ctx); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); + await mw.onStart?.(ctx); + await mw.onConfig?.(ctx, chatConfig()); + await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); + await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); - const spans = spanBatches[0]!; + const spans = spanBatches[0]!; - for (const span of spans) { - expect(span.attributes["gen_ai.conversation.id"]).toBe("sess_9"); - } + for (const s of spans) { + expect(s.attributes["gen_ai.conversation.id"]).toBe(conversation); + } - const root = spans.find((span) => span.kind === SpanKind.INTERNAL)!; - expect(root.attributes["user.id"]).toBe("u2"); -}); + const root = spans.find((s) => s.kind === SpanKind.INTERNAL)!; + expect(root.attributes["user.id"]).toBe(user); + // Reserved keys never leak into td.metadata.*. + expect(root.attributes["td.metadata.userId"]).toBeUndefined(); + expect(root.attributes["td.metadata.sessionId"]).toBeUndefined(); + }, +); test("multi-iteration run with a tool yields invoke_agent root and parents the tool to its iteration", async () => { const { spanBatches, metrics, overrides } = makeCapture(); @@ -559,23 +545,53 @@ test("onAbort closes everything as cancelled", async () => { expect(tool.attributes["error.type"]).toBe("cancelled"); }); -test("no apiKey is a complete no-op (transport never invoked)", async () => { +test("no apiKey makes chat and generation hooks a complete no-op (transport never invoked)", async () => { // The middleware falls back to this variable, which a developer's shell can export. vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); - onTestFinished(() => { - vi.unstubAllEnvs(); - }); - const { spanBatches, metrics, overrides } = makeCapture(); - const mw = telemetryDev({ environment: "test", serviceName: "svc" }, overrides); - const ctx = makeCtx(); - await mw.onStart?.(ctx); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "y" } as never); + try { + let calls = 0; - expect(spanBatches).toHaveLength(0); - expect(metrics).toHaveLength(0); + const mw = telemetryDev({ + environment: "test", + serviceName: "svc", + fetch: async () => { + calls += 1; + + return new Response(null, { status: 200 }); + }, + }); + + const ctx = makeCtx(); + + await mw.onStart?.(ctx); + await mw.onConfig?.(ctx, chatConfig()); + await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); + await mw.onUsage?.(ctx, { promptTokens: 1, completionTokens: 1, totalTokens: 2 } as never); + await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "y" } as never); + + const result = await embed({ + adapter: { + kind: "embedding", + name: "openai", + model: "text-embedding-3-small", + "~types": {}, + createEmbeddings: async () => ({ + id: "e1", + model: "text-embedding-3-small", + embeddings: [{ index: 0, vector: [0.1] }], + usage: { promptTokens: 1, completionTokens: 0, totalTokens: 1 }, + }), + } as unknown as EmbeddingAdapter, + input: "x", + middleware: [mw as never], + }); + + expect(result).toMatchObject({ id: "e1" }); + expect(calls).toBe(0); + } finally { + vi.unstubAllEnvs(); + } }); test("one middleware instance handles concurrent chats independently", async () => { @@ -720,22 +736,20 @@ test.each(["finish", "error", "abort"])( }, ); -test("metric export retries a transient ingest failure", async () => { +test("chat telemetry reaches the configured ingest through the configured fetch", async () => { const errors: unknown[] = []; - let metricCalls = 0; + const posts: Array<{ url: string; authorization: string | null }> = []; const mw = telemetryDev({ - apiKey: "td_live_metric_retry", + apiKey: "td_live_wiring", + baseUrl: "https://ingest.example.test/", environment: "test", - serviceName: "metric-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - const href = url instanceof Request ? url.url : url.toString(); - - if (href.endsWith("/v1/metrics")) { - metricCalls += 1; - - return new Response(null, { status: metricCalls === 1 ? 503 : 200 }); - } + serviceName: "wiring", + fetch: async (url: string | Request | URL, init?: RequestInit) => { + posts.push({ + url: url instanceof Request ? url.url : url.toString(), + authorization: new Headers(init?.headers).get("authorization"), + }); return new Response(null, { status: 200 }); }, @@ -752,8 +766,11 @@ test("metric export retries a transient ingest failure", async () => { await mw.onUsage?.(ctx, { promptTokens: 10, completionTokens: 5, totalTokens: 15 } as never); await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); - expect(metricCalls).toBe(2); - expect(errors).toHaveLength(0); + expect(posts).toEqual([ + { url: "https://ingest.example.test/v1/traces", authorization: "Bearer td_live_wiring" }, + { url: "https://ingest.example.test/v1/metrics", authorization: "Bearer td_live_wiring" }, + ]); + expect(errors).toEqual([]); }); test("Ollama-style nested sampling options populate gen_ai.request attrs", async () => { @@ -799,23 +816,56 @@ test("an iteration with no usage records no token attributes or metrics (not zer expect(String(summary?.attributes?.["log.message"])).not.toContain("tokens"); }); -test("onConfig during init opens no iteration span", async () => { - const { spanBatches, overrides } = makeCapture(); - const mw = middleware(overrides); - const ctx = makeCtx(); +// The other chat tests script the hook order; this one pins it to the real engine, which fires +// onConfig for phase "init" before onStart and again before each model call. +test("a real chat() run opens one iteration span per model call", async () => { + const { spanBatches, metrics, overrides } = makeCapture(); + const timestamp = Date.now(); - await mw.onStart?.(ctx); - // The engine may fire onConfig at init before any model call; init must not open an iteration span. - Object.assign(ctx, { phase: "init" }); - await mw.onConfig?.(ctx, chatConfig()); - Object.assign(ctx, { phase: "beforeModel" }); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); + const adapter = { + kind: "text", + name: "openai", + model: "gpt-4o", + "~types": {}, + async *chatStream() { + yield { type: "RUN_STARTED", runId: "run_real", threadId: "thread_real", timestamp }; + yield { type: "TEXT_MESSAGE_START", messageId: "m1", role: "assistant", timestamp }; + yield { type: "TEXT_MESSAGE_CONTENT", messageId: "m1", delta: "hello", timestamp }; + yield { type: "TEXT_MESSAGE_END", messageId: "m1", timestamp }; + yield { + type: "RUN_FINISHED", + runId: "run_real", + threadId: "thread_real", + finishReason: "stop", + model: "gpt-4o-2024-11-20", + usage: { promptTokens: 3, completionTokens: 2, totalTokens: 5 }, + timestamp, + }; + }, + structuredOutput: async () => ({ data: {}, rawText: "{}" }), + }; + + const stream = chat({ + adapter: adapter as never, + messages: [{ role: "user", content: "hi" }], + middleware: [middleware(overrides)], + }); + + for await (const _chunk of stream); + expect(spanBatches).toHaveLength(1); const spans = spanBatches[0]!; - expect(spans).toHaveLength(2); - expect(spans.filter((s) => s.kind === SpanKind.CLIENT)).toHaveLength(1); + expect(spans.filter((s) => s.kind === SpanKind.INTERNAL)).toHaveLength(1); + const iterations = spans.filter((s) => s.kind === SpanKind.CLIENT); + expect(iterations).toHaveLength(1); + const iteration = iterations[0]!; + expect(JSON.parse(String(iteration.attributes["gen_ai.input.messages"]))).toEqual([ + { role: "user", content: "hi" }, + ]); + expect(iteration.attributes["gen_ai.output.messages"]).toBe("hello"); + expect(iteration.attributes["gen_ai.usage.input_tokens"]).toBe(3); + expect(iteration.attributes["gen_ai.response.model"]).toBe("gpt-4o-2024-11-20"); + expect(metrics.filter((m) => m.metric === "duration")).toHaveLength(1); }); test("structured output finalization records a separate iteration span", async () => { diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index d33efd0..c09fc39 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -21,8 +21,6 @@ __version__ = "0.1.6" ProviderResolver = Callable[[object | None], str] -RequestMapper = Callable[[Mapping[str, Any]], tuple[str, dict[str, Any]]] -ResponseMapper = Callable[[Any], dict[str, Any]] _WRAPPED_ATTR = "_telemetry_dev_anthropic_wrapped" _ORIGINAL_ATTR = "_telemetry_dev_anthropic_original" @@ -141,7 +139,7 @@ def _raw_response_requested(params: Mapping[str, Any]) -> bool: def _native(value: Any) -> Any: model_dump = getattr(value, "model_dump", None) if callable(model_dump): - return model_dump(mode="json", exclude_none=True) + return model_dump(mode="json", by_alias=True, exclude_none=True) if isinstance(value, Mapping): mapping = cast(Mapping[Any, Any], value) return {str(key): _native(item) for key, item in mapping.items() if item is not None} @@ -457,13 +455,8 @@ def end(**fields: Any) -> None: class _StreamState: - def __init__( - self, - capture_output: bool = True, - expect_message_stop: bool = False, - ) -> None: + def __init__(self, capture_output: bool = True) -> None: self.capture_output = capture_output - self.expect_message_stop = expect_message_stop self.message_stopped = False self.blocks: dict[int, dict[str, Any]] = {} self.tool_json: dict[int, str] = {} @@ -582,15 +575,28 @@ def _bounded_stream_native( remaining_items, depth, seen, + _model_fields(value), ) +def _model_fields(value: Any) -> Mapping[str, Any]: + # Pydantic keeps an aliased field under its Python name (`from_` for the wire key + # `from`). Read the class's field table directly, so that no property runs. + for klass in _TYPE_MRO.__get__(type(cast(object, value))): + namespace = _TYPE_NAMESPACE.__get__(klass) + fields = namespace.get("__pydantic_fields__", namespace.get("model_fields")) + if type(fields) is dict: + return cast(dict[str, Any], fields) + return {} + + def _bounded_stream_mappings( mappings: tuple[Mapping[Any, Any], ...], source_id: int, remaining_items: list[int], depth: int, seen: set[int], + fields: Mapping[str, Any] | None = None, ) -> dict[str, Any]: if sum(len(mapping) for mapping in mappings) * 2 > remaining_items[0] or source_id in seen: raise _StreamCaptureLimit @@ -603,6 +609,10 @@ def _bounded_stream_mappings( if remaining_items[0] < 2: raise _StreamCaptureLimit remaining_items[0] -= 1 + if fields: + alias = _stored(fields.get(key), "alias") + if type(alias) is str: + key = alias if key in seen_keys: remaining_items[0] -= 1 continue @@ -712,7 +722,7 @@ def _stream_partial( output_incomplete = bool( state.budget.truncated or output_truncated - or (state.expect_message_stop and not state.message_stopped) + or not state.message_stopped or state.unresolved_replacements or state.unresolved_encrypted_content ) @@ -1214,27 +1224,9 @@ def _stream_event_has_output(event: Any) -> bool: return input_value not in (None, "", [], {}) -def _stream_span_capture_policy(handle: telemetry_dev.SpanHandle) -> tuple[bool, bool]: - compatible = cast(Any, handle) - client = getattr(compatible, "_client", None) - state = getattr(compatible, "_state", None) - capture_output = getattr(compatible, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(state, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(client, "capture_output", False) - capture_masked = getattr(compatible, "capture_masked", None) - if not isinstance(capture_masked, bool): - capture_masked = getattr(client, "mask", _OMIT) is not None if client is not None else True - return capture_output, capture_masked - - def _stream_span_reporter( handle: telemetry_dev.SpanHandle, ) -> Callable[[BaseException], None]: - compatible = cast(Any, handle) - reporter = getattr(compatible, "report_error", None) - client = getattr(compatible, "_client", None) reported = False def report(cause: BaseException) -> None: @@ -1243,12 +1235,7 @@ def report(cause: BaseException) -> None: return reported = True try: - if callable(reporter): - reporter(cause) - return - client_report = getattr(client, "report", None) - if callable(client_report): - client_report("provider instrumentation failed", cause) + handle.report_error(cause) except Exception: return @@ -1256,13 +1243,10 @@ def report(cause: BaseException) -> None: def _end_mapped_response( - handle: telemetry_dev.SpanHandle, - end: Callable[..., None], - mapper: ResponseMapper, - response: Any, + handle: telemetry_dev.SpanHandle, end: Callable[..., None], response: Any ) -> None: try: - fields = mapper(response) + fields = _messages_response(response) except Exception as exc: _stream_span_reporter(handle)(exc) end(attributes={"telemetry.dev.capture.truncated": True}) @@ -1276,8 +1260,8 @@ def __init__(self, inner: Any, handle: telemetry_dev.SpanHandle, started_at: flo self._handle = handle self._end = _end_once(handle) self._started_at = started_at - capture_output, self._mask_output_when_incomplete = _stream_span_capture_policy(handle) - self._state = _StreamState(capture_output, expect_message_stop=True) + self._mask_output_when_incomplete = handle.capture_masked + self._state = _StreamState(handle.capture_output) self._report_error = _stream_span_reporter(handle) self._saw_first = False self._consume: Iterator[Any] | None = None @@ -1320,9 +1304,7 @@ def _iterate(self) -> Iterator[Any]: update = {} has_output = False if has_output: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) self._update_first(update) yield event finally: @@ -1365,8 +1347,8 @@ def __init__(self, inner: Any, handle: telemetry_dev.SpanHandle, started_at: flo self._handle = handle self._end = _end_once(handle) self._started_at = started_at - capture_output, self._mask_output_when_incomplete = _stream_span_capture_policy(handle) - self._state = _StreamState(capture_output, expect_message_stop=True) + self._mask_output_when_incomplete = handle.capture_masked + self._state = _StreamState(handle.capture_output) self._report_error = _stream_span_reporter(handle) self._saw_first = False self._consume: AsyncIterator[Any] | None = None @@ -1409,9 +1391,7 @@ async def _aiterate(self) -> AsyncIterator[Any]: update = {} has_output = False if has_output: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) self._update_first(update) yield event finally: @@ -1535,19 +1515,14 @@ def __getattr__(self, name: str) -> Any: def _start_span( - params: Mapping[str, Any], mapper: RequestMapper, provider: str + params: Mapping[str, Any], provider: str ) -> tuple[telemetry_dev.SpanHandle, Callable[..., None], float]: - name, fields = mapper(params) + name, fields = _messages_request(params) handle = telemetry_dev.start_span(name, provider=provider, **_clean_fields(fields)) return handle, _end_once(handle), time.perf_counter() -def _wrap_sync( - original: Callable[..., Any], - request_mapper: RequestMapper, - response_mapper: ResponseMapper, - provider: ProviderResolver, -) -> Callable[..., Any]: +def _wrap_sync(original: Callable[..., Any], provider: ProviderResolver) -> Callable[..., Any]: @wraps(original) def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None @@ -1555,7 +1530,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, request_mapper, provider(resource)) + handle, end, started_at = _start_span(call_kwargs, provider(resource)) try: result = original(*args, **call_kwargs) except BaseException as exc: @@ -1563,7 +1538,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: raise if streaming: return _InstrumentedStream(result, handle, started_at) - _end_mapped_response(handle, end, response_mapper, result) + _end_mapped_response(handle, end, result) return result setattr(wrapper, _WRAPPED_ATTR, True) @@ -1571,12 +1546,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return wrapper -def _wrap_async( - original: Callable[..., Any], - request_mapper: RequestMapper, - response_mapper: ResponseMapper, - provider: ProviderResolver, -) -> Callable[..., Any]: +def _wrap_async(original: Callable[..., Any], provider: ProviderResolver) -> Callable[..., Any]: @wraps(original) async def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None @@ -1584,7 +1554,7 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return await original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, request_mapper, provider(resource)) + handle, end, started_at = _start_span(call_kwargs, provider(resource)) try: result = await original(*args, **call_kwargs) except BaseException as exc: @@ -1592,7 +1562,7 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: raise if streaming: return _InstrumentedAsyncStream(result, handle, started_at) - _end_mapped_response(handle, end, response_mapper, result) + _end_mapped_response(handle, end, result) return result setattr(wrapper, _WRAPPED_ATTR, True) @@ -1601,13 +1571,13 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: def _wrap_stream_manager_sync( - original: Callable[..., Any], request_mapper: RequestMapper, provider: ProviderResolver + original: Callable[..., Any], provider: ProviderResolver ) -> Callable[..., Any]: @wraps(original) def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) - name, fields = request_mapper(call_kwargs) + name, fields = _messages_request(call_kwargs) manager = original(*args, **call_kwargs) return _InstrumentedMessageStreamManager(manager, name, fields, provider(resource)) @@ -1617,13 +1587,13 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: def _wrap_stream_manager_async( - original: Callable[..., Any], request_mapper: RequestMapper, provider: ProviderResolver + original: Callable[..., Any], provider: ProviderResolver ) -> Callable[..., Any]: @wraps(original) def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) - name, fields = request_mapper(call_kwargs) + name, fields = _messages_request(call_kwargs) manager = original(*args, **call_kwargs) return _InstrumentedAsyncMessageStreamManager(manager, name, fields, provider(resource)) @@ -1651,9 +1621,9 @@ def _wrap_method( ) -> Callable[..., Any]: if name == "stream": factory = _wrap_stream_manager_async if async_resource else _wrap_stream_manager_sync - return factory(original, _messages_request, provider) - response_factory = _wrap_async if async_resource else _wrap_sync - return response_factory(original, _messages_request, _messages_response, provider) + else: + factory = _wrap_async if async_resource else _wrap_sync + return factory(original, provider) def _patch_instance_method( diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index ab8b724..26708f6 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -2,11 +2,9 @@ import importlib import importlib.util -import inspect import json import random import sys -import time import types from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence from types import SimpleNamespace @@ -25,7 +23,7 @@ from anthropic.types.beta import BetaMessageParam from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.trace import StatusCode -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field import telemetry_dev_anthropic from telemetry_dev_anthropic import instrument_anthropic, uninstrument_anthropic, wrap_anthropic @@ -62,42 +60,35 @@ async def test_response_mapping_failure_returns_provider_result_and_marks_captur ) -> None: errors: list[BaseException] = [] memory = make(on_error=errors.append) - mapping_error = RuntimeError("mapping failed") - provider_result = object() - def request_mapper(_params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: - return "chat claude-sonnet-4-6", { - "type": "generation", - "model": "claude-sonnet-4-6", - } - - def response_mapper(_response: Any) -> dict[str, Any]: - raise mapping_error + class MappingFailure: + @property + def content(self) -> Any: + raise RuntimeError("mapping failed") - def provider(_resource: object | None) -> str: - return "anthropic" + provider_result = MappingFailure() if async_mode: - async def async_original(**_kwargs: Any) -> object: + async def async_create(**_kwargs: Any) -> MappingFailure: return provider_result - wrapper = vars(telemetry_dev_anthropic)["_wrap_async"]( - async_original, request_mapper, response_mapper, provider - ) - result = await wrapper(model="claude-sonnet-4-6") + client: Any = object.__new__(AsyncAnthropic) + messages: Any = object.__new__(AsyncMessages) + messages.create = async_create + client.messages = messages + result = await wrap_anthropic(client).messages.create(model="claude-sonnet-4-6") else: - def sync_original(**_kwargs: Any) -> object: + def create(**_kwargs: Any) -> MappingFailure: return provider_result - wrapper = vars(telemetry_dev_anthropic)["_wrap_sync"]( - sync_original, request_mapper, response_mapper, provider - ) - result = wrapper(model="claude-sonnet-4-6") + client = object.__new__(Anthropic) + client.messages = SimpleNamespace(create=create) + result = wrap_anthropic(client).messages.create(model="claude-sonnet-4-6") assert result is provider_result - assert errors == [mapping_error] + assert [str(error) for error in errors] == ["mapping failed"] assert attrs(only_span(memory))["telemetry.dev.capture.truncated"] is True @@ -695,138 +686,88 @@ def handler(request: httpx.Request) -> httpx.Response: assert len(timestamps) == 2 -def test_stream_mapping_error_reports_and_still_yields_provider_event(make: Any) -> None: - errors: list[BaseException] = [] - memory = make(on_error=errors.append) - - def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(stream_events()) - - class BrokenDelta(Mapping[str, Any]): - def __len__(self) -> int: - return 1 - - def __iter__(self) -> Iterator[str]: - raise ValueError("mapping failed") - - def __getitem__(self, key: str) -> Any: - return key - - provider_event = SimpleNamespace( - type="content_block_delta", - index=0, - delta=BrokenDelta(), - ) - client = wrapped_sync_client(handler) - stream = client.messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True - ) - stream._inner = iter([provider_event, provider_event]) - - assert next(stream) is provider_event - assert next(stream) is provider_event - with pytest.raises(StopIteration): - next(stream) - client.close() - - assert len(errors) == 1 - assert isinstance(errors[0], ValueError) - assert str(errors[0]) == "mapping failed" - span = only_span(memory) - assert span.status.status_code == StatusCode.UNSET - assert attrs(span)["telemetry.dev.capture.truncated"] is True - - -def test_stream_capture_does_not_invoke_expanding_model_dump() -> None: - model_dump_calls = 0 - - class ExpandingDelta: - def __init__(self) -> None: - self.type = "text_delta" - self.text = "safe" - - def model_dump(self, **_kwargs: Any) -> dict[str, Any]: - nonlocal model_dump_calls - model_dump_calls += 1 - return {"type": "text_delta", "text": "safe", "items": list(range(5_000))} - - captured = telemetry_dev_anthropic._bounded_native(ExpandingDelta()) # pyright: ignore[reportPrivateUsage] - - assert captured == {"type": "text_delta", "text": "safe"} - assert model_dump_calls == 0 - - @pytest.mark.parametrize("async_mode", [False, True]) @pytest.mark.parametrize("capture_output", [False, True]) -async def test_stream_uses_bound_handle_capture_policy_and_reporter( - monkeypatch: pytest.MonkeyPatch, async_mode: bool, capture_output: bool +async def test_stream_mapping_error_reports_and_still_yields_provider_event( + make: Any, async_mode: bool, capture_output: bool ) -> None: errors: list[BaseException] = [] - global_reports: list[BaseException] = [] - ended: list[dict[str, Any]] = [] + memory = make(capture_output=capture_output, on_error=errors.append) class BrokenMessage: @property def id(self) -> str: raise RuntimeError("broken message") - def end(**fields: Any) -> None: - ended.append(fields) - - def update(**_fields: Any) -> None: - return None - - def report_global(_message: str, error: BaseException) -> None: - global_reports.append(error) - - handle = SimpleNamespace( - capture_output=capture_output, - capture_masked=True, - report_error=errors.append, - end=end, - update=update, - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(report=report_global), - ) + broken = {"type": "message_start", "message": BrokenMessage()} + # message_stop makes the stream complete, so only the mapping error can flag the span. events = [ { "type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": "sensitive"}, }, - {"type": "message_start", "message": BrokenMessage()}, + broken, + broken, + {"type": "message_stop"}, ] if async_mode: + async def handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(stream_events()) + async def source() -> AsyncIterator[Any]: for event in events: yield event - stream = telemetry_dev_anthropic._InstrumentedAsyncStream( # pyright: ignore[reportPrivateUsage] - source(), cast(Any, handle), time.perf_counter() + client = wrapped_async_client(handler) + stream = await client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True ) + stream._inner = source() delivered = [event async for event in stream] + await client.close() else: - stream = telemetry_dev_anthropic._InstrumentedStream( # pyright: ignore[reportPrivateUsage] - iter(events), cast(Any, handle), time.perf_counter() + + def sync_handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(stream_events()) + + client = wrapped_sync_client(sync_handler) + stream = client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True ) + stream._inner = iter(events) delivered = list(stream) + client.close() - state = cast(Any, stream)._state assert delivered == events - if capture_output: - assert state.blocks == {0: {"type": "text", "text": "sensitive"}} - else: - assert state.blocks == {} - assert "output" not in ended[0] - assert len(errors) == 1 - assert isinstance(errors[0], RuntimeError) - assert global_reports == [] - assert ended[0]["attributes"] == {"telemetry.dev.capture.truncated": True} + assert stream._state.blocks == ( + {0: {"type": "text", "text": "sensitive"}} if capture_output else {} + ) + assert [str(error) for error in errors] == ["broken message"] + span = only_span(memory) + assert span.status.status_code == StatusCode.UNSET + assert attrs(span)["telemetry.dev.capture.truncated"] is True + + +def test_stream_capture_does_not_invoke_expanding_model_dump() -> None: + model_dump_calls = 0 + + class ExpandingDelta: + def __init__(self) -> None: + self.type = "text_delta" + self.text = "safe" + + def model_dump(self, **_kwargs: Any) -> dict[str, Any]: + nonlocal model_dump_calls + model_dump_calls += 1 + return {"type": "text_delta", "text": "safe", "items": list(range(5_000))} + + captured = telemetry_dev_anthropic._bounded_native(ExpandingDelta()) # pyright: ignore[reportPrivateUsage] + + assert captured == {"type": "text_delta", "text": "safe"} + assert model_dump_calls == 0 def test_streaming_response_helper_preserves_api_response(memory: SimpleNamespace) -> None: @@ -886,19 +827,33 @@ async def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.usage.output_tokens"] == 2 -def test_sync_stream_bounds_retained_events_without_dropping_chunks( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_stream_bounds_retained_events_without_dropping_chunks( + memory: SimpleNamespace, async_mode: bool ) -> None: source_events = oversized_stream_events() - def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(source_events) + if async_mode: - client = wrapped_sync_client(handler) - stream = client.messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True - ) - delivered = list(stream) + async def handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(source_events) + + client = wrapped_async_client(handler) + stream = await client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True + ) + delivered = [event async for event in stream] + await client.close() + else: + + def sync_handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(source_events) + + client = wrapped_sync_client(sync_handler) + stream = client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True + ) + delivered = list(stream) state = stream._state assert len(delivered) == len(source_events) @@ -908,24 +863,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert attrs(only_span(memory))["telemetry.dev.capture.truncated"] is True -def test_stream_capture_never_exceeds_the_48_kib_provider_ceiling() -> None: - state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] - telemetry_dev_anthropic._record_stream_event( # pyright: ignore[reportPrivateUsage] - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": "x" * (49 * 1024)}, - }, - state, - ) - - partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] - - assert state.budget.max_bytes == 48 * 1024 - assert partial["output"] is None - assert partial["attributes"] == {"telemetry.dev.capture.truncated": True} - - def test_stream_revalidates_expanded_tool_json_against_the_item_limit() -> None: state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] record = telemetry_dev_anthropic._record_stream_event # pyright: ignore[reportPrivateUsage] @@ -948,6 +885,7 @@ def test_stream_revalidates_expanded_tool_json_against_the_item_limit() -> None: }, state, ) + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] @@ -978,7 +916,8 @@ def test_stream_rejects_normalized_tool_block_over_budget_but_keeps_metadata( "id": "toolu_oversized", "name": "oversized_tool", "input": {}, - "oversized": "x" * (64 * 1024), + # Over the 48 KiB stream bound but under the 64 KiB core cap. + "oversized": "x" * (49 * 1024), }, }, { @@ -1001,33 +940,12 @@ def handler(request: httpx.Request) -> httpx.Response: assert stream._state.budget.truncated is True a = attrs(only_span(memory)) assert "gen_ai.output.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True assert a["gen_ai.usage.input_tokens"] == 5 assert a["gen_ai.usage.output_tokens"] == 3 assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["tool_use"] -async def test_async_stream_bounds_retained_events_without_dropping_chunks( - memory: SimpleNamespace, -) -> None: - source_events = oversized_stream_events() - - async def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(source_events) - - client = wrapped_async_client(handler) - stream = await client.messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True - ) - delivered = [event async for event in stream] - state = stream._state - await client.close() - - assert len(delivered) == len(source_events) - assert state.budget.truncated is True - assert state.budget.bytes_used <= state.budget.max_bytes - assert len(state.blocks[0]["text"]) < 64 * 1024 - - def test_create_streaming_preserves_citation_deltas(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return named_sse_response(citation_stream_events()) @@ -1229,14 +1147,21 @@ async def handler(request: httpx.Request) -> httpx.Response: def test_messages_stream_enter_failure_records_error(memory: SimpleNamespace) -> None: + requests: list[httpx.Request] = [] + def handler(request: httpx.Request) -> httpx.Response: - return error_response(500, "server broke") + requests.append(request) + response = error_response(500, "server broke") + response.headers["retry-after-ms"] = "1" + return response client = wrapped_sync_client(handler) with pytest.raises(anthropic.APIStatusError): with client.messages.stream(model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES): pass + # The SDK retries inside one span. + assert len(requests) > 1 span = only_span(memory) assert span.status.status_code == StatusCode.ERROR assert any(event.name == "exception" for event in span.events) @@ -1429,41 +1354,6 @@ def stream_stub(**_kwargs: Any) -> None: assert attrs(only_span(memory))["gen_ai.provider.name"] == expected -def test_provider_mapping_for_bedrock_vertex_and_plain_client(memory: SimpleNamespace) -> None: - anthropic_any: Any = anthropic - bedrock = object.__new__(anthropic_any.AnthropicBedrock) - vertex = object.__new__(anthropic_any.AnthropicVertex) - - def bedrock_create(**_kwargs: Any) -> dict[str, Any]: - return message_payload(id="msg_bedrock") - - def vertex_create(**_kwargs: Any) -> dict[str, Any]: - return message_payload(id="msg_vertex") - - def stream_stub(**_kwargs: Any) -> None: - return None - - bedrock.messages = SimpleNamespace(create=bedrock_create, stream=stream_stub) - vertex.messages = SimpleNamespace(create=vertex_create, stream=stream_stub) - - def handler(request: httpx.Request) -> httpx.Response: - return json_response(message_payload()) - - wrap_anthropic(bedrock).messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) - wrap_anthropic(vertex).messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) - wrapped_sync_client(handler).messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) - bedrock_span, vertex_span, plain_span = memory.span_exporter.get_finished_spans() - assert attrs(bedrock_span)["gen_ai.provider.name"] == "aws.bedrock" - assert attrs(vertex_span)["gen_ai.provider.name"] == "gcp.vertex_ai" - assert attrs(plain_span)["gen_ai.provider.name"] == "anthropic" - - def test_beta_create_records_generation_span(memory: SimpleNamespace) -> None: requests: list[httpx.Request] = [] @@ -1488,52 +1378,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.response.finish_reasons"] == ("end_turn",) -async def test_async_beta_create_records_generation_span(memory: SimpleNamespace) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return json_response(message_payload(id="msg_beta_async")) - - client = wrapped_async_client(handler) - await client.beta.messages.create(model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES) - await client.close() - - assert attrs(only_span(memory))["gen_ai.response.id"] == "msg_beta_async" - - -def test_beta_stream_context_manager_records_one_span(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(stream_events()) - - client = wrapped_sync_client(handler) - with client.beta.messages.stream( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) as stream: - text = "".join(stream.text_stream) - - assert text == "Hello world" - a = attrs(only_span(memory)) - assert json.loads(str(a["gen_ai.output.messages"])) == [ - {"role": "assistant", "content": [{"type": "text", "text": "Hello world"}]} - ] - assert a["gen_ai.usage.output_tokens"] == 2 - - -async def test_async_beta_stream_context_manager_records_one_span( - memory: SimpleNamespace, -) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(stream_events()) - - client = wrapped_async_client(handler) - async with client.beta.messages.stream( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) as stream: - parts = [text async for text in stream.text_stream] - await client.close() - - assert "".join(parts) == "Hello world" - assert attrs(only_span(memory))["gen_ai.usage.output_tokens"] == 2 - - @pytest.mark.parametrize("namespace", ["stable", "beta"]) def test_parse_records_one_generation_span(memory: SimpleNamespace, namespace: str) -> None: def handler(request: httpx.Request) -> httpx.Response: @@ -1835,7 +1679,10 @@ def identity_mask(value: Any, _context: Any) -> Any: memory = make(mask=identity_mask) if masked else make() events = beta_stream_events( ({"type": "text", "text": ""}, [{"type": "text_delta", "text": "first"}]), - ({"type": "text", "text": ""}, [{"type": "text_delta", "text": '"' * 500}]), + ( + {"type": "text", "text": ""}, + [{"type": "text_delta", "text": "ab"}, {"type": "text_delta", "text": '"' * 500}], + ), ) a = streamed_beta_span(events, memory) @@ -1845,8 +1692,7 @@ def identity_mask(value: Any, _context: Any) -> Any: assert "gen_ai.output.messages" not in a else: content = json.loads(str(a["gen_ai.output.messages"]))[0]["content"] - assert content[0] == {"type": "text", "text": "first"} - assert all(('"' * 500).startswith(block["text"]) for block in content[1:]) + assert content == [{"type": "text", "text": "first"}, {"type": "text", "text": "ab"}] @pytest.mark.parametrize(("stream_limit", "complete"), [(48 * 1024, True), (1_000, False)]) @@ -1944,25 +1790,6 @@ def test_escape_heavy_deltas_are_charged_at_their_serialized_size( assert a["telemetry.dev.capture.truncated"] is True -def test_block_creating_text_delta_is_charged_at_its_serialized_size() -> None: - implementation = cast(Any, telemetry_dev_anthropic) - chunk = "\x01" * 100 - state = implementation._StreamState() - - implementation._record_stream_event( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": chunk}, - }, - state, - ) - - assert state.blocks[0] == {"type": "text", "text": chunk} - serialized = len(json.dumps(chunk, ensure_ascii=False).encode()) - 2 - assert state.budget.bytes_used >= serialized - - def test_block_creating_text_delta_also_charges_the_block_it_creates() -> None: implementation = cast(Any, telemetry_dev_anthropic) state = implementation._StreamState() @@ -2073,62 +1900,6 @@ def test_lone_surrogate_keeps_the_exported_span_whole(memory: SimpleNamespace) - assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["end_turn"] -def test_lone_surrogate_does_not_discard_the_output_that_already_fit() -> None: - implementation = cast(Any, telemetry_dev_anthropic) - state = implementation._StreamState() - record = implementation._record_stream_event - - record( - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": "keep me"}, - }, - state, - ) - record( - { - "type": "content_block_start", - "index": 1, - "content_block": {"type": "text", "text": "a\ud800"}, - }, - state, - ) - - partial = implementation._stream_partial(state) - - assert partial["output"] is not None - assert partial["output"][0]["content"][0]["text"] == "keep me" - - -def test_tool_input_over_the_bound_is_caught_while_streaming() -> None: - implementation = cast(Any, telemetry_dev_anthropic) - state = implementation._StreamState() - record = implementation._record_stream_event - - record( - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "tool_use", "id": "t1", "name": "calc"}, - }, - state, - ) - # Quote-light JSON: nothing to escape, but every element gains ", " once serialized. - payload = '{"n":[' + ",".join(["1"] * 20_000) + "]}" - record( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "input_json_delta", "partial_json": payload}, - }, - state, - ) - record({"type": "content_block_stop", "index": 0}, state) - - assert state.budget.truncated is True - - def _tool_stream(payloads: list[str], start_input: dict[str, Any] | None = None) -> Any: implementation = cast(Any, telemetry_dev_anthropic) state = implementation._StreamState() @@ -2167,15 +1938,20 @@ def test_settled_tool_input_charge_covers_what_is_still_retained( assert retained <= state.budget.bytes_used -@pytest.mark.parametrize("elements", [8, 1_500, 5_000, 20_000]) -def test_settled_tool_input_agrees_with_the_final_bounded_check(elements: int) -> None: +@pytest.mark.parametrize( + ("elements", "truncated"), [(8, False), (1_500, True), (5_000, True), (20_000, True)] +) +def test_settled_tool_input_agrees_with_the_final_bounded_check( + elements: int, truncated: bool +) -> None: implementation = cast(Any, telemetry_dev_anthropic) state = _tool_stream(['{"n":[' + ",".join(["1"] * elements) + "]}"]) truncated_while_streaming = state.budget.truncated dropped = implementation._stream_partial(state)["output"] is None - assert dropped is truncated_while_streaming + assert truncated_while_streaming is truncated + assert dropped is truncated def test_many_small_tool_blocks_are_all_kept() -> None: @@ -2474,22 +2250,6 @@ def fallback_stream(iterations: list[dict[str, Any]]) -> list[dict[str, Any]]: ) -def test_beta_stream_records_fallback_model_that_served_the_response( - memory: SimpleNamespace, -) -> None: - a = streamed_beta_span(fallback_stream([DECLINED_ITERATION, SERVED_ITERATION]), memory) - - assert a["gen_ai.response.model"] == "claude-opus-4-8" - - -def test_beta_stream_keeps_requested_model_when_fallback_request_failed( - memory: SimpleNamespace, -) -> None: - a = streamed_beta_span(fallback_stream([DECLINED_ITERATION]), memory) - - assert a["gen_ai.response.model"] == "claude-opus-5" - - ADVISOR_ITERATION: dict[str, Any] = { **DECLINED_ITERATION, "type": "advisor_message", @@ -2544,20 +2304,30 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: output = json.loads(str(a["gen_ai.output.messages"])) assert output == [ - { - "role": "assistant", - "content": [ - {**FALLBACK_BLOCK, "from_": {"model": "claude-opus-5"}}, - {"type": "text", "text": "Hi"}, - ], - } + {"role": "assistant", "content": [FALLBACK_BLOCK, {"type": "text", "text": "Hi"}]} ] assert a["telemetry.dev.capture.truncated"] is True +def test_beta_create_records_a_fallback_block_under_its_wire_keys(memory: SimpleNamespace) -> None: + content = [FALLBACK_BLOCK, {"type": "text", "text": "Hi"}] + + def handler(request: httpx.Request) -> httpx.Response: + return json_response(message_payload(content=content)) + + wrapped_sync_client(handler).beta.messages.create( + model="claude-opus-5", max_tokens=64, messages=MESSAGES + ) + + output = json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) + assert output[0]["content"] == content + + @pytest.mark.parametrize( ("iterations", "expected"), [ + ([DECLINED_ITERATION, SERVED_ITERATION], "claude-opus-4-8"), + ([DECLINED_ITERATION], "claude-opus-5"), ([DECLINED_ITERATION, ADVISOR_ITERATION], "claude-opus-5"), ([DECLINED_ITERATION, SERVED_ITERATION, ADVISOR_ITERATION], "claude-opus-4-8"), ( @@ -2565,9 +2335,15 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: "claude-sonnet-5", ), ], - ids=["advisor_without_fallback", "advisor_after_fallback", "last_fallback_wins"], + ids=[ + "fallback_served", + "fallback_failed", + "advisor_without_fallback", + "advisor_after_fallback", + "last_fallback_wins", + ], ) -def test_beta_stream_ignores_advisor_iterations_for_the_served_model( +def test_beta_stream_response_model_follows_the_last_served_fallback( memory: SimpleNamespace, iterations: list[dict[str, Any]], expected: str ) -> None: a = streamed_beta_span(fallback_stream(iterations), memory) @@ -2730,9 +2506,10 @@ def record_output_chunk( assert len(timestamps) == 2 +@pytest.mark.parametrize("provider", ["anthropic", "gcp.vertex_ai"]) @pytest.mark.parametrize("mode", ["wrap", "instrument"]) -async def test_vertex_beta_stream_records_one_span_sync_and_async( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, mode: str +async def test_beta_stream_helper_records_one_span_sync_and_async( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, mode: str, provider: str ) -> None: def sync_handler(request: httpx.Request) -> httpx.Response: return named_sse_response(stream_events()) @@ -2742,28 +2519,32 @@ async def handler(request: httpx.Request) -> httpx.Response: if mode == "instrument": instrument_anthropic() - sync_vertex, async_vertex = provider_clients(monkeypatch, handler, sync_handler)[ - "gcp.vertex_ai" - ] + if provider == "anthropic": + sync_provider, async_provider = sync_client(sync_handler), async_client(handler) + else: + sync_provider, async_provider = provider_clients(monkeypatch, handler, sync_handler)[ + provider + ] if mode == "wrap": - sync_vertex = wrap_anthropic(sync_vertex) - async_vertex = wrap_anthropic(async_vertex) + sync_provider = wrap_anthropic(sync_provider) + async_provider = wrap_anthropic(async_provider) - with sync_vertex.beta.messages.stream( + with sync_provider.beta.messages.stream( model="claude-sonnet-4-6", max_tokens=64, messages=BETA_MESSAGES ) as stream: sync_text = "".join(stream.text_stream) - async with async_vertex.beta.messages.stream( + async with async_provider.beta.messages.stream( model="claude-sonnet-4-6", max_tokens=64, messages=BETA_MESSAGES ) as stream: async_text = "".join([text async for text in stream.text_stream]) - await async_vertex.close() + await async_provider.close() assert sync_text == async_text == "Hello world" spans = memory.span_exporter.get_finished_spans() assert len(spans) == 2 for span in spans: - assert attrs(span)["gen_ai.provider.name"] == "gcp.vertex_ai" + assert attrs(span)["gen_ai.provider.name"] == provider + assert attrs(span)["gen_ai.usage.output_tokens"] == 2 assert json.loads(str(attrs(span)["gen_ai.output.messages"])) == [ {"role": "assistant", "content": [{"type": "text", "text": "Hello world"}]} ] @@ -2781,14 +2562,6 @@ def test_missing_provider_beta_module_is_skipped(monkeypatch: pytest.MonkeyPatch assert all("bedrock" not in cls.__module__ for cls in classes) -def test_global_instrumentation_wraps_async_provider_classes_as_async() -> None: - instrument_anthropic() - for cls in telemetry_dev_anthropic._provider_beta_classes(): # pyright: ignore[reportPrivateUsage] - original = getattr(cls.create, "_telemetry_dev_anthropic_original", None) - assert original is not None - assert inspect.iscoroutinefunction(cls.create) == cls.__name__.startswith("Async") - - def test_beta_stream_compaction_delta_without_start_is_a_compaction_block( memory: SimpleNamespace, ) -> None: @@ -2918,6 +2691,7 @@ def test_oversized_compaction_replacement_drops_stale_block_and_recovers( assert state.budget.truncated is False assert state.budget.bytes_used == expected.bytes_used assert state.blocks[0]["content"] == "final" + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] assert "attributes" not in partial @@ -2968,6 +2742,7 @@ def test_rejected_inherited_encrypted_content_stays_unresolved_until_replaced( assert state.blocks[0] == {"type": "compaction", "content": "recovered"} assert state.unresolved_encrypted_content == {0} + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial( # pyright: ignore[reportPrivateUsage] state, mask_output_when_incomplete=True ) @@ -3136,6 +2911,47 @@ class Model(BaseModel): assert bounded == {"type": "text", "citations_extra": "kept"} +def test_stream_capture_alias_inspection_is_bounded( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + implementation = cast(Any, telemetry_dev_anthropic) + stored = implementation._stored + alias_reads = 0 + + def counted_stored(value: Any, name: str) -> Any: + nonlocal alias_reads + if name == "alias": + alias_reads += 1 + return stored(value, name) + + class SparseModel: + __pydantic_fields__ = { + f"unused_{index}": Field(alias=f"unused-{index}") for index in range(5_000) + } | {"from_": Field(alias="from")} + + def __init__(self) -> None: + self.type = "fallback" + self.from_ = {"model": "claude-opus-5"} + + monkeypatch.setattr(implementation, "_stored", counted_stored) + events = [ + {"type": "content_block_start", "index": 0, "content_block": SparseModel()}, + {"type": "message_stop"}, + ] + + def create(**_kwargs: Any) -> Iterator[Any]: + return iter(events) + + client: Any = object.__new__(Anthropic) + client.messages = SimpleNamespace(create=create) + delivered = list(wrap_anthropic(client).messages.create(model="claude-opus-5", stream=True)) + + assert delivered == events + output = json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) + assert output[0]["content"] == [{"type": "fallback", "from": {"model": "claude-opus-5"}}] + assert alias_reads <= 1_000 + + def _spoofed_and_aliased_storage(calls: list[str]) -> list[Any]: class Payload: def __get__(self, instance: Any, owner: Any) -> dict[str, Any]: @@ -3241,32 +3057,6 @@ def test_rejected_unseen_compaction_replacement_recovers( assert state.blocks[0]["content"] == "recovered" -def test_rejected_final_compaction_marks_capture_incomplete( - make: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(telemetry_dev_anthropic, "_STREAM_CAPTURE_MAX_BYTES", 300) - make() - state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] - record = telemetry_dev_anthropic._record_stream_event # pyright: ignore[reportPrivateUsage] - - record( - {"type": "content_block_start", "index": 0, "content_block": {"type": "compaction"}}, - state, - ) - record( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "compaction_delta", "content": "x" * 400}, - }, - state, - ) - - partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] - assert partial["attributes"] == {"telemetry.dev.capture.truncated": True} - assert state.budget.truncated is False - - def test_masked_beta_stream_omits_incomplete_retained_output(make: Any) -> None: def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: return value @@ -3492,6 +3282,7 @@ def test_stream_drops_compaction_superseded_by_a_rejected_final_replacement( assert json.loads(str(a["gen_ai.output.messages"])) == [ {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]} ] + assert a["telemetry.dev.capture.truncated"] is True def test_rejected_signature_keeps_thinking_and_drops_the_stale_signature( @@ -3691,41 +3482,6 @@ def test_rejected_compaction_releases_budget_for_later_blocks( ] -def test_compaction_replacement_is_measured_once( - make: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - make() - state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] - record = telemetry_dev_anthropic._record_stream_event # pyright: ignore[reportPrivateUsage] - record( - {"type": "content_block_start", "index": 0, "content_block": {"type": "compaction"}}, - state, - ) - accept_calls = 0 - original_accept = telemetry_dev.CaptureBudget.accept - - def count_accept(self: telemetry_dev.CaptureBudget, value: object) -> bool: - nonlocal accept_calls - accept_calls += 1 - return original_accept(self, value) - - monkeypatch.setattr(telemetry_dev.CaptureBudget, "accept", count_accept) - record( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "compaction_delta", "content": "summary"}, - }, - state, - ) - - assert accept_calls == 1 - assert state.blocks[0] == { - "type": "compaction", - "content": "summary", - } - - def test_rejected_compactions_after_truncation_leave_no_state( make: Any, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -3820,6 +3576,7 @@ def test_oversized_compaction_replacement_keeps_inherited_encrypted_content_unre assert state.blocks[0] == {"type": "compaction", "content": "recovered"} assert state.unresolved_replacements == set() assert state.unresolved_encrypted_content == {0} + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] assert partial["attributes"] == {"telemetry.dev.capture.truncated": True} masked = telemetry_dev_anthropic._stream_partial( # pyright: ignore[reportPrivateUsage] diff --git a/sdks/python-bedrock/tests/test_bedrock.py b/sdks/python-bedrock/tests/test_bedrock.py index 7f3fff1..3b6acac 100644 --- a/sdks/python-bedrock/tests/test_bedrock.py +++ b/sdks/python-bedrock/tests/test_bedrock.py @@ -489,6 +489,7 @@ def _streamed_converse( [ (1_000, False, "東" * 400), (1_000, True, "[redacted]"), + (100, False, "...[truncated]"), (0, False, None), ], ) @@ -512,28 +513,17 @@ def mask(value: Any, _context: Any) -> Any: assert state.budget.truncated is False assert state.blocks[0]["text"] == text assert "telemetry.dev.capture.truncated" not in span.attributes + exported = span.attributes.get("gen_ai.output.messages", "") if expected is None: - assert span.attributes.get("gen_ai.output.messages", "") == "" + assert exported == "" + elif expected == "...[truncated]": + assert isinstance(exported, str) + assert len(exported) <= max_attribute_length + assert exported.endswith(expected) else: assert _json_attr(span, "gen_ai.output.messages")[0]["parts"][0]["content"] == expected -def test_small_configured_cap_cuts_the_exported_stream_output_not_retention( - make: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - memory = make(max_attribute_length=100) - text = "x" * 5_000 - - state, span = _streamed_converse(memory, monkeypatch, _converse_text_stream(text)) - - assert state.blocks[0]["text"] == text - exported = span.attributes["gen_ai.output.messages"] - assert isinstance(exported, str) - assert len(exported) <= 100 - assert exported.endswith("...[truncated]") - assert "telemetry.dev.capture.truncated" not in span.attributes - - @pytest.mark.parametrize("masked", [False, True]) @pytest.mark.parametrize("max_attribute_length", [0, 100, 1024 * 1024]) def test_stream_beyond_the_fixed_bound_is_flagged_and_withheld_from_a_mask( @@ -662,7 +652,6 @@ def test_invoke_model_stream_bounds_content_but_keeps_terminal_metadata( assert state.budget.truncated is True assert state.budget.bytes_used <= state.budget.max_bytes span = memory.span_exporter.get_finished_spans()[0] - assert not hasattr(state, "text") retained_text = _json_attr(span, "gen_ai.output.messages")[0]["parts"][0]["content"] expected_prefix = "".join(f"{index:04d}" + "x" * 96 for index in range(len(state.chunks))) assert retained_text == expected_prefix @@ -691,6 +680,7 @@ def test_unconsumed_stream_finishes_partial( spans = memory.span_exporter.get_finished_spans() assert len(spans) == 1 assert spans[0].attributes["gen_ai.operation.name"] == "chat" + assert spans[0].attributes["telemetry.dev.capture.truncated"] is True @pytest.mark.parametrize("kind", ["absent", "compaction", "at-bound", "bounded"]) @@ -1586,36 +1576,42 @@ def test_invoke_model_stream_titan_counts_finish_and_unrewindable_body( assert list(titan_span.attributes["gen_ai.response.finish_reasons"]) == ["FINISHED"] +@pytest.mark.parametrize( + ("chunks", "content", "usage"), + [ + ( + [ + { + "text": "choice", + "finish_reason": "COMPLETE", + "meta": {"billed_units": {"input_tokens": 8, "output_tokens": 3}}, + } + ], + "choice", + (8, 3), + ), + ( + [ + {"generations": [{"text": "Cohere "}]}, + {"generations": [{"text": "done", "finish_reason": "COMPLETE"}]}, + ], + "Cohere done", + None, + ), + ], + ids=["text", "generations"], +) def test_invoke_model_stream_cohere_generations_finish( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch + memory: SimpleNamespace, + monkeypatch: pytest.MonkeyPatch, + chunks: list[dict[str, Any]], + content: str, + usage: tuple[int, int] | None, ) -> None: client = _client("bedrock-runtime") _stub_api_call( client, - [ - { - "body": FakeEventStream( - [ - { - "chunk": { - "bytes": _body( - { - "text": "choice", - "finish_reason": "COMPLETE", - "meta": { - "billed_units": { - "input_tokens": 8, - "output_tokens": 3, - } - }, - } - ) - } - } - ] - ) - }, - ], + [{"body": FakeEventStream([{"chunk": {"bytes": _body(chunk)}} for chunk in chunks])}], monkeypatch, ) wrap_bedrock(client) @@ -1628,11 +1624,13 @@ def test_invoke_model_stream_cohere_generations_finish( list(cohere_response["body"]) cohere_span = memory.span_exporter.get_finished_spans()[0] - assert _json_attr(cohere_span, "gen_ai.output.messages")[0]["parts"][0]["content"] == "choice" - assert _json_attr(cohere_span, "gen_ai.output.messages")[0]["finish_reason"] == "COMPLETE" - assert cohere_span.attributes["gen_ai.usage.input_tokens"] == 8 - assert cohere_span.attributes["gen_ai.usage.output_tokens"] == 3 + output = _json_attr(cohere_span, "gen_ai.output.messages")[0] + assert output["parts"][0]["content"] == content + assert output["finish_reason"] == "COMPLETE" assert list(cohere_span.attributes["gen_ai.response.finish_reasons"]) == ["COMPLETE"] + if usage is not None: + assert cohere_span.attributes["gen_ai.usage.input_tokens"] == usage[0] + assert cohere_span.attributes["gen_ai.usage.output_tokens"] == usage[1] def test_embeddings_apply_guardrail_and_error( diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index 5173ecb..f045e68 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -400,15 +400,22 @@ def sync_generate_images(**_kwargs: Any) -> dict[str, Any]: def test_media_capture_projects_cyclic_labels(memory: SimpleNamespace) -> None: + client, _transport = client_with_transport([]) + + def generate_images(**_kwargs: Any) -> dict[str, Any]: + return {"generatedImages": []} + + client.models.generate_images = generate_images labels: dict[str, Any] = {"ordinary": "value"} labels["cycle"] = labels - contents, truncated = google_genai_instrumentation._media_contents( - {"config": {"labels": labels}} - ) + wrap_google_genai(client).models.generate_images(model="imagen-4.0", config={"labels": labels}) - assert truncated is False - assert contents == {"labels": {"ordinary": "value", "cycle": None}} + a = attrs(only_span(memory)) + assert json.loads(str(a["gen_ai.input.messages"])) == { + "labels": {"ordinary": "value", "cycle": None} + } + assert "telemetry.dev.capture.truncated" not in a @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) @@ -699,6 +706,7 @@ async def run() -> Any: assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["STOP"] assert a["gen_ai.request.temperature"] == 0.2 assert a["gen_ai.request.top_p"] == 0.9 + assert a["gen_ai.request.top_k"] == 40 assert a["gen_ai.request.max_tokens"] == 64 assert list(cast(Any, a["gen_ai.request.stop_sequences"])) == ["END"] assert a["gen_ai.request.seed"] == 7 @@ -1560,8 +1568,11 @@ async def test_prompt_feedback_text_is_kept_only_when_output_is_captured_unmaske } await generate([response] if stream else response, is_async=is_async, stream=stream) - a = attrs(only_span(memory)) + span = only_span(memory) + a = attrs(span) assert a["google_genai.response.block_reason"] == "SAFETY" + assert "gen_ai.output.messages" not in a + assert span.status.status_code == StatusCode.UNSET prompt_ratings = json.loads(cast(str, a["google_genai.response.prompt_safety_ratings"])) if capture == "on": assert a["google_genai.response.block_reason_message"] == "private block message" @@ -1830,19 +1841,6 @@ def test_api_error_records_error_and_reraises(memory: SimpleNamespace, async_mod assert span.status.status_code == StatusCode.ERROR -def test_blocked_prompt_records_block_attrs_without_output(memory: SimpleNamespace) -> None: - payload = {"promptFeedback": {"blockReason": "SAFETY", "blockReasonMessage": "blocked"}} - client, _transport = client_with_transport([payload]) - wrapped = wrap_google_genai(client) - response = wrapped.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) - assert response.candidates is None - a = attrs(only_span(memory)) - assert a["google_genai.response.block_reason"] == "SAFETY" - assert a["google_genai.response.block_reason_message"] == "blocked" - assert "gen_ai.output.messages" not in a - assert only_span(memory).status.status_code == StatusCode.UNSET - - @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) def test_embed_content_maps_embedding_attrs(memory: SimpleNamespace, async_mode: bool) -> None: client, _transport = client_with_transport([embed_payload()]) @@ -1900,11 +1898,10 @@ def test_chats_emit_spans_with_history(memory: SimpleNamespace) -> None: assert json.loads(str(attrs(span)["gen_ai.input.messages"])) == expected_input -def test_vertex_provider(memory: SimpleNamespace) -> None: - _client, _transport = client_with_transport([happy_payload()]) - wrapped = wrap_google_genai(genai.Client(api_key="test", vertexai=True)) - wrapped._api_client.request = _transport.request - wrapped.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) +def test_global_instrumentation_reports_vertex_provider(memory: SimpleNamespace) -> None: + instrument_google_genai() + client, _transport = client_with_transport([happy_payload()], vertexai=True) + client.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) assert attrs(only_span(memory))["gen_ai.provider.name"] == "gcp.vertex_ai" @@ -2104,26 +2101,12 @@ def generate_videos(_self: Any, **_kwargs: Any) -> dict[str, Any]: @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize("poll_fails", [False, True], ids=["terminal", "exception"]) -def test_uninstrument_claims_callback_while_video_poll_is_blocked( +def test_uninstrument_ends_video_span_while_poll_is_blocked( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, async_mode: bool, poll_fails: bool, ) -> None: - callback_calls = 0 - original_end_once = google_genai_instrumentation._end_once - - def counted_end_once(handle: telemetry_dev.SpanHandle) -> Any: - end = original_end_once(handle) - - def counted_end(**fields: Any) -> None: - nonlocal callback_calls - callback_calls += 1 - end(**fields) - - return counted_end - - monkeypatch.setattr(google_genai_instrumentation, "_end_once", counted_end_once) failure = RuntimeError("blocked poll failed") if async_mode: @@ -2200,7 +2183,6 @@ def get_operation(_self: Any, operation: Any) -> dict[str, Any]: else: assert cast(Any, poll.result(timeout=5))["done"] is True - assert callback_calls == 1 span = only_span(memory) assert span.status.status_code is StatusCode.ERROR assert "google_genai.response.operation_done" not in attrs(span) @@ -2291,12 +2273,12 @@ def test_sync_stream_close_ends_partial_span(memory: SimpleNamespace) -> None: ] -def test_sync_stream_supports_next_send_close_protocol(memory: SimpleNamespace) -> None: +def test_sync_stream_supports_next_and_send_protocol(memory: SimpleNamespace) -> None: client, _transport = client_with_transport([stream_chunks()]) wrapped = wrap_google_genai(client) stream = wrapped.models.generate_content_stream(model="gemini-2.5-flash", contents=USER_CONTENT) assert next(stream).text == "Hello" - assert next(stream).text == " world" + assert stream.send(None).text == " world" next(stream) with pytest.raises(StopIteration): next(stream) @@ -2346,23 +2328,6 @@ async def run() -> None: ] -def test_blocked_prompt_records_prompt_safety_ratings(memory: SimpleNamespace) -> None: - payload = { - "promptFeedback": { - "blockReason": "SAFETY", - "safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "HIGH"}], - } - } - client, _transport = client_with_transport([payload]) - wrapped = wrap_google_genai(client) - wrapped.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) - a = attrs(only_span(memory)) - assert a["google_genai.response.block_reason"] == "SAFETY" - ratings = json.loads(str(a["google_genai.response.prompt_safety_ratings"])) - assert ratings[0]["category"] == "HARM_CATEGORY_HATE_SPEECH" - assert ratings[0]["probability"] == "HIGH" - - def test_mapper_failures_are_fail_open( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/sdks/python-litellm/tests/test_litellm.py b/sdks/python-litellm/tests/test_litellm.py index a591c26..f7b08a1 100644 --- a/sdks/python-litellm/tests/test_litellm.py +++ b/sdks/python-litellm/tests/test_litellm.py @@ -142,25 +142,16 @@ def test_completion_multiple_choices(memory: SimpleNamespace) -> None: assert tuple(cast(Any, a["gen_ai.response.finish_reasons"])) == ("stop", "stop") -def test_completion_metadata_param_maps_to_td_metadata(memory: SimpleNamespace) -> None: - instrument_litellm() - llm.completion( - model="gpt-4o-mini", - messages=CHAT_MESSAGES, - metadata={"tenant": "acme"}, - mock_response="ok", - ) - - assert attrs(only_span(memory))["td.metadata.tenant"] == "acme" - - -def test_completion_litellm_metadata_maps_to_td_metadata(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("argument", ["metadata", "litellm_metadata"]) +def test_completion_metadata_param_maps_to_td_metadata( + memory: SimpleNamespace, argument: str +) -> None: instrument_litellm() llm.completion( model="gpt-4o-mini", messages=CHAT_MESSAGES, - litellm_metadata={"tenant": "acme"}, mock_response="ok", + **{argument: {"tenant": "acme"}}, ) assert attrs(only_span(memory))["td.metadata.tenant"] == "acme" @@ -339,6 +330,7 @@ def test_router_fallback_drops_the_failed_call_history(memory: SimpleNamespace) }, ], fallbacks=[{"primary": ["backup"]}], + num_retries=0, ) router.completion( model="primary", @@ -409,22 +401,6 @@ def test_completion_error_records_error_span(memory: SimpleNamespace) -> None: assert a["gen_ai.provider.name"] == "openai" -def test_completion_timeout_error(memory: SimpleNamespace) -> None: - instrument_litellm() - - with pytest.raises(llm.Timeout): - llm.completion( - model="gpt-4o-mini", - messages=CHAT_MESSAGES, - mock_timeout=True, - timeout=0.01, - ) - - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "Timeout" - - def test_completion_response_mapping_base_exception_escapes( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -841,13 +817,10 @@ def fake_completion(*args: Any, **kwargs: Any) -> Any: instrument_litellm() stream = llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) delivered = list(stream) - budget = stream._budget assert delivered == chunks - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert len(stream._chunks) < len(chunks) a = attrs(only_span(memory)) + assert a["telemetry.dev.capture.truncated"] is True assert a["gen_ai.response.id"] == "bounded-stream" assert a["gen_ai.response.model"] == "gpt-4o-mini" assert tuple(cast(Any, a["gen_ai.response.finish_reasons"])) == ( @@ -1007,7 +980,10 @@ def test_configured_cap_applies_after_the_mask_not_to_stream_retention( redact: bool, expected: str | None, ) -> None: + seen: list[Any] = [] + def mask(value: Any, _context: Any) -> Any: + seen.append(value) if not redact: return value return [{"role": "assistant", "content": "[redacted]"}] @@ -1024,10 +1000,10 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: stream = llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) assert list(stream) == chunks - assert stream._budget.truncated is False - assert len(stream._chunks) == len(chunks) a = attrs(only_span(memory)) assert "telemetry.dev.capture.truncated" not in a + if redact: + assert [{"role": "assistant", "content": text}] in seen if expected is None: assert a.get("gen_ai.output.messages", "") == "" else: @@ -1071,18 +1047,17 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: stream = llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) assert list(stream) == chunks - budget = stream._budget - assert budget.truncated is True - assert budget.max_bytes == telemetry_dev_litellm._STREAM_CAPTURE_MAX_BYTES # pyright: ignore[reportPrivateUsage] - assert 0 < len(stream._chunks) < len(chunks) a = attrs(only_span(memory)) assert a["telemetry.dev.capture.truncated"] is True if masked: assert "gen_ai.output.messages" not in a assert not any("x" * 100 in value for value in seen) - elif max_attribute_length > budget.max_bytes: - retained = "x" * 100 * len(stream._chunks) - assert json_attr(a["gen_ai.output.messages"])[0]["content"] == retained + elif max_attribute_length == 1024 * 1024: + # The 48 KiB retention bound, not the 1 MiB cap, cuts the stream at a chunk boundary. + content = json_attr(a["gen_ai.output.messages"])[0]["content"] + assert set(content) == {"x"} + assert 0 < len(content) <= 48 * 1024 + assert len(content) % 100 == 0 else: assert len(str(a.get("gen_ai.output.messages", ""))) <= max_attribute_length @@ -1139,9 +1114,7 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: if extra_indices == 0: assert finish_reasons == ("length", "content_filter") return - max_items = telemetry_dev_litellm._STREAM_CAPTURE_MAX_ITEMS # pyright: ignore[reportPrivateUsage] - assert len(stream._finish_reasons) <= max_items - assert len(finish_reasons) <= max_items + assert len(finish_reasons) <= 1_000 assert a["telemetry.dev.capture.truncated"] is True assert "gen_ai.output.messages" not in a assert not any('"first"' in value for value in seen) @@ -1182,7 +1155,6 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: a = attrs(only_span(memory)) finish_reasons = tuple(cast(Any, a["gen_ai.response.finish_reasons"])) assert finish_reasons == ("r" * max_length, "stop") - assert all(len(reason) <= max_length for reason in stream._finish_reasons.values()) assert (a.get("telemetry.dev.capture.truncated") is True) is spent @@ -1377,80 +1349,6 @@ def record_output_chunk( assert timestamps -async def test_async_streaming_bounds_retained_chunks_without_dropping_output( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch -) -> None: - chunks = [ - SimpleNamespace( - id=None, - model=None, - choices=[ - SimpleNamespace( - index=0, - delta=SimpleNamespace(content="x" * 100), - finish_reason=None, - ) - ], - ) - for _ in range(1100) - ] - chunks.append( - SimpleNamespace( - id="bounded-async-stream", - model="gpt-4o-mini", - choices=[ - SimpleNamespace( - index=0, - delta=SimpleNamespace(content=None), - finish_reason="stop", - ) - ], - usage=SimpleNamespace( - prompt_tokens=13, - completion_tokens=8, - total_tokens=21, - ), - ) - ) - - class AsyncChunks: - def __init__(self) -> None: - self._iterator = iter(chunks) - - def __aiter__(self) -> AsyncChunks: - return self - - async def __anext__(self) -> Any: - try: - return next(self._iterator) - except StopIteration as error: - raise StopAsyncIteration from error - - async def aclose(self) -> None: - return None - - async def fake_acompletion(*args: Any, **kwargs: Any) -> Any: - return AsyncChunks() - - monkeypatch.setattr(litellm, "acompletion", fake_acompletion) - instrument_litellm() - stream = await llm.acompletion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) - delivered = [chunk async for chunk in stream] - budget = stream._budget - - assert delivered == chunks - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert len(stream._chunks) < len(chunks) - a = attrs(only_span(memory)) - assert a["gen_ai.response.id"] == "bounded-async-stream" - assert a["gen_ai.response.model"] == "gpt-4o-mini" - assert tuple(cast(Any, a["gen_ai.response.finish_reasons"])) == ("stop",) - assert a["gen_ai.usage.input_tokens"] == 13 - assert a["gen_ai.usage.output_tokens"] == 8 - assert a["gen_ai.usage.total_tokens"] == 21 - - async def test_async_positional_stream_argument_is_instrumented(memory: SimpleNamespace) -> None: instrument_litellm() stream = await llm.acompletion( @@ -1914,6 +1812,7 @@ def __len__(self) -> int: list(llm.responses(input=UnreadableInput(), model="gpt-4.1-mini", stream=True)) + assert only_span(env).name == "chat gpt-4.1-mini" assert "gen_ai.input.messages" not in attrs(only_span(env)) @@ -2251,10 +2150,8 @@ def test_responses_stream_small_cap_keeps_retention_and_cuts_after_the_mask( monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() - stream = llm.responses(input="bounded", model="gpt-4.1-mini", stream=True) - list(stream) + list(llm.responses(input="bounded", model="gpt-4.1-mini", stream=True)) - assert stream._budget.truncated is False a = attrs(only_span(memory)) assert "telemetry.dev.capture.truncated" not in a exported = str(a["gen_ai.output.messages"]) @@ -2368,6 +2265,7 @@ def test_responses_stream_rejected_annotation_preserves_text_and_marks_truncated annotation_index=0, annotation={"type": "url_citation", "url": "https://example.com/" + "c" * 7_000}, ), + completed_without_output(), ] monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() @@ -2396,6 +2294,7 @@ def test_responses_stream_rejects_sparse_annotation_indexes( annotation_index=1_000_000_000, annotation={"type": "url_citation", "url": "https://example.com"}, ), + completed_without_output(), ] monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() @@ -2492,45 +2391,75 @@ def test_responses_stream_merges_seeded_textual_event_families( assert output[2]["code"] == "print(1)" -def test_bounded_responses_conversion_stops_without_model_dump(memory: SimpleNamespace) -> None: - class Bomb: - def __init__(self) -> None: - self.output = [{"type": "output_text", "text": "x" * 100_000}] - self.status = "completed" +class _Opaque: + __slots__ = () - def model_dump(self, **kwargs: Any) -> Any: - raise AssertionError("model_dump must not run") - fields = vars(telemetry_dev_litellm)["_responses_response"](Bomb()) - assert fields["output"] is None - assert fields["attributes"]["telemetry.dev.capture.truncated"] is True +class _ModelDumpBomb: + """A response whose model_dump must not run; its output exceeds the 64 KiB non-stream bound.""" + def __init__(self) -> None: + self.output = [{"type": "output_text", "text": "x" * 100_000}] + self.status = "completed" -def test_bounded_responses_conversion_repeats_shared_acyclic_values( - memory: SimpleNamespace, -) -> None: - shared = {"text": "same"} + def model_dump(self, **kwargs: Any) -> Any: + raise AssertionError("model_dump must not run") - captured, budget = vars(telemetry_dev_litellm)["_bounded_responses_native"]([shared, shared]) - assert captured == [{"text": "same"}, {"text": "same"}] - assert budget.truncated is False +_SHARED_ITEM = {"text": "same"} -@pytest.mark.parametrize("value", ["\x01" * 1000, {"\x01" * 1000: 1}], ids=["string", "key"]) -def test_bounded_responses_conversion_charges_strings_at_their_escaped_size( - memory: SimpleNamespace, value: object +@pytest.mark.parametrize( + ("response", "expected"), + [ + pytest.param( + SimpleNamespace(status="completed", output=[_SHARED_ITEM, _SHARED_ITEM]), + [{"text": "same"}, {"text": "same"}], + id="shared-acyclic-values-repeat", + ), + pytest.param( + SimpleNamespace(status="completed", output=[{"opaque": _Opaque()}]), + [{"opaque": None}], + id="opaque-value-is-null", + ), + # Control characters cost six bytes each once escaped, one as raw UTF-8, so 10,000 of + # them fit the 64 KiB bound and 11,000 do not. + pytest.param( + SimpleNamespace(status="completed", output=[{"payload": "\x01" * 10_000}]), + [{"payload": "\x01" * 10_000}], + id="escaped-string-within-bound", + ), + pytest.param( + SimpleNamespace(status="completed", output=[{"payload": "\x01" * 11_000}]), + None, + id="escaped-string-over-bound", + ), + pytest.param( + SimpleNamespace(status="completed", output=[{"\x01" * 11_000: 1}]), + None, + id="escaped-key-over-bound", + ), + pytest.param(_ModelDumpBomb(), None, id="oversized-without-model-dump"), + ], +) +def test_responses_output_conversion_is_bounded_and_json_shaped( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, response: Any, expected: Any ) -> None: - # Control characters cost six bytes each once escaped, one as raw UTF-8. - convert = vars(telemetry_dev_litellm)["_bounded_responses_native"] + def fake_responses(*args: Any, **kwargs: Any) -> Any: + return response - rejected, small = convert({"payload": value}, budget=telemetry_dev.CaptureBudget(2048)) - kept, large = convert({"payload": value}, budget=telemetry_dev.CaptureBudget(8192)) + monkeypatch.setattr(litellm, "responses", fake_responses) + instrument_litellm() + + llm.responses(input="hi", model="gpt-4.1-mini") - assert rejected is None - assert small.truncated is True - assert kept == {"payload": value} - assert large.bytes_used >= len(json.dumps(kept)) + a = attrs(only_span(memory)) + if expected is None: + assert a["telemetry.dev.capture.truncated"] is True + assert "gen_ai.output.messages" not in a + else: + assert json_attr(a["gen_ai.output.messages"]) == expected + assert "telemetry.dev.capture.truncated" not in a def test_responses_stream_charges_appended_deltas_at_their_escaped_size( @@ -2554,19 +2483,6 @@ def test_responses_stream_charges_appended_deltas_at_their_escaped_size( assert json_attr(a["gen_ai.output.messages"]) == [{"type": "output_text", "text": "kept"}] -def test_bounded_responses_conversion_normalizes_opaque_values(memory: SimpleNamespace) -> None: - class Opaque: - __slots__ = () - - captured, budget = vars(telemetry_dev_litellm)["_bounded_responses_native"]( - {"opaque": Opaque()} - ) - - assert captured == {"opaque": None} - assert json.dumps(captured) == '{"opaque": null}' - assert budget.truncated is False - - def test_responses_stream_terminal_item_preserves_truncation_overflow( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -2593,6 +2509,7 @@ def test_responses_stream_terminal_item_preserves_truncation_overflow( content_index=0, delta="fits", ), + completed_without_output(), ] ) monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) @@ -2630,6 +2547,7 @@ def events() -> list[SimpleNamespace]: ) for index in range(8) ], + completed_without_output(), ] sync_inner = ResponsesStream(events()) @@ -2804,16 +2722,19 @@ def test_responses_stream_charges_one_character_fragments_against_item_budget( ) for _ in range(2_000) ] + events.append(completed_without_output()) monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() stream = llm.responses(input="bounded fragments", model="gpt-4.1-mini", stream=True) assert list(stream) == events - fragments = stream._partial_content[("index", 0)][("index", 0)]["_fragments"] - assert len(fragments) <= stream._budget.max_items - assert stream._budget.items_used <= stream._budget.max_items - assert attrs(only_span(memory))["telemetry.dev.capture.truncated"] is True + a = attrs(only_span(memory)) + text = json_attr(a["gen_ai.output.messages"])[0]["text"] + # 2,000 one-character deltas fit the byte bound; only the 1,000-item bound cuts them. + assert set(text) == {"x"} + assert 0 < len(text) <= 1_000 + assert a["telemetry.dev.capture.truncated"] is True async def test_responses_stream_surrogate_deltas_fail_closed_and_finish( @@ -3395,10 +3316,3 @@ def test_provider_attribution_non_openai(memory: SimpleNamespace) -> None: assert azure_flag_attrs["gen_ai.request.model"] == "my-azure-deploy" assert deployment_attrs["gen_ai.provider.name"] == "azure.ai.openai" assert deployment_attrs["gen_ai.request.model"] == "deployment-from-kwarg" - - -def test_cost_maps_to_cost_usd_when_available(memory: SimpleNamespace) -> None: - instrument_litellm() - llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, mock_response="priced") - - assert number_attr(attrs(only_span(memory))["gen_ai.usage.cost"]) > 0 diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index 5b74984..9a13a82 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -524,11 +524,8 @@ def _media_response(response: Any) -> dict[str, Any]: } -def _text_media_response(response: Any, *, capture_output: bool | None = None) -> dict[str, Any]: +def _text_media_response(response: Any, *, capture_output: bool = False) -> dict[str, Any]: fields = _media_response(response) - if capture_output is None: - client = telemetry_dev.get_client() - capture_output = client is not None and client.capture_output if capture_output: text = _string(_field(response, "text")) if text is not None: @@ -1458,26 +1455,12 @@ async def _aclose_hook(*args: Any, **kwargs: Any) -> Any: def _chat_span_capture_policy(handle: telemetry_dev.SpanHandle) -> tuple[bool, bool]: - compatible = cast(Any, handle) - client = getattr(compatible, "_client", None) - state = getattr(compatible, "_state", None) - capture_output = getattr(compatible, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(state, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(client, "capture_output", False) - capture_masked = getattr(compatible, "capture_masked", None) - if not isinstance(capture_masked, bool): - capture_masked = getattr(client, "mask", _OMIT) is not None if client is not None else True - return capture_output, capture_masked + return handle.capture_output, handle.capture_masked def _chat_span_reporter( handle: telemetry_dev.SpanHandle, ) -> Callable[[BaseException], None]: - compatible = cast(Any, handle) - reporter = getattr(compatible, "report_error", None) - client = getattr(compatible, "_client", None) reported = False def report(cause: BaseException) -> None: @@ -1486,12 +1469,7 @@ def report(cause: BaseException) -> None: return reported = True try: - if callable(reporter): - reporter(cause) - return - client_report = getattr(client, "report", None) - if callable(client_report): - client_report("provider instrumentation failed", cause) + handle.report_error(cause) except Exception: return @@ -1668,9 +1646,7 @@ def _record(self, chunk: Any, received_at: float) -> None: self._unterminated_choices, ) if update["has_output"]: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( @@ -1838,9 +1814,7 @@ def _record(self, chunk: Any, received_at: float) -> None: self._unterminated_choices, ) if update["has_output"]: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( @@ -1955,9 +1929,7 @@ def __getattr__(self, name: str) -> Any: def _record(self, event: Any, received_at: float) -> None: if _response_event_has_output(event): - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( @@ -2108,9 +2080,7 @@ def __getattr__(self, name: str) -> Any: def _record(self, event: Any, received_at: float) -> None: if _response_event_has_output(event): - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index 86a6ee6..238c7e6 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -12,8 +12,13 @@ import pytest import telemetry_dev from openai import AsyncOpenAI, OpenAI +from openai.resources.audio.speech import AsyncSpeech, Speech +from openai.resources.audio.transcriptions import AsyncTranscriptions, Transcriptions +from openai.resources.audio.translations import AsyncTranslations, Translations +from openai.resources.batches import AsyncBatches, Batches from openai.resources.chat.completions.completions import AsyncCompletions, Completions from openai.resources.embeddings import AsyncEmbeddings, Embeddings +from openai.resources.images import AsyncImages, Images from openai.resources.responses.responses import AsyncResponses, Responses from openai.types.chat.chat_completion_chunk import ChoiceDelta from opentelemetry.sdk.trace import ReadableSpan @@ -150,6 +155,30 @@ async def __aiter__(self) -> AsyncIterator[bytes]: raise httpx.ReadError(self._message) +class CloseFailingSyncByteStream(httpx.SyncByteStream): + def __init__(self, body: bytes, error: Exception) -> None: + self._body = body + self._error = error + + def __iter__(self) -> Iterator[bytes]: + yield self._body + + def close(self) -> None: + raise self._error + + +class CloseFailingAsyncByteStream(httpx.AsyncByteStream): + def __init__(self, body: bytes, error: Exception) -> None: + self._body = body + self._error = error + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._body + + async def aclose(self) -> None: + raise self._error + + class UnencodableText(str): def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: raise AssertionError("capture must not encode the complete transcript") @@ -177,6 +206,7 @@ def sync_client(handler: SyncHandler) -> OpenAI: return OpenAI( api_key="test", base_url="https://api.test/v1", + max_retries=0, http_client=httpx.Client(transport=httpx.MockTransport(handler)), ) @@ -185,6 +215,7 @@ def async_client(handler: AsyncHandler) -> AsyncOpenAI: return AsyncOpenAI( api_key="test", base_url="https://api.test/v1", + max_retries=0, http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), ) @@ -465,20 +496,24 @@ def test_chat_capture_recursively_omits_binary_media(memory: SimpleNamespace) -> assert "spoken reply" in captured -def test_disabled_responses_capture_does_not_convert_hostile_values(make: Any) -> None: - class Hostile: - def model_dump(self, **_kwargs: Any) -> Any: - raise AssertionError("capture-disabled payload was traversed") +def test_disabled_responses_capture_does_not_read_values(make: Any) -> None: + reads: list[str] = [] + + class Counting(dict[str, Any]): + def items(self) -> Any: + reads.append("items") + return super().items() make(capture_input=False, capture_output=False) responses_request = vars(telemetry_dev_openai)["_responses_request"] responses_response = vars(telemetry_dev_openai)["_responses_response"] - _, request = responses_request({"model": "gpt-4.1", "input": Hostile()}) - response = responses_response({"status": "completed", "output": Hostile()}) + _, request = responses_request({"model": "gpt-4.1", "input": Counting(text="x")}) + response = responses_response({"status": "completed", "output": Counting(text="y")}) assert request["input"] is None assert response["output"] is None + assert reads == [] def test_streamed_image_edit_maps_aggregate_output_tokens_to_image(memory: SimpleNamespace) -> None: @@ -535,59 +570,85 @@ def test_image_stream_is_flagged_only_without_a_terminal_event( assert a["telemetry.dev.capture.truncated"] is True -def test_media_stream_preserves_sync_context_manager_cleanup(memory: SimpleNamespace) -> None: - client = wrap_openai( - sync_client( - lambda _request: sse_response( - [ - { - "type": "image_edit.completed", - "usage": {"input_tokens": 4, "output_tokens": 9, "total_tokens": 13}, - } - ] - ) - ) - ) - - with cast(Any, client.images.edit)( - image=b"image", - prompt="Add telemetry", - model="gpt-image-1", - stream=True, - ) as stream: - assert next(stream).type == "image_edit.completed" - - assert len(memory.span_exporter.get_finished_spans()) == 1 - - -async def test_media_stream_preserves_async_context_manager_cleanup( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("exit_kind", ["clean", "body_error", "delegated_error"]) +async def test_media_stream_context_exit_ends_the_span_once( + memory: SimpleNamespace, async_mode: bool, exit_kind: str ) -> None: - async def handler(_request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "type": "image_edit.completed", - "usage": {"input_tokens": 4, "output_tokens": 9, "total_tokens": 13}, - } - ] - ) + # A non-terminal event leaves the span open, so only the context exit can end it. + event = {"type": "image_edit.partial_image", "b64_json": "QQ==", "partial_image_index": 0} + error = { + "clean": None, + "body_error": ValueError("body failed"), + "delegated_error": RuntimeError("exit failed"), + }[exit_kind] - client = wrap_openai(async_client(handler)) - async with await cast(Any, client.images.edit)( - image=b"image", - prompt="Add telemetry", - model="gpt-image-1", - stream=True, - ) as stream: - assert (await stream.__anext__()).type == "image_edit.completed" - await client.close() + def response() -> httpx.Response: + if not isinstance(error, RuntimeError): + return sse_response([event]) + body = f"data: {json.dumps(event)}\n\n".encode() + stream = ( + CloseFailingAsyncByteStream(body, error) + if async_mode + else CloseFailingSyncByteStream(body, error) + ) + return httpx.Response(200, stream=stream, headers={"Content-Type": "text/event-stream"}) + + async def read_one_event() -> Any: + edit: dict[str, Any] = { + "image": b"image", + "prompt": "Add telemetry", + "model": "gpt-image-1", + "stream": True, + } + if async_mode: - assert len(memory.span_exporter.get_finished_spans()) == 1 + async def handler(_request: httpx.Request) -> httpx.Response: + return response() + + async with await cast(Any, wrap_openai(async_client(handler)).images.edit)( + **edit + ) as stream: + assert (await stream.__anext__()).type == "image_edit.partial_image" + if isinstance(error, ValueError): + raise error + return stream + with cast(Any, wrap_openai(sync_client(lambda _request: response())).images.edit)( + **edit + ) as stream: + assert next(stream).type == "image_edit.partial_image" + if isinstance(error, ValueError): + raise error + return stream + + if error is None: + stream = await read_one_event() + span = only_span(memory) + assert span.status.status_code == StatusCode.UNSET + assert attrs(span)["telemetry.dev.capture.truncated"] is True + assert stream.response.is_closed + else: + with pytest.raises(type(error)) as exc_info: + await read_one_event() + assert exc_info.value is error + span = only_span(memory) + assert span.status.status_code == StatusCode.ERROR + assert [dict(e.attributes or {})["exception.message"] for e in span.events] == [str(error)] -def test_streaming_transcription_records_terminal_text_and_usage(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_streaming_transcription_records_terminal_text_and_usage( + memory: SimpleNamespace, async_mode: bool +) -> None: events = [ + { + "type": "transcript.text.segment", + "id": "segment-1", + "start": 0, + "end": 1, + "speaker": "A", + "text": "partial", + }, {"type": "transcript.text.delta", "delta": "partial"}, { "type": "transcript.text.done", @@ -595,13 +656,26 @@ def test_streaming_transcription_records_terminal_text_and_usage(memory: SimpleN "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, }, ] - client = wrap_openai(sync_client(lambda _request: sse_response(events))) - stream = cast(Any, client.audio.transcriptions.create)( - model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True - ) + if async_mode: - assert [event.type for event in stream] == [event["type"] for event in events] + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True + ) + received = [event.type async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True + ) + received = [event.type for event in stream] + + assert received == [event["type"] for event in events] span = only_span(memory) assert span.name == "transcription gpt-4o-transcribe" a = attrs(span) @@ -685,40 +759,6 @@ def test_streaming_transcription_terminal_text_replaces_truncated_deltas( assert "telemetry.dev.capture.truncated" not in a -async def test_async_streaming_transcription_records_terminal_text_and_usage( - memory: SimpleNamespace, -) -> None: - events = [ - { - "type": "transcript.text.segment", - "id": "segment-1", - "start": 0, - "end": 1, - "speaker": "A", - "text": "partial", - }, - { - "type": "transcript.text.done", - "text": "complete transcript", - "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, - }, - ] - - async def handler(_request: httpx.Request) -> httpx.Response: - return sse_response(events) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.audio.transcriptions.create)( - model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True - ) - assert [event.type async for event in stream] == [event["type"] for event in events] - await client.close() - - a = attrs(only_span(memory)) - assert a["gen_ai.output.messages"] == "complete transcript" - assert a["gen_ai.usage.text.output_tokens"] == 3 - - @pytest.mark.parametrize("async_mode", [False, True]) async def test_interrupted_transcription_stream_records_bounded_partial_text( memory: SimpleNamespace, async_mode: bool @@ -763,7 +803,10 @@ def mask(value: Any, _context: Any) -> Any: memory = make(max_attribute_length=1_000, mask=mask) transcript = "x" * 1_500 - client = wrap_openai(sync_client(lambda _request: json_response({"text": transcript}))) + usage = {"type": "tokens", "input_tokens": 4, "output_tokens": 7, "total_tokens": 11} + client = wrap_openai( + sync_client(lambda _request: json_response({"text": transcript, "usage": usage})) + ) cast(Any, client.audio.transcriptions.create)( model="gpt-4o-transcribe", file=("audio.wav", b"audio") @@ -774,6 +817,7 @@ def mask(value: Any, _context: Any) -> Any: assert len(str(a["gen_ai.output.messages"])) == 1_000 assert str(a["gen_ai.output.messages"]).endswith("...[truncated]") assert "telemetry.dev.capture.truncated" not in a + assert a["gen_ai.usage.text.output_tokens"] == 7 @pytest.mark.parametrize( @@ -993,88 +1037,65 @@ def test_terminal_transcription_capture_is_incrementally_bounded( assert fields["attributes"]["telemetry.dev.capture.truncated"] is True -@pytest.mark.parametrize("capture_output", [False, True]) -def test_text_media_response_uses_the_policy_that_created_its_span( - monkeypatch: pytest.MonkeyPatch, capture_output: bool -) -> None: - ended: list[dict[str, Any]] = [] - reported: list[BaseException] = [] - - def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: - return value - - def end(**fields: Any) -> None: - ended.append(fields) - - handle = SimpleNamespace( - _client=SimpleNamespace(capture_output=capture_output, mask=mask), - _state=SimpleNamespace(capture_output=capture_output), - report_error=reported.append, - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(capture_output=not capture_output, mask=None), - ) - implementation = cast(Any, telemetry_dev_openai) - - implementation._end_mapped_response( - handle, - end, - implementation._text_media_response, - SimpleNamespace(text="sensitive transcript"), - ) - - assert reported == [] - assert ended[-1].get("output") == ("sensitive transcript" if capture_output else None) - assert "attributes" not in ended[-1] - - +@pytest.mark.parametrize("streaming", [False, True]) @pytest.mark.parametrize("async_mode", [False, True]) -async def test_transcription_stream_uses_the_policy_that_created_its_span( - monkeypatch: pytest.MonkeyPatch, async_mode: bool +@pytest.mark.parametrize("capture_output", [False, True]) +async def test_transcription_capture_uses_the_policy_that_created_its_span( + make: Any, + monkeypatch: pytest.MonkeyPatch, + streaming: bool, + async_mode: bool, + capture_output: bool, ) -> None: - ended: list[dict[str, Any]] = [] - reported: list[BaseException] = [] - event = SimpleNamespace(type="transcript.text.done", text="sensitive transcript") + memory = make(capture_output=capture_output) + text_reads = 0 + original_getattribute = BaseModel.__getattribute__ - def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: - return value + def counted_getattribute(self: BaseModel, name: str) -> Any: + nonlocal text_reads + if name == "text": + text_reads += 1 + return original_getattribute(self, name) - def end(**fields: Any) -> None: - ended.append(fields) + monkeypatch.setattr(BaseModel, "__getattribute__", counted_getattribute) - handle = SimpleNamespace( - _client=SimpleNamespace(capture_output=True, mask=mask), - _state=SimpleNamespace(capture_output=True), - end=end, - report_error=reported.append, - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(capture_output=False, mask=None), - ) - implementation = cast(Any, telemetry_dev_openai) + def response() -> httpx.Response: + # The span has started by now; a later global policy must not apply to it. + monkeypatch.setattr( + telemetry_dev, + "get_client", + lambda: SimpleNamespace( + capture_output=not capture_output, capture_input=False, mask=None + ), + ) + if streaming: + return sse_response([{"type": "transcript.text.done", "text": "sensitive transcript"}]) + return json_response({"text": "sensitive transcript"}) if async_mode: - async def source() -> AsyncIterator[Any]: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return response() - stream = implementation._InstrumentedAsyncMediaStream( - source(), handle, implementation._text_media_response + client = wrap_openai(async_client(handler)) + result = await cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=streaming ) - assert await stream.__anext__() is event + if streaming: + _ = [event async for event in result] + await client.close() else: - stream = implementation._InstrumentedMediaStream( - iter([event]), handle, implementation._text_media_response + client = wrap_openai(sync_client(lambda _request: response())) + result = cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=streaming ) - assert next(stream) is event + if streaming: + list(result) - assert reported == [] - assert ended[-1]["output"] == "sensitive transcript" - assert "attributes" not in ended[-1] + a = attrs(only_span(memory)) + assert a.get("gen_ai.output.messages") == ("sensitive transcript" if capture_output else None) + assert (text_reads > 0) is capture_output + assert "telemetry.dev.capture.truncated" not in a @pytest.mark.parametrize("async_mode", [False, True]) @@ -1130,35 +1151,29 @@ async def __anext__(self) -> Any: async def test_failed_media_stream_events_record_errors( memory: SimpleNamespace, async_mode: bool ) -> None: - event = SimpleNamespace( - type="image_generation.failed", - error=SimpleNamespace(code="provider_error", message="provider failed"), - ) - handle = telemetry_dev.start_span("image failed", type="generation") + # The SDK raises APIError for an event with an `error` object, so a failed event + # reaches the wrapper only with top-level code and message. + events = [ + {"type": "image_generation.failed", "code": "provider_error", "message": "provider failed"} + ] if async_mode: - class Inner: - def __init__(self) -> None: - self._remaining = True - - def __aiter__(self) -> Inner: - return self - - async def __anext__(self) -> Any: - if not self._remaining: - raise StopAsyncIteration - self._remaining = False - return event + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) - stream_type = vars(telemetry_dev_openai)["_InstrumentedAsyncMediaStream"] - stream = stream_type(Inner(), handle, vars(telemetry_dev_openai)["_media_response"]) - assert await stream.__anext__() is event + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.images.generate)( + model="gpt-image-1", prompt="otter", stream=True + ) + received = [event.type async for event in stream] + await client.close() else: - stream_type = vars(telemetry_dev_openai)["_InstrumentedMediaStream"] - stream = stream_type(iter([event]), handle, vars(telemetry_dev_openai)["_media_response"]) - assert next(stream) is event + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.images.generate)(model="gpt-image-1", prompt="otter", stream=True) + received = [event.type for event in stream] + assert received == ["image_generation.failed"] span = only_span(memory) assert span.status.status_code == StatusCode.ERROR assert attrs(span)["error.type"] == "RuntimeError" @@ -1176,79 +1191,19 @@ class Event: def text(self) -> str: raise AssertionError("capture-disabled output was read") - class Handle: - capture_output = False - capture_masked = False - max_attribute_length = 5 - - def end(self, **_fields: Any) -> None: - pass - - make(capture_output=False) - stream_type = vars(telemetry_dev_openai)["_InstrumentedMediaStream"] - stream = stream_type( - iter([Event()]), cast(Any, Handle()), vars(telemetry_dev_openai)["_text_media_response"] - ) - - assert next(stream).type == "transcript.text.done" - - -@pytest.mark.parametrize("delegated", [False, True]) -def test_media_stream_context_exit_records_body_and_delegated_errors(delegated: bool) -> None: errors: list[BaseException] = [] - - class Handle: - def end(self, **fields: Any) -> None: - errors.append(fields["error"]) - - class Inner: - def __exit__(self, _type: Any, _exc: Any, _tb: Any) -> bool: - if delegated: - raise RuntimeError("exit failed") - return False - + memory = make(capture_output=False, on_error=errors.append) + handle = telemetry_dev.start_span("transcription gpt-4o-transcribe", type="generation") stream_type = vars(telemetry_dev_openai)["_InstrumentedMediaStream"] stream = stream_type( - Inner(), cast(Any, Handle()), vars(telemetry_dev_openai)["_media_response"] + iter([Event()]), handle, vars(telemetry_dev_openai)["_text_media_response"] ) - body_error = ValueError("body failed") - if delegated: - with pytest.raises(RuntimeError, match="exit failed"): - stream.__exit__(None, None, None) - assert str(errors[0]) == "exit failed" - else: - assert stream.__exit__(ValueError, body_error, None) is False - assert errors == [body_error] - - -@pytest.mark.parametrize("delegated", [False, True]) -async def test_async_media_stream_context_exit_records_body_and_delegated_errors( - delegated: bool, -) -> None: - errors: list[BaseException] = [] - - class Handle: - def end(self, **fields: Any) -> None: - errors.append(fields["error"]) - class Inner: - async def __aexit__(self, _type: Any, _exc: Any, _tb: Any) -> bool: - if delegated: - raise RuntimeError("exit failed") - return False - - stream_type = vars(telemetry_dev_openai)["_InstrumentedAsyncMediaStream"] - stream = stream_type( - Inner(), cast(Any, Handle()), vars(telemetry_dev_openai)["_media_response"] - ) - body_error = ValueError("body failed") - if delegated: - with pytest.raises(RuntimeError, match="exit failed"): - await stream.__aexit__(None, None, None) - assert str(errors[0]) == "exit failed" - else: - assert await stream.__aexit__(ValueError, body_error, None) is False - assert errors == [body_error] + assert next(stream).type == "transcript.text.done" + assert errors == [] + a = attrs(only_span(memory)) + assert "telemetry.dev.capture.truncated" not in a + assert a["gen_ai.usage.text.output_tokens"] == 2 def test_chat_completion_maps_native_messages_usage_finish_provider_and_sampling( @@ -1442,7 +1397,7 @@ def handler(request: httpx.Request) -> httpx.Response: { "index": 1, "message": {"role": "assistant", "content": "second"}, - "finish_reason": "stop", + "finish_reason": "length", }, ] ) @@ -1499,6 +1454,7 @@ def handler(request: httpx.Request) -> httpx.Response: {"role": "assistant", "content": "first"}, {"role": "assistant", "content": "second"}, ] + assert attrs(choices_span)["gen_ai.response.finish_reasons"] == ("stop", "length") def test_chat_streaming_injects_usage_when_opted_in_and_filters_synthetic_chunk( @@ -1705,54 +1661,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: wrapped.close() assert len(received) == len(events) assert len(calls) == 6 - assert memory.metric_reader.get_metrics_data() is not None - - -def test_chat_streaming_reconstructs_legacy_function_call() -> None: - implementation = cast(Any, telemetry_dev_openai) - states: dict[int, Any] = {} - output_budget = implementation._chat_capture_budget(reserve_output_list=True) - finish_reason_budget = implementation._chat_capture_budget() - events = [ - { - "choices": [ - { - "index": 0, - "delta": {"function_call": {"name": "lookup", "arguments": '{"city":'}}, - } - ] - }, - { - "choices": [ - { - "index": 0, - "delta": {"function_call": {"arguments": '"Paris"}'}}, - "finish_reason": "stop", - } - ] - }, - ] - - for event in events: - implementation._record_chat_chunk( - event, - states, - {}, - {}, - set(), - output_budget, - finish_reason_budget, - True, - ) - - assert implementation._chat_output(states) == [ - { - "role": "assistant", - "content": None, - "function_call": {"name": "lookup", "arguments": '{"city":"Paris"}'}, - } - ] - assert output_budget.truncated is False @pytest.mark.parametrize( @@ -1930,82 +1838,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: assert only_span(memory).status.status_code == StatusCode.UNSET -@pytest.mark.parametrize( - "stream_kind", ["sync-chat", "async-chat", "sync-responses", "async-responses"] -) -async def test_streaming_preserves_tracing_without_output_chunk_support( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, stream_kind: str -) -> None: - monkeypatch.delattr(telemetry_dev.SpanHandle, "record_output_chunk") - - if stream_kind == "sync-chat": - - def sync_chat_handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) - - client = wrap_openai(sync_client(sync_chat_handler)) - received = list( - cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - ) - client.close() - elif stream_kind == "async-chat": - - async def async_chat_handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) - - client = wrap_openai(async_client(async_chat_handler)) - stream = await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - received = [chunk async for chunk in stream] - await client.close() - else: - delta = { - "type": "response.output_text.delta", - "sequence_number": 0, - "item_id": "item_old_core", - "output_index": 0, - "content_index": 0, - "delta": "Still traced.", - } - response = { - "type": "response.completed", - "sequence_number": 1, - "response": response_payload(response_id="resp_old_core", text="Still traced."), - } - events = [("response.output_text.delta", delta), ("response.completed", response)] - if stream_kind == "sync-responses": - - def sync_responses_handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(events) - - client = wrap_openai(sync_client(sync_responses_handler)) - received = list( - cast(Any, client.responses.create)(model="gpt-4o-mini", input="stream", stream=True) - ) - client.close() - else: - - async def async_responses_handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(events) - - client = wrap_openai(async_client(async_responses_handler)) - stream = await cast(Any, client.responses.create)( - model="gpt-4o-mini", input="stream", stream=True - ) - received = [event async for event in stream] - await client.close() - assert len(received) == 2 - assert received[0].delta == "Still traced." - - assert received - span = only_span(memory) - assert span.status.status_code == StatusCode.UNSET - assert attrs(span)["gen_ai.response.id"] in {"chatcmpl_stream", "resp_old_core"} - - def test_chat_streaming_default_leaves_request_unchanged(memory: SimpleNamespace) -> None: requests: list[dict[str, Any]] = [] @@ -2028,8 +1860,9 @@ def handler(request: httpx.Request) -> httpx.Response: assert "gen_ai.usage.total_tokens" not in a -def test_chat_streaming_bounds_retained_state_without_dropping_metadata( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_streaming_bounds_retained_state_without_dropping_metadata( + memory: SimpleNamespace, async_mode: bool ) -> None: source_events = oversized_chat_stream_events() source_content = "".join( @@ -2038,25 +1871,34 @@ def test_chat_streaming_bounds_retained_state_without_dropping_metadata( if event["choices"] ) - def handler(request: httpx.Request) -> httpx.Response: - return sse_response(source_events) + if async_mode: - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - delivered = list(stream) - budget = stream._budget - state = stream._states[0] + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(source_events) + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + delivered = [chunk async for chunk in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(source_events))) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + delivered = list(stream) assert len(delivered) == len(source_events) - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert source_content.startswith(state.content) - assert len(state.content) < len(source_content) - assert state.role is None - assert not state.content.endswith("not-retained") a = attrs(only_span(memory)) + output = json.loads(str(a["gen_ai.output.messages"])) + assert len(output) == 1 + assert output[0]["role"] == "assistant" + assert set(output[0]) == {"role", "content"} + content = output[0]["content"] + assert source_content.startswith(content) + assert len(content) < len(source_content) + assert not content.endswith("not-retained") assert len(str(a["gen_ai.output.messages"]).encode()) <= 48 * 1024 assert a["gen_ai.response.finish_reasons"] == ("stop",) assert a["gen_ai.usage.total_tokens"] == 7 @@ -2116,21 +1958,32 @@ async def handler(_request: httpx.Request) -> httpx.Response: @pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("capture_output", [False, True]) async def test_multi_choice_finish_reasons_survive_a_zero_content_limit( - make: Any, async_mode: bool + make: Any, async_mode: bool, capture_output: bool ) -> None: - memory = make(max_attribute_length=0, capture_input=False, capture_output=False) + memory = make(max_attribute_length=0, capture_input=False, capture_output=capture_output) + base = { + "id": "chatcmpl_two", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o", + } events: list[dict[str, Any]] = [ { - "id": "chatcmpl_two", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o", + **base, + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "first"}}, + {"index": 1, "delta": {"role": "assistant", "content": "second"}}, + ], + }, + { + **base, "choices": [ {"index": 0, "delta": {}, "finish_reason": "stop"}, {"index": 1, "delta": {}, "finish_reason": "length"}, ], - } + }, ] if async_mode: @@ -2151,7 +2004,7 @@ async def handler(_request: httpx.Request) -> httpx.Response: ) delivered = list(stream) - assert len(delivered) == 1 + assert len(delivered) == 2 a = attrs(only_span(memory)) assert a["gen_ai.response.finish_reasons"] == ("stop", "length") assert "telemetry.dev.capture.truncated" not in a @@ -2315,7 +2168,6 @@ async def async_handler(_request: httpx.Request) -> httpx.Response: client.close() assert len(delivered) == 1 - assert len(stream._states) == 333 a = attrs(only_span(memory)) output = json.loads(str(a["gen_ai.output.messages"])) finish_reasons = cast(Sequence[str], a["gen_ai.response.finish_reasons"]) @@ -2325,26 +2177,43 @@ async def async_handler(_request: httpx.Request) -> httpx.Response: assert a["telemetry.dev.capture.truncated"] is True -def test_chat_streaming_reclaims_finish_reason_items_on_replacement(make: Any) -> None: - memory = make(capture_output=False) - events: list[dict[str, Any]] = [ - { - "id": "chatcmpl_finish_reasons", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - {"index": index, "delta": {}, "finish_reason": "stop"} for index in range(200) +def finish_reason_chunk(choices: list[dict[str, Any]]) -> dict[str, Any]: + return { + "id": "chatcmpl_finish_reasons", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": choices, + } + + +@pytest.mark.parametrize( + ("events", "expected"), + [ + pytest.param( + [ + finish_reason_chunk( + [{"index": index, "delta": {}, "finish_reason": "stop"} for index in range(200)] + ), + finish_reason_chunk([{"index": 0, "delta": {}, "finish_reason": "length"}]), ], - }, - { - "id": "chatcmpl_finish_reasons", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini", - "choices": [{"index": 0, "delta": {}, "finish_reason": "length"}], - }, - ] + ("length",) + ("stop",) * 199, + id="replacement-reclaims-items", + ), + pytest.param( + [ + finish_reason_chunk([{"index": 0, "delta": {}, "finish_reason": "x" * 70_000}]), + finish_reason_chunk([{"index": 0, "delta": {}, "finish_reason": "stop"}]), + ], + ("stop",), + id="fitting-reason-clears-rejected-oversized-reason", + ), + ], +) +def test_chat_streaming_reclaims_finish_reason_items_on_replacement( + make: Any, events: list[dict[str, Any]], expected: tuple[str, ...] +) -> None: + memory = make(capture_output=False) client = wrap_openai(sync_client(lambda _request: sse_response(events))) list( @@ -2354,9 +2223,7 @@ def test_chat_streaming_reclaims_finish_reason_items_on_replacement(make: Any) - ) a = attrs(only_span(memory)) - finish_reasons = cast(Sequence[str], a["gen_ai.response.finish_reasons"]) - assert len(finish_reasons) == 200 - assert finish_reasons[0] == "length" + assert a["gen_ai.response.finish_reasons"] == expected assert "gen_ai.output.messages" not in a assert "telemetry.dev.capture.truncated" not in a @@ -2388,49 +2255,6 @@ def test_chat_streaming_retains_exact_multibyte_finish_reason_budget(make: Any) assert "telemetry.dev.capture.truncated" not in a -def test_chat_streaming_clears_recovered_finish_reason_truncation(memory: SimpleNamespace) -> None: - implementation = cast(Any, telemetry_dev_openai) - states: dict[int, Any] = {} - finish_reason_states: dict[int, str] = {} - finish_reason_reservations: dict[int, tuple[int, int]] = {} - rejected_finish_reasons: set[int] = set() - output_budget = implementation._chat_capture_budget(reserve_output_list=True) - finish_reason_budget = implementation._chat_capture_budget() - - implementation._record_chat_chunk( - {"choices": [{"index": 0, "delta": {}, "finish_reason": "x" * 70_000}]}, - states, - finish_reason_states, - finish_reason_reservations, - rejected_finish_reasons, - output_budget, - finish_reason_budget, - False, - ) - implementation._record_chat_chunk( - {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, - states, - finish_reason_states, - finish_reason_reservations, - rejected_finish_reasons, - output_budget, - finish_reason_budget, - False, - ) - - assert finish_reason_states == {0: "stop"} - assert rejected_finish_reasons == set() - partial = implementation._chat_partial( - states, - finish_reason_states, - None, - False, - output_budget.truncated or finish_reason_budget.truncated or bool(rejected_finish_reasons), - ) - assert finish_reason_budget.truncated is False - assert partial["attributes"] is None - - def test_chat_streaming_incrementally_accounts_for_many_tiny_deltas( memory: SimpleNamespace, ) -> None: @@ -2915,12 +2739,17 @@ def test_chat_streaming_retains_exact_content_for_all_fragment_accumulators() -> @pytest.mark.parametrize("async_mode", [False, True]) async def test_chat_streaming_reconstructs_legacy_function_calls_and_custom_tools( - make: Any, async_mode: bool + memory: SimpleNamespace, async_mode: bool ) -> None: - memory = make() - implementation = cast(Any, telemetry_dev_openai) - events = [ + base = { + "id": "chatcmpl_legacy", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + } + events: list[dict[str, Any]] = [ { + **base, "choices": [ { "index": 0, @@ -2936,9 +2765,10 @@ async def test_chat_streaming_reconstructs_legacy_function_calls_and_custom_tool ], }, } - ] + ], }, { + **base, "choices": [ { "index": 0, @@ -2948,31 +2778,35 @@ async def test_chat_streaming_reconstructs_legacy_function_calls_and_custom_tool }, "finish_reason": "stop", } - ] + ], }, ] - handle = telemetry_dev.start_span( - "chat gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) if async_mode: - async def source() -> AsyncIterator[Any]: - for event in events: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) - stream = implementation._InstrumentedAsyncStream( - source(), handle, False, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) - delivered = [event async for event in stream] + delivered = [chunk async for chunk in stream] + await client.close() else: - stream = implementation._InstrumentedStream( - iter(events), handle, False, time.perf_counter() + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) delivered = list(stream) - assert delivered == events - assert json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) == [ + assert [chunk.choices[0].delta.function_call.arguments for chunk in delivered] == [ + '{"city":', + '"Paris"}', + ] + a = attrs(only_span(memory)) + assert "telemetry.dev.capture.truncated" not in a + assert json.loads(str(a["gen_ai.output.messages"])) == [ { "role": "assistant", "content": None, @@ -3284,83 +3118,89 @@ async def test_chat_stream_clean_eof_without_finish_reason_is_incomplete_when_ou make: Any, async_mode: bool ) -> None: memory = make(capture_output=False) - implementation = cast(Any, telemetry_dev_openai) - events = [{"choices": [{"index": 0, "delta": {"content": "partial"}}]}] - handle = telemetry_dev.start_span( - "chat gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) + events = [ + { + "id": "chatcmpl_partial", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {"content": "partial"}}], + } + ] if async_mode: - async def source() -> AsyncIterator[Any]: - for event in events: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) - stream = implementation._InstrumentedAsyncStream( - source(), handle, False, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) - delivered = [event async for event in stream] + delivered = [chunk async for chunk in stream] + await client.close() else: - stream = implementation._InstrumentedStream( - iter(events), handle, False, time.perf_counter() + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) delivered = list(stream) - assert delivered == events + assert len(delivered) == 1 a = attrs(only_span(memory)) assert "gen_ai.output.messages" not in a assert "gen_ai.response.finish_reasons" not in a assert a["telemetry.dev.capture.truncated"] is True -def test_chat_stream_uses_the_client_policy_that_created_its_span( - make: Any, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_stream_uses_the_client_policy_that_created_its_span( + make: Any, monkeypatch: pytest.MonkeyPatch, async_mode: bool ) -> None: def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: return value - original = make( - capture_output=True, - mask=mask, - max_attribute_length=256, - ) - handle = telemetry_dev.start_span( - "chat gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(capture_output=False, mask=None, max_attribute_length=1), - ) - old_handle = SimpleNamespace( - _client=cast(Any, handle)._client, - _state=cast(Any, handle)._state, - end=handle.end, - update=handle.update, - ) - implementation = cast(Any, telemetry_dev_openai) - stream = implementation._InstrumentedStream( - iter( - [ - { - "choices": [ - { - "index": 0, - "delta": {"content": "x" * 100}, - "finish_reason": "stop", - } - ] - } - ] - ), - old_handle, - False, - time.perf_counter(), - ) - list(stream) - - a = attrs(only_span(original)) - assert json.loads(str(a["gen_ai.output.messages"])) == [ + memory = make(capture_output=True, mask=mask, max_attribute_length=256) + event = { + "id": "chatcmpl_policy", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {"content": "x" * 100}, "finish_reason": "stop"}], + } + + def response() -> httpx.Response: + # The span has started by now; a later global policy must not apply to it. + monkeypatch.setattr( + telemetry_dev, + "get_client", + lambda: SimpleNamespace( + capture_output=False, capture_input=False, mask=None, max_attribute_length=1 + ), + ) + return sse_response([event]) + + if async_mode: + + async def handler(_request: httpx.Request) -> httpx.Response: + return response() + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + _ = [chunk async for chunk in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: response())) + list( + cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + ) + + a = attrs(only_span(memory)) + assert json.loads(str(a["gen_ai.output.messages"])) == [ {"role": "assistant", "content": "x" * 100} ] assert "telemetry.dev.capture.truncated" not in a @@ -3477,10 +3317,13 @@ def handler(request: httpx.Request) -> httpx.Response: assert requests[1]["stream"] is True -def test_responses_stream_existing_response_is_instrumented(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_existing_response_is_instrumented( + memory: SimpleNamespace, async_mode: bool +) -> None: requests: list[httpx.Request] = [] - def handler(request: httpx.Request) -> httpx.Response: + def response(request: httpx.Request) -> httpx.Response: requests.append(request) events: list[tuple[str, Mapping[str, Any]]] = [ ( @@ -3502,9 +3345,19 @@ def handler(request: httpx.Request) -> httpx.Response: ] return named_sse_response(events) - client = wrap_openai(sync_client(handler)) - with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: - events = list(stream) + if async_mode: + + async def handler(request: httpx.Request) -> httpx.Response: + return response(request) + + client = wrap_openai(async_client(handler)) + async with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: + events = [event async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(response)) + with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: + events = list(stream) assert [event.type for event in events] == ["response.created", "response.completed"] assert requests[0].method == "GET" @@ -3521,14 +3374,15 @@ def handler(request: httpx.Request) -> httpx.Response: ) -def test_responses_stream_bounds_output_without_dropping_terminal_metadata( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_bounds_output_without_dropping_terminal_metadata( + memory: SimpleNamespace, async_mode: bool ) -> None: retained = response_payload(response_id="resp_bounded", text="prefix") retained["status"] = "in_progress" completed = response_payload(response_id="resp_bounded", text="x" * 70_000) - def handler(request: httpx.Request) -> httpx.Response: + def response() -> httpx.Response: events: list[tuple[str, Mapping[str, Any]]] = [ ( "response.created", @@ -3549,10 +3403,25 @@ def handler(request: httpx.Request) -> httpx.Response: ] return named_sse_response(events) - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="bounded", stream=True) - assert [event.type for event in stream] == ["response.created", "response.completed"] + if async_mode: + async def handler(_request: httpx.Request) -> httpx.Response: + return response() + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="bounded", stream=True + ) + received = [event.type async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: response())) + stream = cast(Any, client.responses.create)( + model="gpt-4o-mini", input="bounded", stream=True + ) + received = [event.type for event in stream] + + assert received == ["response.created", "response.completed"] a = attrs(only_span(memory)) assert a["gen_ai.response.id"] == "resp_bounded" assert a["gen_ai.response.model"] == "gpt-4o-2024-08-06" @@ -3592,51 +3461,68 @@ def empty_created_then_oversized_terminal() -> list[tuple[str, dict[str, Any]]]: ] -def assert_empty_created_output_not_reported(memory: SimpleNamespace) -> None: - a = attrs(only_span(memory)) - assert "gen_ai.output.messages" not in a - assert a["telemetry.dev.capture.truncated"] is True +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_does_not_report_empty_created_output_for_truncated_terminal( + memory: SimpleNamespace, async_mode: bool +) -> None: + events = empty_created_then_oversized_terminal() + if async_mode: -def test_responses_stream_does_not_report_empty_created_output_for_truncated_terminal( - memory: SimpleNamespace, -) -> None: - def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response(empty_created_then_oversized_terminal()) + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) - list( - cast(Any, wrap_openai(sync_client(handler)).responses.create)( + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( model="gpt-4o-mini", input="x", stream=True ) - ) - assert_empty_created_output_not_reported(memory) + delivered = [event async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + delivered = list( + cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + ) + + assert len(delivered) == len(events) + a = attrs(only_span(memory)) + assert "gen_ai.output.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True -def test_responses_stream_fitting_terminal_clears_prior_truncation(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_fitting_terminal_clears_prior_truncation( + memory: SimpleNamespace, async_mode: bool +) -> None: retained = response_payload(text="partial") retained["status"] = "in_progress" oversized = response_payload(text="x" * 70_000) oversized["status"] = "in_progress" completed = response_payload(text="final") + events: list[tuple[str, Mapping[str, Any]]] = [ + ("response.created", {"type": "response.created", "response": retained}), + ("response.in_progress", {"type": "response.in_progress", "response": oversized}), + ("response.completed", {"type": "response.completed", "response": completed}), + ] - def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response( - [ - ("response.created", {"type": "response.created", "response": retained}), - ( - "response.in_progress", - {"type": "response.in_progress", "response": oversized}, - ), - ("response.completed", {"type": "response.completed", "response": completed}), - ] - ) + if async_mode: - list( - cast(Any, wrap_openai(sync_client(handler)).responses.create)( + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( model="gpt-4o-mini", input="x", stream=True ) - ) + delivered = [event async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + delivered = list( + cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + ) + assert len(delivered) == len(events) a = attrs(only_span(memory)) assert json.loads(str(a["gen_ai.output.messages"]))[0]["content"][0]["text"] == "final" assert "telemetry.dev.capture.truncated" not in a @@ -3705,32 +3591,28 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: retained = response_payload(response_id="resp_masked_terminal", text="public") retained["status"] = "in_progress" terminal = response_payload(response_id="resp_masked_terminal", text=f"SECRET{'x' * 70_000}") - events = [ - {"type": "response.in_progress", "response": retained}, - {"type": "response.completed", "response": terminal}, + events: list[tuple[str, Mapping[str, Any]]] = [ + ("response.in_progress", {"type": "response.in_progress", "response": retained}), + ("response.completed", {"type": "response.completed", "response": terminal}), ] - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) if async_mode: - async def source() -> AsyncIterator[Any]: - for event in events: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) - stream = implementation._InstrumentedAsyncResponsesStream( - source(), handle, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="x", stream=True ) - delivered = [event async for event in stream] + delivered = [event.type async for event in stream] + await client.close() else: - stream = implementation._InstrumentedResponsesStream( - iter(events), handle, time.perf_counter() - ) - delivered = list(stream) + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + delivered = [event.type for event in stream] - assert delivered == events + assert delivered == ["response.in_progress", "response.completed"] a = attrs(only_span(memory)) assert "gen_ai.output.messages" not in a assert a["telemetry.dev.capture.truncated"] is True @@ -3835,77 +3717,44 @@ async def source() -> AsyncIterator[Any]: @pytest.mark.parametrize("async_mode", [False, True]) -@pytest.mark.parametrize("stream_limited", [False, True]) +@pytest.mark.parametrize( + ("max_attribute_length", "text_length", "kept"), + [(128, 1_000, True), (100 * 1024, 60 * 1024, False)], +) async def test_responses_stream_retention_follows_the_stream_ceiling_not_the_cap( - make: Any, monkeypatch: pytest.MonkeyPatch, async_mode: bool, stream_limited: bool + make: Any, async_mode: bool, max_attribute_length: int, text_length: int, kept: bool ) -> None: - if stream_limited: - monkeypatch.setattr(telemetry_dev_openai, "_CHAT_STREAM_CAPTURE_MAX_BYTES", 128) - memory = make() - else: - memory = make(max_attribute_length=128) - terminal = response_payload(response_id="resp_small_limit", text="x" * 1_000) - event = {"type": "response.completed", "response": terminal} - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) + memory = make(max_attribute_length=max_attribute_length) + terminal = response_payload(response_id="resp_stream_ceiling", text="x" * text_length) + events: list[tuple[str, Mapping[str, Any]]] = [ + ("response.completed", {"type": "response.completed", "response": terminal}) + ] if async_mode: - async def source() -> AsyncIterator[Any]: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) - stream = implementation._InstrumentedAsyncResponsesStream( - source(), handle, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="x", stream=True ) - delivered = [item async for item in stream] + delivered = [event.type async for event in stream] + await client.close() else: - stream = implementation._InstrumentedResponsesStream( - iter([event]), handle, time.perf_counter() - ) - delivered = list(stream) + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + delivered = [event.type for event in stream] - assert delivered == [event] + assert delivered == ["response.completed"] a = attrs(only_span(memory)) - if stream_limited: - assert "gen_ai.output.messages" not in a - assert a["telemetry.dev.capture.truncated"] is True - else: + if kept: assert len(str(a["gen_ai.output.messages"])) == 128 assert str(a["gen_ai.output.messages"]).endswith("...[truncated]") assert "telemetry.dev.capture.truncated" not in a - - -@pytest.mark.parametrize("async_mode", [False, True]) -async def test_responses_stream_honors_hard_capture_limit(make: Any, async_mode: bool) -> None: - memory = make(max_attribute_length=100 * 1024) - terminal = response_payload(response_id="resp_hard_limit", text="x" * (60 * 1024)) - event = {"type": "response.completed", "response": terminal} - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) - - if async_mode: - - async def source() -> AsyncIterator[Any]: - yield event - - stream = implementation._InstrumentedAsyncResponsesStream( - source(), handle, time.perf_counter() - ) - delivered = [item async for item in stream] else: - stream = implementation._InstrumentedResponsesStream( - iter([event]), handle, time.perf_counter() - ) - delivered = list(stream) - - assert delivered == [event] - a = attrs(only_span(memory)) - assert "gen_ai.output.messages" not in a - assert a["telemetry.dev.capture.truncated"] is True + assert "gen_ai.output.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True @pytest.mark.parametrize("async_mode", [False, True]) @@ -3920,57 +3769,52 @@ def identity_mask(value: Any, _context: Any) -> Any: retained_response = response_payload(response_id="resp_interrupted", text="partial") retained_response["status"] = "in_progress" retained = {"type": "response.in_progress", "response": retained_response} - error_event = {"type": "error", "code": "stream_error", "message": "interrupted"} terminal = { "type": "response.completed", "response": {**retained_response, "status": "completed"}, } - stream_error = RuntimeError("responses stream interrupted") - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) + + def response() -> httpx.Response: + if mode == "eof": + return sse_response([retained]) + if mode == "early_close": + return sse_response([retained, terminal]) + if mode == "error_event": + error_event = {"type": "error", "code": "stream_error", "message": "interrupted"} + return sse_response([retained, error_event]) + if async_mode: + return failing_async_sse_response(retained, "interrupted") + return failing_sync_sse_response(retained, "interrupted") if async_mode: - async def async_source() -> AsyncIterator[Any]: - yield retained - if mode == "error_event": - yield error_event - elif mode == "throw": - raise stream_error - elif mode == "early_close": - yield terminal + async def handler(_request: httpx.Request) -> httpx.Response: + return response() - stream = implementation._InstrumentedAsyncResponsesStream( - async_source(), handle, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="x", stream=True ) if mode == "early_close": - assert await stream.__anext__() is retained + assert (await stream.__anext__()).type == "response.in_progress" await stream.close() elif mode == "throw": - with pytest.raises(RuntimeError, match="responses stream interrupted"): - [item async for item in stream] + with pytest.raises(Exception) as exc_info: + _ = [event async for event in stream] + assert_stream_transport_error(exc_info.value, "interrupted") else: - [item async for item in stream] + _ = [event async for event in stream] + await client.close() else: - - def source() -> Iterator[Any]: - yield retained - if mode == "error_event": - yield error_event - elif mode == "throw": - raise stream_error - elif mode == "early_close": - yield terminal - - stream = implementation._InstrumentedResponsesStream(source(), handle, time.perf_counter()) + client = wrap_openai(sync_client(lambda _request: response())) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) if mode == "early_close": - assert next(stream) is retained + assert next(stream).type == "response.in_progress" stream.close() elif mode == "throw": - with pytest.raises(RuntimeError, match="responses stream interrupted"): + with pytest.raises(Exception) as exc_info: list(stream) + assert_stream_transport_error(exc_info.value, "interrupted") else: list(stream) @@ -4104,42 +3948,6 @@ async def handler(request: httpx.Request) -> httpx.Response: assert attrs(stream_span)["gen_ai.usage.total_tokens"] == 7 -async def test_async_chat_streaming_bounds_retained_state_without_dropping_metadata( - memory: SimpleNamespace, -) -> None: - source_events = oversized_chat_stream_events() - source_content = "".join( - cast(str, event["choices"][0]["delta"].get("content", "")) - for event in source_events - if event["choices"] - ) - - async def handler(request: httpx.Request) -> httpx.Response: - return sse_response(source_events) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - delivered = [chunk async for chunk in stream] - budget = stream._budget - state = stream._states[0] - await client.close() - - assert len(delivered) == len(source_events) - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert source_content.startswith(state.content) - assert len(state.content) < len(source_content) - assert state.role is None - assert not state.content.endswith("not-retained") - a = attrs(only_span(memory)) - assert len(str(a["gen_ai.output.messages"]).encode()) <= 48 * 1024 - assert a["gen_ai.response.finish_reasons"] == ("stop",) - assert a["gen_ai.usage.total_tokens"] == 7 - assert a["telemetry.dev.capture.truncated"] is True - - async def test_async_responses_create_and_embeddings_map_usage_like_sync( memory: SimpleNamespace, ) -> None: @@ -4186,35 +3994,27 @@ async def handler(request: httpx.Request) -> httpx.Response: assert embedding_attrs["gen_ai.usage.total_tokens"] == 6 +@pytest.mark.parametrize("async_mode", [False, True]) @pytest.mark.parametrize("method", ["retrieve", "cancel"]) -def test_batch_positional_id_is_recorded_when_sync_request_errors( - memory: SimpleNamespace, method: str +async def test_batch_positional_id_is_recorded_when_request_errors( + memory: SimpleNamespace, async_mode: bool, method: str ) -> None: - def handler(_request: httpx.Request) -> httpx.Response: + def response() -> httpx.Response: return httpx.Response(500, json={"error": {"message": "batch failed"}}) - client = wrap_openai(sync_client(handler)) - - with pytest.raises(openai.APIStatusError): - getattr(client.batches, method)("batch_positional") - - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["openai.batch.id"] == "batch_positional" - - -@pytest.mark.parametrize("method", ["retrieve", "cancel"]) -async def test_batch_positional_id_is_recorded_when_async_request_errors( - memory: SimpleNamespace, method: str -) -> None: - async def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(500, json={"error": {"message": "batch failed"}}) + if async_mode: - client = wrap_openai(async_client(handler)) + async def handler(_request: httpx.Request) -> httpx.Response: + return response() - with pytest.raises(openai.APIStatusError): - await getattr(client.batches, method)("batch_positional") - await client.close() + client = wrap_openai(async_client(handler)) + with pytest.raises(openai.APIStatusError): + await getattr(client.batches, method)("batch_positional") + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: response())) + with pytest.raises(openai.APIStatusError): + getattr(client.batches, method)("batch_positional") span = only_span(memory) assert span.status.status_code == StatusCode.ERROR @@ -4233,41 +4033,6 @@ async def handler(_request: httpx.Request) -> httpx.Response: assert memory.span_exporter.get_finished_spans() == () -async def test_async_chat_stream_iteration_error_records_one_error_span( - memory: SimpleNamespace, -) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return failing_async_sse_response(terminal_chat_stream_event(), "async stream broke") - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - first = await stream.__anext__() - assert first.choices[0].delta.content == "Hello" - - with pytest.raises(Exception) as exc_info: - await stream.__anext__() - await client.close() - - assert_stream_transport_error(exc_info.value, "async stream broke") - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - a = attrs(span) - assert a["error.type"] == type(exc_info.value).__name__ - assert json.loads(str(a["gen_ai.output.messages"])) == [ - {"role": "assistant", "content": "Hello"} - ] - assert a["gen_ai.response.finish_reasons"] == ("stop",) - assert a["telemetry.dev.capture.truncated"] is True - events = list(span.events) - assert len(events) == 1 - event_attrs = dict(events[0].attributes or {}) - assert events[0].name == "exception" - assert event_attrs["exception.type"] == type(exc_info.value).__name__ - assert event_attrs["exception.message"] == str(exc_info.value) - - @pytest.mark.parametrize("async_mode", [False, True]) async def test_chat_stream_terminal_chunk_then_early_close_is_incomplete( memory: SimpleNamespace, async_mode: bool @@ -4325,21 +4090,37 @@ def test_chat_stream_plain_for_break_ends_span_once(memory: SimpleNamespace) -> client.close() -def test_chat_stream_iteration_error_records_one_error_span(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return failing_sync_sse_response(terminal_chat_stream_event(), "sync stream broke") +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_stream_iteration_error_records_one_error_span( + memory: SimpleNamespace, async_mode: bool +) -> None: + event = terminal_chat_stream_event() - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - first = next(stream) - assert first.choices[0].delta.content == "Hello" + if async_mode: - with pytest.raises(Exception) as exc_info: - next(stream) + async def handler(_request: httpx.Request) -> httpx.Response: + return failing_async_sse_response(event, "stream broke") - assert_stream_transport_error(exc_info.value, "sync stream broke") + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + assert (await stream.__anext__()).choices[0].delta.content == "Hello" + with pytest.raises(Exception) as exc_info: + await stream.__anext__() + await client.close() + else: + client = wrap_openai( + sync_client(lambda _request: failing_sync_sse_response(event, "stream broke")) + ) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + assert next(stream).choices[0].delta.content == "Hello" + with pytest.raises(Exception) as exc_info: + next(stream) + + assert_stream_transport_error(exc_info.value, "stream broke") span = only_span(memory) assert span.status.status_code == StatusCode.ERROR a = attrs(span) @@ -4398,23 +4179,31 @@ def handler(request: httpx.Request) -> httpx.Response: assert span.status.status_code == StatusCode.ERROR -def test_responses_stream_bare_error_event_sets_error_status(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "type": "error", - "code": "server_error", - "message": "model blew up", - "sequence_number": 0, - } - ] - ) +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_bare_error_event_sets_error_status( + memory: SimpleNamespace, async_mode: bool +) -> None: + events = [ + {"type": "error", "code": "server_error", "message": "model blew up", "sequence_number": 0} + ] - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="fail", stream=True) + if async_mode: + + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="fail", stream=True + ) + received = [event.type async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="fail", stream=True) + received = [event.type for event in stream] - assert [event.type for event in stream] == ["error"] + assert received == ["error"] assert only_span(memory).status.status_code == StatusCode.ERROR @@ -4468,50 +4257,46 @@ def handler(request: httpx.Request) -> httpx.Response: def test_double_instrument_is_idempotent_and_uninstrument_restores_once( memory: SimpleNamespace, ) -> None: - originals = ( - Completions.create, - Completions.parse, - AsyncCompletions.create, - AsyncCompletions.parse, - Responses.create, - Responses.retrieve, - Responses.parse, - AsyncResponses.create, - AsyncResponses.parse, - AsyncResponses.retrieve, - Embeddings.create, - AsyncEmbeddings.create, + # The resource classes the README documents as instrumented. + classes: tuple[type[Any], ...] = ( + Completions, + AsyncCompletions, + Responses, + AsyncResponses, + Embeddings, + AsyncEmbeddings, + Images, + AsyncImages, + Speech, + AsyncSpeech, + Transcriptions, + AsyncTranscriptions, + Translations, + AsyncTranslations, + Batches, + AsyncBatches, ) + def snapshot() -> dict[type[Any], dict[str, Any]]: + return {cls: dict(vars(cls)) for cls in classes} + def handler(request: httpx.Request) -> httpx.Response: return json_response(chat_completion()) + before = snapshot() instrument_openai() try: - instrumented_create = Completions.create - assert instrumented_create is not originals[0] + instrumented = snapshot() + assert [cls for cls in classes if instrumented[cls] == before[cls]] == [] instrument_openai() - assert Completions.create is instrumented_create - instrumented = sync_client(handler) - cast(Any, instrumented.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) + assert snapshot() == instrumented + client = sync_client(handler) + cast(Any, client.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) assert len(memory.span_exporter.get_finished_spans()) == 1 finally: uninstrument_openai() - assert ( - Completions.create, - Completions.parse, - AsyncCompletions.create, - AsyncCompletions.parse, - Responses.create, - Responses.retrieve, - Responses.parse, - AsyncResponses.create, - AsyncResponses.parse, - AsyncResponses.retrieve, - Embeddings.create, - AsyncEmbeddings.create, - ) == originals + assert snapshot() == before uninstrumented = sync_client(handler) cast(Any, uninstrumented.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) assert len(memory.span_exporter.get_finished_spans()) == 1 @@ -4524,11 +4309,12 @@ def later_owner(self: Any, *args: Any, **kwargs: Any) -> Any: return original(self, *args, **kwargs) instrument_openai() - Completions.create = later_owner - uninstrument_openai() - - assert Completions.create is later_owner - Completions.create = original + try: + Completions.create = later_owner + uninstrument_openai() + assert Completions.create is later_owner + finally: + Completions.create = original def test_wrap_openai_is_idempotent(memory: SimpleNamespace) -> None: @@ -4627,76 +4413,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert all(span.name == "chat gpt-4o-mini" for span in spans) -def test_chat_completion_finish_reasons_preserve_single_and_multi_choice_order( - memory: SimpleNamespace, -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - body = request_json(request) - if body.get("n") == 2: - return json_response( - chat_completion( - choices=[ - { - "index": 0, - "message": {"role": "assistant", "content": "first"}, - "finish_reason": "stop", - }, - { - "index": 1, - "message": {"role": "assistant", "content": "second"}, - "finish_reason": "length", - }, - ] - ) - ) - return json_response(chat_completion()) - - client = wrap_openai(sync_client(handler)) - cast(Any, client.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) - cast(Any, client.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES, n=2) - - single_span, multi_span = memory.span_exporter.get_finished_spans() - assert attrs(single_span)["gen_ai.response.finish_reasons"] == ("stop",) - assert attrs(multi_span)["gen_ai.response.finish_reasons"] == ("stop", "length") - - -def test_chat_streaming_finish_reasons_include_all_choices(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "id": "chatcmpl_stream_choices", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini-2024-07-18", - "choices": [ - {"index": 0, "delta": {"role": "assistant", "content": "first"}}, - {"index": 1, "delta": {"role": "assistant", "content": "second"}}, - ], - }, - { - "id": "chatcmpl_stream_choices", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini-2024-07-18", - "choices": [ - {"index": 0, "delta": {}, "finish_reason": "stop"}, - {"index": 1, "delta": {}, "finish_reason": "length"}, - ], - }, - ] - ) - - client = wrap_openai(sync_client(handler)) - list( - cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - ) - - assert attrs(only_span(memory))["gen_ai.response.finish_reasons"] == ("stop", "length") - - def test_chat_streaming_reconstructs_split_tool_call_arguments(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return sse_response( @@ -4837,212 +4553,27 @@ async def handler(request: httpx.Request) -> httpx.Response: assert attrs(failed_span)["error.type"] == "RuntimeError" -async def test_async_responses_stream_bounds_output_without_dropping_terminal_metadata( - memory: SimpleNamespace, -) -> None: - retained = response_payload(response_id="resp_bounded_async", text="prefix") - retained["status"] = "in_progress" - completed = response_payload(response_id="resp_bounded_async", text="x" * 70_000) - - async def handler(request: httpx.Request) -> httpx.Response: - events: list[tuple[str, Mapping[str, Any]]] = [ - ( - "response.created", - { - "type": "response.created", - "sequence_number": 0, - "response": retained, - }, - ), - ( - "response.completed", - { - "type": "response.completed", - "sequence_number": 1, - "response": completed, - }, - ), - ] - return named_sse_response(events) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)( - model="gpt-4o-mini", input="bounded", stream=True - ) - assert [event.type async for event in stream] == [ - "response.created", - "response.completed", - ] - await client.close() - - a = attrs(only_span(memory)) - assert a["gen_ai.response.id"] == "resp_bounded_async" - assert a["gen_ai.response.model"] == "gpt-4o-2024-08-06" - assert a["gen_ai.usage.total_tokens"] == 17 - assert a["gen_ai.response.finish_reasons"] == ("stop",) - assert json.loads(str(a["gen_ai.output.messages"])) == [ - { - "id": "msg_123", - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "prefix", "annotations": []}], - } - ] - assert a["telemetry.dev.capture.truncated"] is True - - -async def test_async_responses_stream_does_not_report_empty_created_output_for_truncated_terminal( - memory: SimpleNamespace, -) -> None: - async def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response(empty_created_then_oversized_terminal()) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) - [event async for event in stream] - await client.close() - assert_empty_created_output_not_reported(memory) - - -async def test_async_responses_stream_fitting_terminal_clears_prior_truncation( - memory: SimpleNamespace, -) -> None: - retained = response_payload(text="partial") - retained["status"] = "in_progress" - oversized = response_payload(text="x" * 70_000) - oversized["status"] = "in_progress" - completed = response_payload(text="final") - - async def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response( - [ - ("response.created", {"type": "response.created", "response": retained}), - ( - "response.in_progress", - {"type": "response.in_progress", "response": oversized}, - ), - ("response.completed", {"type": "response.completed", "response": completed}), - ] - ) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) - assert len([event async for event in stream]) == 3 - await client.close() - - a = attrs(only_span(memory)) - assert json.loads(str(a["gen_ai.output.messages"]))[0]["content"][0]["text"] == "final" - assert "telemetry.dev.capture.truncated" not in a - - -async def test_async_responses_stream_existing_response_is_instrumented( - memory: SimpleNamespace, -) -> None: - requests: list[httpx.Request] = [] - - async def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - events: list[tuple[str, Mapping[str, Any]]] = [ - ( - "response.created", - { - "type": "response.created", - "sequence_number": 0, - "response": {"id": "resp_existing", "status": "in_progress", "output": []}, - }, - ), - ( - "response.completed", - { - "type": "response.completed", - "sequence_number": 1, - "response": response_payload(response_id="resp_existing", text="Streamed."), - }, - ), - ] - return named_sse_response(events) - - client = wrap_openai(async_client(handler)) - async with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: - events = [event async for event in stream] - await client.close() - - assert [event.type for event in events] == ["response.created", "response.completed"] - assert requests[0].method == "GET" - assert requests[0].url.path == "/v1/responses/resp_existing" - span = only_span(memory) - a = attrs(span) - assert a["gen_ai.response.id"] == "resp_existing" - assert ( - json.loads(str(a["gen_ai.output.messages"])) - == response_payload( - response_id="resp_existing", - text="Streamed.", - )["output"] - ) - - -async def test_async_responses_stream_bare_error_event_sets_error_status( - memory: SimpleNamespace, -) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "type": "error", - "code": "server_error", - "message": "model blew up", - "sequence_number": 0, - } - ] - ) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)( - model="gpt-4o-mini", input="fail", stream=True - ) - events = [event async for event in stream] - await client.close() - - assert [event.type for event in events] == ["error"] - assert only_span(memory).status.status_code == StatusCode.ERROR - - -def test_chat_stream_context_manager_exit_ends_partial_span_once( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_stream_context_manager_exit_ends_partial_span_once( + memory: SimpleNamespace, async_mode: bool ) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) - - client = wrap_openai(sync_client(handler)) - with cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) as stream: - first = next(stream) - - spans = memory.span_exporter.get_finished_spans() - assert first.choices[0].delta.content == "Hello" - assert len(spans) == 1 - assert spans[0].status.status_code == StatusCode.UNSET - assert json.loads(str(attrs(spans[0])["gen_ai.output.messages"])) == [ - {"role": "assistant", "content": "Hello"} - ] - assert attrs(spans[0])["telemetry.dev.capture.truncated"] is True - + if async_mode: -async def test_async_chat_stream_context_manager_exit_ends_partial_span_once( - memory: SimpleNamespace, -) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(chat_stream_events()) - client = wrap_openai(async_client(handler)) - async with await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) as stream: - first = await stream.__anext__() - await client.close() + client = wrap_openai(async_client(handler)) + async with await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) as stream: + first = await stream.__anext__() + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(chat_stream_events()))) + with cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) as stream: + first = next(stream) spans = memory.span_exporter.get_finished_spans() assert first.choices[0].delta.content == "Hello" @@ -5099,39 +4630,31 @@ def handler(request: httpx.Request) -> httpx.Response: assert attrs(spans[0])["telemetry.dev.capture.truncated"] is True -def test_text_media_response_captures_text_and_falls_back_text_tokens( - memory: SimpleNamespace, -) -> None: - mapper = vars(telemetry_dev_openai)["_text_media_response"] - fields = mapper( - SimpleNamespace(text="transcript", usage=SimpleNamespace(output_tokens=7, total_tokens=7)) - ) - assert fields["output"] == "transcript" - assert fields["usage"]["text_output_tokens"] == 7 - - -def test_text_media_response_passes_the_complete_transcript_to_the_core(make: Any) -> None: - make(max_attribute_length=5) - mapper = vars(telemetry_dev_openai)["_text_media_response"] - - fields = mapper(SimpleNamespace(text="ééé")) - - assert fields["output"] == "ééé" - assert "attributes" not in fields +def test_bounded_responses_conversion_does_not_model_dump(make: Any) -> None: + class Item: + def __init__(self) -> None: + self.type = "output_text" + self.text = "x" * 100_000 + def model_dump(self, **_kwargs: Any) -> Any: + raise AssertionError("model_dump must not run") -def test_bounded_responses_conversion_does_not_model_dump(memory: SimpleNamespace) -> None: class Bomb: def __init__(self) -> None: - self.output = [{"type": "output_text", "text": "x" * 100_000}] + self.output = [Item()] self.status = "completed" - def model_dump(self, **kwargs: Any) -> Any: + def model_dump(self, **_kwargs: Any) -> Any: raise AssertionError("model_dump must not run") + errors: list[BaseException] = [] + make(on_error=errors.append) + fields = vars(telemetry_dev_openai)["_responses_response"](Bomb()) + assert fields["output"] is None assert fields["attributes"]["telemetry.dev.capture.truncated"] is True + assert errors == [] def test_bounded_responses_capture_repeats_shared_acyclic_values( @@ -5182,9 +4705,10 @@ def items(self) -> Any: @pytest.mark.parametrize("kind", ["oversized", "deep", "cyclic", "hostile"]) -def test_responses_input_capture_failure_never_blocks_provider( - memory: SimpleNamespace, kind: str -) -> None: +def test_responses_input_capture_failure_never_blocks_provider(make: Any, kind: str) -> None: + errors: list[BaseException] = [] + memory = make(on_error=errors.append) + class HostileMapping(dict[str, Any]): def items(self) -> Any: raise RuntimeError("hostile mapping") @@ -5228,7 +4752,11 @@ def provider_resolver(_resource: object | None) -> str: wrapper(model="gpt-4.1", input=value) assert called - if kind == "oversized": - span = memory.span_exporter.get_finished_spans()[-1] - assert "gen_ai.input.messages" not in attrs(span) - assert attrs(span)["telemetry.dev.capture.truncated"] is True + a = attrs(only_span(memory)) + if kind == "cyclic": + assert a["gen_ai.input.messages"] == '{"self":null}' + assert "telemetry.dev.capture.truncated" not in a + else: + assert "gen_ai.input.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True + assert [str(error) for error in errors] == (["hostile mapping"] if kind == "hostile" else []) diff --git a/sdks/python-openrouter/tests/test_openrouter.py b/sdks/python-openrouter/tests/test_openrouter.py index 4206b9d..3f6ea13 100644 --- a/sdks/python-openrouter/tests/test_openrouter.py +++ b/sdks/python-openrouter/tests/test_openrouter.py @@ -23,7 +23,6 @@ from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.trace import StatusCode -import telemetry_dev_openrouter from telemetry_dev_openrouter import ( instrument_openrouter, uninstrument_openrouter, @@ -199,6 +198,14 @@ def responses_payload( } +def completed_without_output(sequence_number: int) -> dict[str, Any]: + """A terminal event whose null output fails SDK validation, so it arrives raw and ends the + stream without replacing the reconstructed output.""" + response = responses_payload() + response["output"] = None + return {"type": "response.completed", "sequence_number": sequence_number, "response": response} + + def embeddings_payload() -> dict[str, Any]: return { "id": "embed_123", @@ -535,7 +542,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: assert len(calls) == 3 assert len(memory.span_exporter.get_finished_spans()) == 1 - assert memory.metric_reader.get_metrics_data() is not None @pytest.mark.parametrize("async_mode", [False, True]) @@ -590,7 +596,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: assert len(calls) == 3 assert len(memory.span_exporter.get_finished_spans()) == 1 - assert memory.metric_reader.get_metrics_data() is not None def test_chat_stream_capture_is_bounded_without_losing_terminal_metadata( @@ -623,7 +628,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.usage.cost"] == 0.04 -def _streamed_chat(memory: SimpleNamespace, events: list[dict[str, Any]]) -> tuple[Any, Any]: +def _streamed_chat(memory: SimpleNamespace, events: list[dict[str, Any]]) -> dict[str, object]: def handler(request: httpx.Request) -> httpx.Response: return sse_response(events) @@ -631,7 +636,7 @@ def handler(request: httpx.Request) -> httpx.Response: model="openai/gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) assert len(list(stream)) == len(events) - return stream, attrs(only_span(memory)) + return attrs(only_span(memory)) @pytest.mark.parametrize( @@ -645,7 +650,10 @@ def handler(request: httpx.Request) -> httpx.Response: def test_configured_cap_applies_after_the_mask_not_to_stream_retention( make: Any, max_attribute_length: int, redact: bool, expected: str | None ) -> None: + seen: list[Any] = [] + def mask(value: Any, _context: Any) -> Any: + seen.append(value) if not redact: return value return [{"role": "assistant", "content": "[redacted]"}] @@ -657,11 +665,11 @@ def mask(value: Any, _context: Any) -> Any: chat_chunk({}, finish_reason="stop"), ] - stream, a = _streamed_chat(memory, events) + a = _streamed_chat(memory, events) - assert stream._budget.truncated is False - assert stream._states[0].content == text assert "telemetry.dev.capture.truncated" not in a + if redact: + assert [{"role": "assistant", "content": text}] in seen if expected is None: assert a.get("gen_ai.output.messages", "") == "" else: @@ -688,19 +696,18 @@ def mask(value: Any, _context: Any) -> Any: chat_chunk({}, finish_reason="stop"), ] - stream, a = _streamed_chat(memory, events) + a = _streamed_chat(memory, events) - budget = stream._budget - retained = stream._states[0].content - assert budget.truncated is True - assert budget.max_bytes == telemetry_dev_openrouter._STREAM_CAPTURE_MAX_BYTES # pyright: ignore[reportPrivateUsage] - assert 0 < len(retained) < 1_100 * 100 assert a["telemetry.dev.capture.truncated"] is True if masked: assert "gen_ai.output.messages" not in a assert not any("x" * 100 in value for value in seen) - elif max_attribute_length > len(retained): - assert json.loads(str(a["gen_ai.output.messages"]))[0]["content"] == retained + elif max_attribute_length == 1024 * 1024: + # The 64 KiB retention bound, not the 1 MiB cap, cuts the stream at a chunk boundary. + content = json.loads(str(a["gen_ai.output.messages"]))[0]["content"] + assert set(content) == {"x"} + assert 0 < len(content) <= 64 * 1024 + assert len(content) % 100 == 0 else: assert len(str(a.get("gen_ai.output.messages", ""))) <= max_attribute_length @@ -797,7 +804,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert len(chunks) == 1 a = attrs(only_span(memory)) output = json.loads(str(a["gen_ai.output.messages"])) - assert len(output) == telemetry_dev_openrouter._STREAM_CAPTURE_MAX_ITEMS # pyright: ignore[reportPrivateUsage] + assert len(output) == 1024 assert output[0] == {"role": "assistant", "content": "Hi"} assert a["telemetry.dev.capture.truncated"] is True @@ -819,10 +826,9 @@ def test_chat_stream_finish_reasons_are_capped_at_a_fixed_length( }, ] - stream, a = _streamed_chat(memory, events) + a = _streamed_chat(memory, events) - assert list(a["gen_ai.response.finish_reasons"]) == ["r" * max_length, "stop"] - assert all(len(state.finish_reason or "") <= max_length for state in stream._states.values()) + assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["r" * max_length, "stop"] assert (a.get("telemetry.dev.capture.truncated") is True) is spent @@ -1056,30 +1062,18 @@ def handler(request: httpx.Request) -> httpx.Response: ) -def test_responses_stream_failed_event_sets_error(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - event = { - "type": "response.failed", - "sequence_number": 0, - "response": responses_payload( - response_id="resp_failed", - status="failed", - error={"code": "server_error", "message": "model blew up"}, - ), - } - return sse_response([event]) - - client = wrap_open_router(sync_client(handler)) - stream = cast(Any, client.responses.send)(model="openai/gpt-4o-mini", input="fail", stream=True) - events = list(stream) - - assert [event.type for event in events] == ["response.failed"] - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "RuntimeError" - - -def test_responses_stream_failed_event_retains_numeric_code(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize( + "code", + [ + # Fails SDK validation, so it arrives as a raw unknown event. + pytest.param(502, id="numeric-code"), + # Parses as the typed failed event. + pytest.param("server_error", id="string-code"), + ], +) +def test_responses_stream_failed_event_retains_error_code( + memory: SimpleNamespace, code: int | str +) -> None: def handler(request: httpx.Request) -> httpx.Response: event = { "type": "response.failed", @@ -1087,7 +1081,7 @@ def handler(request: httpx.Request) -> httpx.Response: "response": responses_payload( response_id="resp_failed", status="failed", - error={"code": 502, "message": "model blew up"}, + error={"code": code, "message": "model blew up"}, ), } return sse_response([event]) @@ -1098,12 +1092,13 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) assert span.status.status_code == StatusCode.ERROR + assert attrs(span)["error.type"] == "RuntimeError" events_recorded = list(span.events) assert len(events_recorded) == 1 assert events_recorded[0].name == "exception" event_attrs = dict(events_recorded[0].attributes or {}) assert event_attrs["exception.type"] == "RuntimeError" - assert event_attrs["exception.message"] == "response.failed: 502: model blew up" + assert event_attrs["exception.message"] == f"response.failed: {code}: model blew up" def test_responses_stream_error_event_retains_numeric_code(memory: SimpleNamespace) -> None: @@ -1623,6 +1618,7 @@ def test_responses_stream_capture_rejects_sparse_annotation_indexes( "url": "https://example.test/unsafe", }, }, + completed_without_output(3), ] def handler(request: httpx.Request) -> httpx.Response: @@ -1631,9 +1627,7 @@ def handler(request: httpx.Request) -> httpx.Response: stream = cast(Any, wrap_open_router(sync_client(handler)).responses.send)( model="openai/gpt-4o-mini", input="annotate", stream=True ) - for _ in events: - next(stream) - stream.close() + list(stream) a = attrs(only_span(memory)) assert json.loads(str(a["gen_ai.output.messages"])) == [ @@ -1648,9 +1642,11 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["telemetry.dev.capture.truncated"] is True -def test_responses_stream_boolean_indexes_do_not_collide_with_index_one() -> None: - state_type = vars(telemetry_dev_openrouter)["_ResponsesStreamState"] - state = state_type() +def test_responses_stream_boolean_indexes_do_not_collide_with_index_one( + memory: SimpleNamespace, +) -> None: + # The SDK coerces a JSON true index to 1 on events that validate, so the boolean-index + # events below omit item_id: they fail validation and arrive raw. events: list[dict[str, Any]] = [ { "type": "response.output_item.added", @@ -1675,11 +1671,9 @@ def test_responses_stream_boolean_indexes_do_not_collide_with_index_one() -> Non { "type": "response.output_text.delta", "sequence_number": 3, - "item_id": "message_zero", "output_index": True, "content_index": True, "delta": "Content zero", - "logprobs": [], }, { "type": "response.reasoning_summary_part.added", @@ -1692,17 +1686,21 @@ def test_responses_stream_boolean_indexes_do_not_collide_with_index_one() -> Non { "type": "response.reasoning_summary_text.delta", "sequence_number": 5, - "item_id": "reasoning_two", "output_index": 2, "summary_index": True, "delta": "Summary zero", }, ] - for event in events: - state.record(event) + def handler(request: httpx.Request) -> httpx.Response: + return sse_response(events) - assert state.partial["output"] == [ + stream = cast(Any, wrap_open_router(sync_client(handler)).responses.send)( + model="openai/gpt-4o-mini", input="indexes", stream=True + ) + list(stream) + + assert json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) == [ { "id": "message_zero", "type": "message", @@ -2000,6 +1998,7 @@ def test_final_responses_text_replaces_delta_truncated_output_with_fresh_budget( "text": "Final answer", "logprobs": [], }, + completed_without_output(len(deltas) + 1), ] def handler(request: httpx.Request) -> httpx.Response: @@ -2020,10 +2019,7 @@ def handler(request: httpx.Request) -> httpx.Response: "content": [{"type": "output_text", "text": "Final answer", "annotations": []}], } ] - # The done event resolves the cut deltas; the flag remains because no terminal - # response arrived. - assert stream._state._budget.truncated is False - assert a["telemetry.dev.capture.truncated"] is True + assert "telemetry.dev.capture.truncated" not in a def test_final_responses_text_preserves_provider_event_truncation( @@ -2047,6 +2043,7 @@ def test_final_responses_text_preserves_provider_event_truncation( "text": "Final answer", "logprobs": [], }, + completed_without_output(3), ] def handler(request: httpx.Request) -> httpx.Response: diff --git a/sdks/python/src/telemetry_dev/_metrics.py b/sdks/python/src/telemetry_dev/_metrics.py index ceb89a5..6aebff4 100644 --- a/sdks/python/src/telemetry_dev/_metrics.py +++ b/sdks/python/src/telemetry_dev/_metrics.py @@ -72,8 +72,7 @@ class _ChunkAggregate: class OutputChunkAggregation: - def __init__(self, max_attribute_sets: int = _MAX_CHUNK_ATTRIBUTE_SETS) -> None: - self._max_attribute_sets = max_attribute_sets + def __init__(self) -> None: self._values: dict[tuple[tuple[str, Any], ...], _ChunkAggregate] = {} self._lock = threading.Lock() self._collection_start = time.time_ns() @@ -85,7 +84,7 @@ def add( ) -> None: key = tuple(sorted(attributes.items())) with self._lock: - if key not in self._values and len(self._values) >= self._max_attribute_sets - 1: + if key not in self._values and len(self._values) >= _MAX_CHUNK_ATTRIBUTE_SETS - 1: key = (("otel.metric.overflow", True),) count, total, minimum, maximum, buckets = aggregate existing = self._values.get(key) diff --git a/sdks/python/tests/otlp_capture.py b/sdks/python/tests/otlp_capture.py index 5b08844..7a88773 100644 --- a/sdks/python/tests/otlp_capture.py +++ b/sdks/python/tests/otlp_capture.py @@ -41,7 +41,8 @@ def do_POST(self) -> None: body = self.rfile.read(length) captured.append( CapturedRequest( - path=self.path, + # self.path collapses a leading "//" (gh-87389); keep the raw target. + path=self.requestline.split(" ")[1], headers={key.lower(): value for key, value in self.headers.items()}, body=body, ) @@ -55,7 +56,9 @@ def log_message(self, format: str, *args: object) -> None: pass server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) thread.start() try: host, port = server.server_address[0], server.server_address[1] diff --git a/sdks/python/tests/test_config.py b/sdks/python/tests/test_config.py index b7f8fb6..44cd783 100644 --- a/sdks/python/tests/test_config.py +++ b/sdks/python/tests/test_config.py @@ -1,7 +1,5 @@ from __future__ import annotations -import pytest - from telemetry_dev._config import resolve_config @@ -11,42 +9,3 @@ def test_defaults_without_env() -> None: assert config.base_url == "https://ingest.telemetry.dev" assert config.environment == "production" assert config.service_name == "unknown_service" - - -def test_env_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_env") - monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", "https://example.test") - monkeypatch.setenv("TELEMETRY_DEV_ENVIRONMENT", "staging") - monkeypatch.setenv("OTEL_SERVICE_NAME", "svc") - config = resolve_config() - assert config.api_key == "td_live_env" - assert config.base_url == "https://example.test" - assert config.environment == "staging" - assert config.service_name == "svc" - - -def test_arguments_take_precedence_over_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_env") - monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", "https://env.test") - monkeypatch.setenv("TELEMETRY_DEV_ENVIRONMENT", "staging") - monkeypatch.setenv("OTEL_SERVICE_NAME", "env-svc") - config = resolve_config( - api_key="td_live_arg", - base_url="https://arg.test", - environment="dev", - service_name="arg-svc", - ) - assert config.api_key == "td_live_arg" - assert config.base_url == "https://arg.test" - assert config.environment == "dev" - assert config.service_name == "arg-svc" - - -def test_base_url_trailing_slashes_stripped() -> None: - assert resolve_config(base_url="https://x.test/").base_url == "https://x.test" - assert resolve_config(base_url="https://x.test///").base_url == "https://x.test" - - -def test_base_url_env_trailing_slash_stripped(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", "https://env.test//") - assert resolve_config().base_url == "https://env.test" diff --git a/sdks/python/tests/test_init.py b/sdks/python/tests/test_init.py index 85c5d13..2b21214 100644 --- a/sdks/python/tests/test_init.py +++ b/sdks/python/tests/test_init.py @@ -12,23 +12,13 @@ from tests.conftest import MakeClient -def test_no_api_key_is_silent_noop() -> None: - client = telemetry_dev.init(log_level="silent", disable_atexit=True) - assert client.enabled is False - handle = telemetry_dev.start_span("x", type="generation", input="hi") - assert handle.traceparent() is None - handle.update(output="y") - handle.end() - telemetry_dev.log("nope") - telemetry_dev.update_current_span(output="y") - telemetry_dev.flush() - - def test_enabled_false_kill_switch_beats_exporter_seams(make: MakeClient) -> None: - env = make(enabled=False) + env = make(enabled=False, session_mode="process") assert env.client.enabled is False telemetry_dev.start_span("x").end() + telemetry_dev.log("x") assert env.span_exporter.get_finished_spans() == () + assert env.log_exporter.get_finished_logs() == () @pytest.mark.parametrize("max_attribute_length", [-1, 1.5, float("nan"), float("inf"), True]) @@ -92,19 +82,6 @@ def test_raw_attributes_follow_the_attribute_backstop_not_the_content_cap( assert attributes[key] == expected -def test_exporter_seam_enables_without_api_key(make: MakeClient) -> None: - env = make() - assert env.client.enabled is True - telemetry_dev.start_span("x").end() - assert len(env.span_exporter.get_finished_spans()) == 1 - - -def test_api_key_from_env_enables(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_test") - client = telemetry_dev.init(log_level="silent", disable_atexit=True) - assert client.enabled is True - - def test_double_init_replaces_previous_client(make: MakeClient) -> None: first = make() second = make() @@ -140,14 +117,6 @@ def test_isolated_by_default(make: MakeClient) -> None: assert env.span_exporter.get_finished_spans() == () -def test_register_global_sets_global_provider( - make: MakeClient, monkeypatch: pytest.MonkeyPatch -) -> None: - _reset_global_tracer_provider(monkeypatch) - env = make(register_global=True) - assert trace.get_tracer_provider() is env.client._tracer_provider # pyright: ignore[reportPrivateUsage] - - def test_register_global_default_filter_is_scope_name_only( make: MakeClient, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/sdks/python/tests/test_logs.py b/sdks/python/tests/test_logs.py index ad0abae..acf3942 100644 --- a/sdks/python/tests/test_logs.py +++ b/sdks/python/tests/test_logs.py @@ -43,20 +43,6 @@ def test_default_level_is_info(memory: SimpleNamespace) -> None: assert record.severity_number.value == 9 -def test_body_is_message(memory: SimpleNamespace) -> None: - telemetry_dev.log("hello world") - assert only_log(memory).log_record.body == "hello world" - - -def test_trace_correlation_inside_span(memory: SimpleNamespace) -> None: - with telemetry_dev.start_span("parent") as handle: - telemetry_dev.log("inside") - record = only_log(memory).log_record - span_context = handle.span.get_span_context() - assert record.trace_id == span_context.trace_id - assert record.span_id == span_context.span_id - - def test_no_trace_correlation_outside_span(memory: SimpleNamespace) -> None: telemetry_dev.log("outside") record = only_log(memory).log_record @@ -64,11 +50,6 @@ def test_no_trace_correlation_outside_span(memory: SimpleNamespace) -> None: assert not record.span_id -def test_event_name(memory: SimpleNamespace) -> None: - telemetry_dev.log("msg", event_name="cache.miss") - assert only_log(memory).log_record.event_name == "cache.miss" - - def test_custom_attributes(memory: SimpleNamespace) -> None: telemetry_dev.log("msg", attributes={"request.id": "r-1", "retry": 2}) log_attrs = dict(only_log(memory).log_record.attributes or {}) @@ -110,12 +91,6 @@ def test_propagated_attributes_use_client_attribute_cap(make: MakeClient) -> Non assert value == "z" * (30 - len(TRUNCATION_MARKER)) + TRUNCATION_MARKER -def test_log_records_use_sdk_scope(memory: SimpleNamespace) -> None: - telemetry_dev.log("msg") - scope = only_log(memory).instrumentation_scope - assert scope is not None and scope.name == "telemetry_dev" - - def test_log_is_noop_without_init() -> None: telemetry_dev.log("nowhere") diff --git a/sdks/python/tests/test_metrics.py b/sdks/python/tests/test_metrics.py index b5d1f20..9eb6df8 100644 --- a/sdks/python/tests/test_metrics.py +++ b/sdks/python/tests/test_metrics.py @@ -3,7 +3,6 @@ import gc import math import threading -import time from types import SimpleNamespace from weakref import ref @@ -191,6 +190,8 @@ def keep(span: ReadableSpan) -> bool: telemetry_dev.start_span("filtered", type="generation").record_output_chunk( 1000 ).record_output_chunk(9000).end(error=RuntimeError("filtered failure")) + telemetry_dev.start_span("zero", type="generation").end() + telemetry_dev.start_span("one", type="generation").record_output_chunk(1000).end() handle = telemetry_dev.start_span( "stream", type="generation", model="gpt-4o", provider="openai" ) @@ -288,22 +289,6 @@ def close() -> None: client.shutdown() -def test_output_chunk_zero_or_one_and_filter_rejection_emit_nothing(make: MakeClient) -> None: - def keep(span: ReadableSpan) -> bool: - return span.name != "rejected" - - errors: list[BaseException] = [] - env = make(span_filter=keep, on_error=errors.append) - telemetry_dev.start_span("zero", type="generation").end() - one = telemetry_dev.start_span("one", type="generation") - one.record_output_chunk(1000).end() - rejected = telemetry_dev.start_span("rejected", type="generation") - rejected.record_output_chunk(1000).record_output_chunk(1020).end() - - assert env.client._output_chunks is None - assert errors == [] - - def test_rejected_chunk_state_does_not_retain_abandoned_handles(make: MakeClient) -> None: def reject(span: ReadableSpan) -> bool: return False @@ -318,15 +303,15 @@ def reject(span: ReadableSpan) -> bool: def test_output_chunk_attribute_cardinality_is_bounded() -> None: - aggregation = OutputChunkAggregation(max_attribute_sets=2) + aggregation = OutputChunkAggregation() value = (1, 0.01, 0.01, 0.01, (1, *([0] * len(DURATION_BUCKETS)))) - for model in ("a", "b", "c", "d"): - aggregation.add({"gen_ai.request.model": model}, value) + for index in range(2001): + aggregation.add({"gen_ai.request.model": f"model-{index}"}, value) points = aggregation.drain_points() - assert len(points) == 2 + assert len(points) == 2000 overflow = next(point for point in points if point.attributes == {"otel.metric.overflow": True}) - assert overflow.count == 3 + assert overflow.count == 2 def test_ended_child_metrics_reference_child_not_active_parent(memory: SimpleNamespace) -> None: @@ -393,24 +378,27 @@ def test_plain_span_records_nothing(memory: SimpleNamespace) -> None: def test_quiet_interval_emits_zero_metric_posts( monkeypatch: pytest.MonkeyPatch, internal_metrics: bool ) -> None: - # Empty-datapoint guard: collections with no data points are never POSTed, even when - # OpenTelemetry records its own metric-reader metrics on the provider. + # Quiet collections are never POSTed, even when OpenTelemetry records its own metric-reader + # metrics on the provider: the otel.sdk.* drop View keeps them out, and the reader skips + # the exporter when a collection is empty. The second flush would carry the internal + # metrics that the first one recorded. # Same exporter, reader, and provider wiring the client uses, against a real local server. if internal_metrics: monkeypatch.setenv("OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED", "true") with otlp_capture_server() as server: exporter = GuardedOTLPMetricExporter(endpoint=f"{server.url}/v1/metrics") - reader = PeriodicExportingMetricReader(exporter, export_interval_millis=50) + reader = PeriodicExportingMetricReader(exporter, export_interval_millis=2**31 - 1) provider = sdk_meter_provider(reader, Resource.create()) meter = provider.get_meter("quiet") histogram = meter.create_histogram("gen_ai.client.operation.duration", unit="s") - time.sleep(0.3) # several quiet collection intervals + provider.force_flush() + provider.force_flush() assert server.requests_for("/v1/metrics") == [] histogram.record(0.5, {"gen_ai.operation.name": "chat"}) provider.force_flush() - assert len(server.requests_for("/v1/metrics")) >= 1 + assert len(server.requests_for("/v1/metrics")) == 1 provider.shutdown() diff --git a/sdks/python/tests/test_otel.py b/sdks/python/tests/test_otel.py index 718d4e8..4bb128f 100644 --- a/sdks/python/tests/test_otel.py +++ b/sdks/python/tests/test_otel.py @@ -137,19 +137,10 @@ def test_create_span_exporter_posts_traces_with_bearer_gzip() -> None: assert request.headers["content-encoding"] == "gzip" -def test_no_op_without_key(caplog: pytest.LogCaptureFixture) -> None: +def test_no_posts_keyless_even_with_base_url_env( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: caplog.set_level(logging.DEBUG, logger="telemetry_dev") - processor = TelemetrySpanProcessor() - provider = provider_with(processor) - try: - end_span(provider, "noop") - assert processor.force_flush() is True - finally: - provider.shutdown() - assert "no-op" in caplog.text - - -def test_no_posts_keyless_even_with_base_url_env(monkeypatch: pytest.MonkeyPatch) -> None: with otlp_capture_server() as server: monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", server.url) processor = TelemetrySpanProcessor() @@ -160,6 +151,7 @@ def test_no_posts_keyless_even_with_base_url_env(monkeypatch: pytest.MonkeyPatch finally: provider.shutdown() assert server.requests == [] + assert "no-op" in caplog.text def test_env_var_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None: @@ -185,7 +177,7 @@ def test_kwargs_beat_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", f"{server.url}/env") processor = TelemetrySpanProcessor( api_key="td_live_kwarg", - base_url=server.url, + base_url=f"{server.url}//", export_mode="immediate", metrics=False, ) diff --git a/sdks/python/tests/test_process_session.py b/sdks/python/tests/test_process_session.py index aa1523c..6e9c21b 100644 --- a/sdks/python/tests/test_process_session.py +++ b/sdks/python/tests/test_process_session.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -from types import SimpleNamespace from typing import Any, cast from opentelemetry.trace import SpanContext, TraceFlags, TraceState @@ -36,7 +35,9 @@ def test_explicit_parent_session_wins_for_children_and_logs(make: MakeClient) -> ): telemetry_dev.start_span("explicit-child").end() telemetry_dev.log("explicit-log") - for span in env.span_exporter.get_finished_spans(): + spans = env.span_exporter.get_finished_spans() + assert [span.name for span in spans] == ["explicit-child", "explicit-root"] + for span in spans: assert attrs(span)["gen_ai.conversation.id"] == "explicit" (record,) = env.log_exporter.get_finished_logs() assert attrs(record.log_record)["gen_ai.conversation.id"] == "explicit" @@ -56,7 +57,9 @@ async def main() -> None: await asyncio.gather(emit("one"), emit("two")) asyncio.run(main()) - for span in env.span_exporter.get_finished_spans(): + spans = env.span_exporter.get_finished_spans() + assert sorted(span.name for span in spans) == ["one-child", "one-root", "two-child", "two-root"] + for span in spans: assert attrs(span)["gen_ai.conversation.id"] == span.name.split("-")[0] @@ -77,10 +80,16 @@ def test_process_mode_honors_remote_parent(make: MakeClient) -> None: def test_reinit_generates_a_new_process_session(make: MakeClient) -> None: first = make(api_key="td_live_k", session_mode="process") - first_id = first.client._process_session_id # pyright: ignore[reportPrivateUsage] + telemetry_dev.start_span("first").end() + (first_span,) = first.span_exporter.get_finished_spans() telemetry_dev.shutdown() second = make(api_key="td_live_k", session_mode="process") - assert second.client._process_session_id != first_id # pyright: ignore[reportPrivateUsage] + telemetry_dev.start_span("second").end() + (second_span,) = second.span_exporter.get_finished_spans() + first_id = attrs(first_span)["gen_ai.conversation.id"] + second_id = attrs(second_span)["gen_ai.conversation.id"] + assert isinstance(first_id, str) and isinstance(second_id, str) + assert first_id != second_id def test_invalid_mode_fails_open_via_on_error() -> None: @@ -94,8 +103,3 @@ def test_invalid_mode_fails_open_via_on_error() -> None: ) assert client.enabled is False assert isinstance(errors[0], ValueError) - - -def test_disabled_process_mode_is_a_noop(make: MakeClient) -> None: - env: SimpleNamespace = make(enabled=False, session_mode="process") - assert env.client._process_session_id is None # pyright: ignore[reportPrivateUsage] diff --git a/sdks/python/tests/test_propagation.py b/sdks/python/tests/test_propagation.py index cd21112..b68c521 100644 --- a/sdks/python/tests/test_propagation.py +++ b/sdks/python/tests/test_propagation.py @@ -23,7 +23,7 @@ from opentelemetry.util.types import Attributes import telemetry_dev -from telemetry_dev._context import session_span_context, with_session_parent +from telemetry_dev._context import session_span_context from telemetry_dev._serialize import MaskContext from tests.conftest import MakeClient, span_context @@ -131,15 +131,6 @@ def test_get_traceparent_round_trip(memory: SimpleNamespace) -> None: assert telemetry_dev.get_traceparent() is None -def test_logs_get_propagated_attributes(memory: SimpleNamespace) -> None: - with telemetry_dev.propagate_attributes(user_id="u-log", metadata={"plan": "pro"}): - telemetry_dev.log("hello") - (record,) = memory.log_exporter.get_finished_logs() - log_attrs = dict(record.log_record.attributes or {}) - assert log_attrs["user.id"] == "u-log" - assert log_attrs["td.metadata.plan"] == "pro" - - def test_invalid_traceparent_falls_back_to_active_context(memory: SimpleNamespace) -> None: with telemetry_dev.start_span("root") as root: telemetry_dev.start_span("child", parent="not-a-traceparent").end() @@ -381,9 +372,7 @@ def mask(_value: object, _context: MaskContext) -> str: assert "gen_ai.output.messages" not in a -def test_session_sampling_keeps_baggage_changed_after_attaching_the_session_parent( - make: MakeClient, -) -> None: +def test_session_root_sampling_sees_caller_baggage(make: MakeClient) -> None: class TenantSampler(Sampler): def should_sample( self, @@ -404,10 +393,12 @@ def get_description(self) -> str: return "TenantSampler" env = make(api_key="td_live_k", sampler=ParentBased(TenantSampler())) - before = baggage.set_baggage("tenant", "denied", Context()) - parent = with_session_parent(before, "s1", "td_live_k") - allowed = baggage.set_baggage("tenant", "allowed", parent) - telemetry_dev.start_span("allowed", parent=allowed).end() - telemetry_dev.start_span("denied", parent=parent).end() - telemetry_dev.start_span("removed", parent=baggage.remove_baggage("tenant", allowed)).end() - assert [span.name for span in env.span_exporter.get_finished_spans()] == ["allowed"] + allowed = baggage.set_baggage("tenant", "allowed", Context()) + denied = baggage.set_baggage("tenant", "denied", Context()) + with telemetry_dev.propagate_attributes(session_id="s1"): + telemetry_dev.start_span("allowed", parent=allowed).end() + telemetry_dev.start_span("denied", parent=denied).end() + telemetry_dev.start_span("removed", parent=baggage.remove_baggage("tenant", allowed)).end() + (span,) = env.span_exporter.get_finished_spans() + assert span.name == "allowed" + assert span_context(span).trace_id == session_span_context("td_live_k", "s1").trace_id diff --git a/sdks/python/tests/test_wire.py b/sdks/python/tests/test_wire.py index f16bd2e..1fc048e 100644 --- a/sdks/python/tests/test_wire.py +++ b/sdks/python/tests/test_wire.py @@ -110,13 +110,33 @@ def test_log_export_wire_format(server: CaptureServer) -> None: def test_no_posts_when_disabled() -> None: with otlp_capture_server() as capture: - telemetry_dev.init( + client = telemetry_dev.init( base_url=capture.url, log_level="silent", disable_atexit=True, ) - telemetry_dev.start_span("nope", type="generation", input="x").end() + assert client.enabled is False + handle = telemetry_dev.start_span("nope", type="generation", input="x") + assert handle.traceparent() is None + handle.update(output="y") + handle.end() + telemetry_dev.update_current_span(output="y") telemetry_dev.log("nope") telemetry_dev.flush() telemetry_dev.shutdown() assert capture.requests == [] + + +def test_env_api_key_and_base_url_reach_the_wire(monkeypatch: pytest.MonkeyPatch) -> None: + with otlp_capture_server() as capture: + monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_env") + monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", f"{capture.url}//") + client = telemetry_dev.init( + export_mode="immediate", log_level="silent", disable_atexit=True + ) + assert client.enabled is True + telemetry_dev.start_span("env").end() + telemetry_dev.flush() + (request,) = capture.requests_for("/v1/traces") + assert request.headers["authorization"] == "Bearer td_live_env" + telemetry_dev.shutdown()