From caeb26ef33a2494291eea4e0d4dca06989dea665 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Mon, 26 Oct 2026 18:10:03 +0000 Subject: [PATCH 1/5] test: remove TypeScript tests that other tests already cover Each package keeps one test per contract at the strongest boundary, which is usually the real provider client with a fake network. The removed tests did the same checks through hand-built events, private helpers, or exact transport retry replays that the otel transport suite already owns. Some kept assertions could not fail. These assertions are repaired. For each repair, a deliberate change to the owner code made the test fail. The ai@7 step-end test now runs through the real generateText dispatcher. Internal exports that only tests used are removed. The public package entries do not change. --- packages/ai/src/integration_v7.ts | 2 +- packages/ai/tests/integration.test.ts | 376 ++++----------- packages/ai/tests/integration_v7.test.ts | 239 ++++------ packages/ai/tests/type_conformance.test.ts | 23 +- packages/ai/tests/wrappers.test.ts | 12 +- packages/anthropic/tests/anthropic.test.ts | 272 +++-------- packages/bedrock/src/internal.ts | 2 +- packages/bedrock/tests/agents.test.ts | 86 ++-- packages/bedrock/tests/bedrock.test.ts | 138 +++--- packages/bedrock/tests/helpers.ts | 14 + packages/cursor/tests/cursor.test.ts | 48 +- packages/eve/tests/eve.test.ts | 43 +- .../google-genai/tests/google-genai.test.ts | 65 +-- packages/mcp/tests/mcp.test.ts | 61 +-- packages/omp/tests/omp.test.ts | 14 +- packages/openai/tests/openai.test.ts | 444 ++++-------------- packages/opencode/tests/opencode.test.ts | 56 +-- packages/openrouter/tests/openrouter.test.ts | 51 +- packages/otel/src/transport.ts | 7 +- packages/otel/tests/otel-byo.test.ts | 87 ++-- packages/otel/tests/transport.test.ts | 136 ++---- packages/pi/tests/pi.test.ts | 30 +- packages/sdk/tests/client.test.ts | 2 +- packages/sdk/tests/metrics.test.ts | 36 +- packages/sdk/tests/process-session.test.ts | 2 + packages/sdk/tests/span.test.ts | 10 - packages/tanstack-ai/tests/evaluate.test.ts | 70 ++- packages/tanstack-ai/tests/generation.test.ts | 55 +-- packages/tanstack-ai/tests/middleware.test.ts | 210 ++++++--- 29 files changed, 900 insertions(+), 1691 deletions(-) diff --git a/packages/ai/src/integration_v7.ts b/packages/ai/src/integration_v7.ts index b721910..c7f0f1f 100644 --- a/packages/ai/src/integration_v7.ts +++ b/packages/ai/src/integration_v7.ts @@ -404,7 +404,7 @@ interface CallState { * onEnd/onError/onAbort). The emitted span/metric shape intentionally matches the v6 path so * ingest and the UI see a single format. */ -export function createV7Hooks( +function createV7Hooks( config: ResolvedConfig, emitter: GenerationEmitter, ): TelemetryDevIntegration { diff --git a/packages/ai/tests/integration.test.ts b/packages/ai/tests/integration.test.ts index a0240ce..00d5d00 100644 --- a/packages/ai/tests/integration.test.ts +++ b/packages/ai/tests/integration.test.ts @@ -10,7 +10,7 @@ import { TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; import { sessionSpanContext } from "@telemetry-dev/otel"; -import { expect, test } from "vitest"; +import { expect, test, vi } from "vitest"; import { telemetryDev } from "../src/v6.ts"; @@ -278,11 +278,13 @@ test("a failed tool call records ERROR status and an exception event", async () const callA = { toolCallId: "a", toolName: "t1", input: { n: 1 } }; const callB = { toolCallId: "b", toolName: "t2", input: { n: 2 } }; + const callC = { toolCallId: "c", toolName: "t3", input: { n: 3 } }; integ.onStart?.({ model, prompt: "use tools" }); integ.onStepStart?.({ stepNumber: 0, model }); integ.onToolCallStart?.({ stepNumber: 0, toolCall: callA }); integ.onToolCallStart?.({ stepNumber: 0, toolCall: callB }); + integ.onToolCallStart?.({ stepNumber: 0, toolCall: callC }); integ.onToolCallFinish?.({ stepNumber: 0, toolCall: callA, @@ -297,6 +299,13 @@ test("a failed tool call records ERROR status and an exception event", async () success: false, error: new Error("kaboom"), }); + integ.onToolCallFinish?.({ + stepNumber: 0, + toolCall: callC, + durationMs: 5, + success: false, + error: { code: "tool_failed", secret: "private" }, + }); integ.onStepFinish?.({ stepNumber: 0, model, @@ -310,9 +319,9 @@ test("a failed tool call records ERROR status and an exception event", async () const spans = spanBatches[0]!; const tools = byOperation(spans, "execute_tool"); const step = spans.find((s) => s.kind === SpanKind.CLIENT)!; - expect(tools).toHaveLength(2); + expect(tools).toHaveLength(3); - // Both tools parent to their step. + // Every tool parents to its step. for (const t of tools) { expect(t.parentSpanContext?.spanId).toBe(spanId(step)); } @@ -324,6 +333,15 @@ test("a failed tool call records ERROR status and an exception event", async () const exc = event(failed, "exception"); expect(exc?.attributes?.["exception.message"]).toBe("kaboom"); expect(exc?.attributes?.["log.severity_number"]).toBe(17); + + // A structured non-Error payload is never exported, only its string form. + const structured = tools.find((t) => t.attributes["gen_ai.tool.name"] === "t3")!; + expect(structured.status.code).toBe(SpanStatusCode.ERROR); + expect(structured.attributes["error.type"]).toBe("tool_error"); + expect(event(structured, "exception")?.attributes?.["exception.message"]).toBe("[object Object]"); + expect( + JSON.stringify(spans.map((s) => ({ attributes: s.attributes, events: s.events }))), + ).not.toContain("private"); }); test("the mask rewrites every exported payload", async () => { @@ -453,33 +471,6 @@ test("gateway provider is relabeled to Vercel AI Gateway on every span", async ( expect(step.attributes["gen_ai.request.model"]).toBe("openai/gpt-4o"); }); -test("non-gateway providers pass through unchanged", async () => { - const { spanBatches, overrides } = makeCapture(); - - const integ = telemetryDev( - { apiKey: "td_live_test", environment: "test", serviceName: "svc" }, - overrides, - ); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "hello", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "hello", finishReason: "stop" }); - - const spans = spanBatches[0]!; - const root = spans.find((s) => s.kind === SpanKind.INTERNAL)!; - const step = spans.find((s) => s.kind === SpanKind.CLIENT)!; - expect(root.attributes["gen_ai.provider.name"]).toBe("openai"); - expect(step.attributes["gen_ai.provider.name"]).toBe("openai"); -}); - test("model warnings become model.warning events at severity 13", async () => { const { spanBatches, overrides } = makeCapture(); @@ -510,23 +501,37 @@ test("model warnings become model.warning events at severity 13", async () => { }); test("no apiKey is a complete no-op (transport never invoked)", async () => { - const { spanBatches, metrics, overrides } = makeCapture(); - const integ = telemetryDev({ environment: "test", serviceName: "svc" }, overrides); + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); - integ.onStart?.({ model, prompt: "x" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "y", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: {}, - }); - await integ.onFinish?.({ text: "y", finishReason: "stop" }); + try { + let calls = 0; + + const integ = telemetryDev({ + environment: "test", + serviceName: "svc", + fetch: async () => { + calls += 1; + + return new Response(null, { status: 200 }); + }, + }); - expect(spanBatches).toHaveLength(0); - expect(metrics).toHaveLength(0); + integ.onStart?.({ model, prompt: "x" }); + integ.onStepStart?.({ stepNumber: 0, model }); + integ.onStepFinish?.({ + stepNumber: 0, + model, + text: "y", + finishReason: "stop", + response: { id: "r", modelId: "gpt-4o" }, + usage: {}, + }); + await integ.onFinish?.({ text: "y", finishReason: "stop" }); + + expect(calls).toBe(0); + } finally { + vi.unstubAllEnvs(); + } }); test("a step with missing usage records no token metric and no summary usage attrs", async () => { @@ -607,8 +612,9 @@ test("export failure reaches onError even on the waitUntil path", async () => { expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe("trace export boom"); }); -test("spans use each emitter's own transport (no cross-talk on a shared cached core)", async () => { +test("spans and metrics use each emitter's own transport (no cross-talk on a shared cached core)", async () => { const traceCalls: string[] = []; + const metricCalls: string[] = []; const aErrors: unknown[] = []; const bErrors: unknown[] = []; @@ -617,6 +623,8 @@ test("spans use each emitter's own transport (no cross-talk on a shared cached c async (url: string | Request | URL, _init?: RequestInit) => { if (requestUrl(url).endsWith("/v1/traces")) traceCalls.push(tag); + if (requestUrl(url).endsWith("/v1/metrics")) metricCalls.push(tag); + return new Response(null, { status: 200 }); }; @@ -638,8 +646,8 @@ test("spans use each emitter's own transport (no cross-talk on a shared cached c onError: (e) => bErrors.push(e), }); - // Run a generation on the FIRST (older) emitter after the second was created: its spans - // must still go through ITS OWN fetch (A), proving span transport is per-call, not shared. + // Run a generation on the FIRST (older) emitter after the second was created: its spans and + // metrics must still go through ITS OWN fetch (A), proving transport is per-call, not shared. integA.onStart?.({ model, prompt: "hi" }); integA.onStepStart?.({ stepNumber: 0, model }); integA.onStepFinish?.({ @@ -654,171 +662,43 @@ test("spans use each emitter's own transport (no cross-talk on a shared cached c expect(traceCalls.length).toBeGreaterThan(0); expect(traceCalls.every((c) => c === "A")).toBe(true); + expect(metricCalls.length).toBeGreaterThan(0); + expect(metricCalls.every((c) => c === "A")).toBe(true); expect(aErrors).toHaveLength(0); expect(bErrors).toHaveLength(0); }); -test("trace export retries a transient ingest failure", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_trace_retry", - environment: "test", - serviceName: "trace-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; - - return new Response(null, { status: traceCalls === 1 ? 503 : 200 }); - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(2); - expect(errors).toHaveLength(0); -}); - -test("trace export retries a transient network error", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_trace_network_retry", - environment: "test", - serviceName: "trace-network-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; - - if (traceCalls === 1) throw new Error("socket closed"); - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(2); - expect(errors).toHaveLength(0); -}); - -test("trace export reports one error after exhausting network failures", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - const networkError = new Error("socket closed"); - - const integ = telemetryDev({ - apiKey: "td_live_trace_network_exhausted", - environment: "test", - serviceName: "trace-network-exhausted", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; - throw networkError; - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(3); - expect(errors).toHaveLength(1); - expect(errors[0]).toBe(networkError); -}); - -test("trace export does not retry non-retryable ingest failures", async () => { - const errors: unknown[] = []; - let traceCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_trace_no_retry", - environment: "test", - serviceName: "trace-no-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/traces")) { - traceCalls += 1; +const networkError = new Error("socket closed"); - return new Response(null, { status: 400 }); - } - - return new Response(null, { status: 200 }); +test.each([ + { + name: "network failures", + respond: (): Response => { + throw networkError; }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(traceCalls).toBe(1); - expect(errors).toHaveLength(1); - expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe( - "telemetry.dev trace ingest failed: 400", - ); -}); - -test("trace export reports one error after exhausting retryable failures", async () => { + check: (error: unknown) => expect(error).toBe(networkError), + }, + { + name: "retryable 503 responses", + respond: () => new Response(null, { status: 503 }), + check: (error: unknown) => + expect(error instanceof Error ? error.message : undefined).toBe( + "telemetry.dev trace ingest failed: 503", + ), + }, +])("trace export reports one error after exhausting $name", async ({ respond, check }) => { const errors: unknown[] = []; let traceCalls = 0; const integ = telemetryDev({ - apiKey: "td_live_trace_retry_exhausted", + apiKey: "td_live_trace_exhausted", environment: "test", - serviceName: "trace-retry-exhausted", + serviceName: "trace-exhausted", fetch: async (url: string | Request | URL, _init?: RequestInit) => { if (requestUrl(url).endsWith("/v1/traces")) { traceCalls += 1; - return new Response(null, { status: 503 }); + return respond(); } return new Response(null, { status: 200 }); @@ -840,9 +720,7 @@ test("trace export reports one error after exhausting retryable failures", async expect(traceCalls).toBe(3); expect(errors).toHaveLength(1); - expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe( - "telemetry.dev trace ingest failed: 503", - ); + check(errors[0]); }); test("step metrics attribute tokens and duration to each step's own model, not the root", async () => { @@ -910,93 +788,6 @@ test("step metrics attribute tokens and duration to each step's own model, not t expect(openaiOutput?.value).toBe(5); }); -test("metrics use each emitter's own transport (no cross-talk on a shared cached core)", async () => { - const metricCalls: string[] = []; - const aErrors: unknown[] = []; - const bErrors: unknown[] = []; - - const makeFetch = - (tag: string): typeof fetch => - async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/metrics")) metricCalls.push(tag); - - return new Response(null, { status: 200 }); - }; - - // First emitter caches a core under identity "dupm" with fetch A (real metrics pipeline). - const integA = telemetryDev({ - apiKey: "td_live_dupm", - environment: "test", - serviceName: "dupm", - fetch: makeFetch("A"), - onError: (e) => aErrors.push(e), - }); - - // Second emitter reuses that cached core but supplies fetch B. - telemetryDev({ - apiKey: "td_live_dupm", - environment: "test", - serviceName: "dupm", - fetch: makeFetch("B"), - onError: (e) => bErrors.push(e), - }); - - // Run a generation on the FIRST (older) emitter after the second was created: its metrics must - // flush through ITS OWN fetch (A). Before the fix the shared core transport was last-writer (B). - integA.onStart?.({ model, prompt: "hi" }); - integA.onStepStart?.({ stepNumber: 0, model }); - integA.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integA.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(metricCalls.length).toBeGreaterThan(0); - expect(metricCalls.every((c) => c === "A")).toBe(true); - expect(aErrors).toHaveLength(0); - expect(bErrors).toHaveLength(0); -}); - -test("metric export retries a transient ingest failure", async () => { - const errors: unknown[] = []; - let metricCalls = 0; - - const integ = telemetryDev({ - apiKey: "td_live_metric_retry", - environment: "test", - serviceName: "metric-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - if (requestUrl(url).endsWith("/v1/metrics")) { - metricCalls += 1; - - return new Response(null, { status: metricCalls === 1 ? 503 : 200 }); - } - - return new Response(null, { status: 200 }); - }, - onError: (e) => errors.push(e), - }); - - integ.onStart?.({ model, prompt: "hi" }); - integ.onStepStart?.({ stepNumber: 0, model }); - integ.onStepFinish?.({ - stepNumber: 0, - model, - text: "ok", - finishReason: "stop", - response: { id: "r", modelId: "gpt-4o" }, - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - - expect(metricCalls).toBe(2); - expect(errors).toHaveLength(0); -}); - test("a non-OK /v1/metrics response is reported as a failed export, not a success", async () => { const errors: unknown[] = []; const exportErrors: unknown[] = []; @@ -1032,10 +823,11 @@ test("a non-OK /v1/metrics response is reported as a failed export, not a succes }); await integ.onFinish?.({ text: "ok", finishReason: "stop" }); - // Our own callback sees the 503 once the retries are exhausted. - expect( - errors.some((e) => e instanceof Error && /metric ingest failed: 503/.test(e.message)), - ).toBe(true); + // Our own callback sees the 503 once, after the retries are exhausted. + expect(errors).toHaveLength(1); + expect(errors[0] instanceof Error ? errors[0].message : undefined).toBe( + "telemetry.dev metric ingest failed: 503", + ); expect(metricCalls).toBe(3); // The export is marked FAILED, so OTel rethrows it to the global handler — this is the bit the // fix changes (code 0 → code 1). diff --git a/packages/ai/tests/integration_v7.test.ts b/packages/ai/tests/integration_v7.test.ts index b39b2da..0a4079f 100644 --- a/packages/ai/tests/integration_v7.test.ts +++ b/packages/ai/tests/integration_v7.test.ts @@ -9,14 +9,9 @@ import { TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; import { sessionSpanContext } from "@telemetry-dev/otel"; -import { expect, test } from "vitest"; +import { expect, test, vi } from "vitest"; import { telemetryDev } from "../src/index.ts"; -import { unknownErrorMessage } from "../src/shared.ts"; - -test("unknown errors do not expose structured payloads", () => { - expect(unknownErrorMessage({ code: "tool_failed", secret: "private" })).toBe("[object Object]"); -}); interface MetricRecord { metric: "duration" | "tokens"; @@ -299,40 +294,30 @@ test("missing or empty runtimeContext emits no identity or metadata attributes", } }); -test("onStepFinish is not implemented, so ai@7's step-end fan-out cannot double spans", async () => { +test("ai@7 step-end fan-out records each step once", async () => { + const { generateText } = await import("ai"); + const { MockLanguageModelV4 } = await import("ai/test"); const { spanBatches, metrics, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); - const callId = "call-dedupe"; - const stepResult = { - callId, - stepNumber: 0, - model, - text: "once", - finishReason: "stop", - response: { id: "r1", modelId: "gpt-4o" }, - usage: { inputTokens: 4, outputTokens: 2 }, - performance: { responseTimeMs: 1000 }, - }; - - integ.onStart?.({ - callId, - operationId: "ai.generateText", - provider: "openai", - modelId: "gpt-4o", - messages: [{ role: "user", content: "x" }], - }); - integ.onStepStart?.({ callId, stepNumber: 0 }); - integ.onStepEnd?.(stepResult); - // ai@7 fans step-end out to BOTH onStepEnd and the deprecated onStepFinish, calling each - // implemented hook once. The integration must not declare onStepFinish, or every step would be - // recorded twice. - expect(Object.keys(integ)).not.toContain("onStepFinish"); - await integ.onEnd?.({ - callId, - text: "once", - finishReason: "stop", - usage: stepResult.usage, + // ai@7 fans step-end out to both onStepEnd and the deprecated onStepFinish, so an integration + // that implemented both would record every step twice. + await generateText({ + model: new MockLanguageModelV4({ + provider: "openai", + modelId: "gpt-4o", + doGenerate: { + content: [{ type: "text", text: "once" }], + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 4, noCache: 4, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 2, text: 2, reasoning: undefined }, + }, + warnings: [], + }, + }), + prompt: "x", + telemetry: { integrations: [integ] }, }); expect(spanBatches).toHaveLength(1); @@ -342,6 +327,53 @@ test("onStepFinish is not implemented, so ai@7's step-end fan-out cannot double expect(metrics.filter((m) => m.metric === "duration")).toHaveLength(1); }); +test("no apiKey is a complete no-op (transport never invoked)", async () => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + + try { + let calls = 0; + + const integ = telemetryDev({ + environment: "test", + serviceName: "svc", + fetch: async () => { + calls += 1; + + return new Response(null, { status: 200 }); + }, + }); + + const callId = "no-key"; + + integ.onStart?.({ + callId, + operationId: "ai.generateText", + provider: "openai", + modelId: "gpt-4o", + messages: [{ role: "user", content: "x" }], + }); + integ.onStepStart?.({ callId, stepNumber: 0 }); + integ.onStepEnd?.({ + callId, + stepNumber: 0, + model, + text: "y", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + await integ.onEnd?.({ + callId, + text: "y", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + + expect(calls).toBe(0); + } finally { + vi.unstubAllEnvs(); + } +}); + test("two interleaved callIds on one integration produce disjoint traces per flush", async () => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); @@ -1147,70 +1179,25 @@ test("provider results recover the latest matching input from a later generation ); }); -test("provider results recover inputs from prepareStep message replacements", async () => { - const { spanBatches, overrides } = makeCapture(); - const integ = telemetryDev(baseOptions, overrides); - const callId = "call-provider-prepare-step-history"; - - const providerCall = { - type: "tool-call", - toolCallId: "deferred-tool", - toolName: "web_search", - input: { query: "initial" }, - providerExecuted: true, - }; - - const messages = [{ role: "assistant", content: [providerCall] }]; - - const providerResult = { - type: "tool-result", - toolCallId: "deferred-tool", - toolName: "web_search", - output: { result: "prepared result" }, - providerExecuted: true, - }; - - integ.onStart?.({ - callId, - operationId: "ai.generateText", - provider: "openai", - modelId: "gpt-4o", - messages, - }); - providerCall.input = { query: "prepared" }; - integ.onStepStart?.({ callId, stepNumber: 0, messages }); - integ.onLanguageModelCallEnd?.({ callId, content: [providerResult] }); - integ.onStepEnd?.({ - callId, - stepNumber: 0, - model, - content: [providerResult], - text: "done", - finishReason: "stop", - usage: { inputTokens: 1, outputTokens: 1 }, - }); - await integ.onEnd?.({ - callId, - text: "done", - finishReason: "stop", - usage: { inputTokens: 1, outputTokens: 1 }, - }); - - const tools = byOperation(spanBatches[0]!, "execute_tool"); - - expect(tools).toHaveLength(1); - expect(tools[0]!.attributes["gen_ai.tool.call.arguments"]).toBe( - JSON.stringify({ query: "prepared" }), - ); - expect(tools[0]!.attributes["gen_ai.tool.call.result"]).toBe( - JSON.stringify({ result: "prepared result" }), - ); -}); - -test("provider results recover inputs from nested history mutations", async () => { +test.each([ + { + name: "prepareStep message replacements", + mutate: (call: { input: { query: string } }) => { + call.input = { query: "prepared" }; + }, + query: "prepared", + }, + { + name: "nested history mutations", + mutate: (call: { input: { query: string } }) => { + call.input.query = "mutated"; + }, + query: "mutated", + }, +])("provider results recover inputs from $name", async ({ mutate, query }) => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); - const callId = "call-provider-history-nested-mutation"; + const callId = `call-provider-history-${query}`; const providerCall = { type: "tool-call", @@ -1226,7 +1213,7 @@ test("provider results recover inputs from nested history mutations", async () = type: "tool-result", toolCallId: "deferred-tool", toolName: "web_search", - output: { result: "mutated result" }, + output: { result: `${query} result` }, providerExecuted: true, }; @@ -1237,7 +1224,7 @@ test("provider results recover inputs from nested history mutations", async () = modelId: "gpt-4o", messages, }); - providerCall.input.query = "mutated"; + mutate(providerCall); integ.onStepStart?.({ callId, stepNumber: 0, messages }); integ.onLanguageModelCallEnd?.({ callId, content: [providerResult] }); integ.onStepEnd?.({ @@ -1259,11 +1246,9 @@ test("provider results recover inputs from nested history mutations", async () = const tools = byOperation(spanBatches[0]!, "execute_tool"); expect(tools).toHaveLength(1); - expect(tools[0]!.attributes["gen_ai.tool.call.arguments"]).toBe( - JSON.stringify({ query: "mutated" }), - ); + expect(tools[0]!.attributes["gen_ai.tool.call.arguments"]).toBe(JSON.stringify({ query })); expect(tools[0]!.attributes["gen_ai.tool.call.result"]).toBe( - JSON.stringify({ result: "mutated result" }), + JSON.stringify({ result: `${query} result` }), ); }); @@ -3840,6 +3825,13 @@ test.each([ : undefined, ); } + + const exported = JSON.stringify( + spanBatches[0]!.map((span) => ({ attributes: span.attributes, events: span.events })), + ); + + expect(/private-action|private-policy/.test(exported)).toBe(recordInputs); + expect(exported.includes("probability")).toBe(recordOutputs); }, ); @@ -3983,43 +3975,6 @@ test("evaluation hooks emit correlated root and model spans without duplicate to ]); }); -test("evaluation capture flags redact payloads while preserving identity and usage", async () => { - const { spanBatches, overrides } = makeCapture(); - const integ = telemetryDev(baseOptions, overrides); - const callId = "evaluation-redacted"; - - integ.experimental_onEvaluateStart?.({ - callId, - provider: "openai", - modelId: "judge", - state: { secret: "input" }, - questions: { secret: { type: "score", question: "private" } }, - recordInputs: false, - recordOutputs: false, - }); - integ.experimental_onEvaluationModelCallStart?.({ callId }); - integ.experimental_onEvaluationModelCallEnd?.({ - callId, - answers: { secret: { type: "score", score: 1 } }, - usage: { inputTokens: 4, outputTokens: 2 }, - }); - await integ.experimental_onEvaluateEnd?.({ - callId, - provider: "openai", - modelId: "judge", - answers: { secret: { type: "score", score: 1 } }, - usage: { inputTokens: 4, outputTokens: 2 }, - }); - - const spans = spanBatches[0]!; - expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("secret"); - expect(spans.every((span) => span.attributes["gen_ai.input.messages"] === undefined)).toBe(true); - expect(spans.every((span) => span.attributes["gen_ai.output.messages"] === undefined)).toBe(true); - expect( - spans.find((span) => span.kind === SpanKind.CLIENT)?.attributes["gen_ai.usage.input_tokens"], - ).toBe(4); -}); - test("overlapping evaluations stay isolated and errors clean up their call state", async () => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); diff --git a/packages/ai/tests/type_conformance.test.ts b/packages/ai/tests/type_conformance.test.ts index d69a383..a502e33 100644 --- a/packages/ai/tests/type_conformance.test.ts +++ b/packages/ai/tests/type_conformance.test.ts @@ -1,23 +1,14 @@ import type { Telemetry } from "ai"; import type { TelemetryIntegration } from "ai-v6"; -import { expect, test } from "vitest"; +import { expectTypeOf, test } from "vitest"; import { telemetryDev } from "../src/index.ts"; import { telemetryDev as telemetryDevV6 } from "../src/v6.ts"; -// Compile-time contract: the emitted types never import from `ai`, so each entry's integration -// object must stay structurally assignable to its major's interface (v6 `TelemetryIntegration` -// was removed in v7; v7 `Telemetry` does not exist in v6). A break here fails typecheck, not -// runtime. -const integ = telemetryDev({ apiKey: "td_live_t" }); - -void (integ satisfies Telemetry); - -const integV6 = telemetryDevV6({ apiKey: "td_live_t" }); - -void (integV6 satisfies TelemetryIntegration); - -test("each entry's telemetryDev satisfies its ai major's integration type", () => { - expect(integ).toBeDefined(); - expect(integV6).toBeDefined(); +// The emitted types never import from `ai`, so each entry's integration object must stay +// structurally assignable to its major's interface (v6 `TelemetryIntegration` was removed in v7; +// v7 `Telemetry` does not exist in v6). A break fails `vp check`, not the test run. +test("each entry's telemetryDev is assignable to its ai major's integration type (compile-time)", () => { + expectTypeOf(telemetryDev).returns.toExtend(); + expectTypeOf(telemetryDevV6).returns.toExtend(); }); diff --git a/packages/ai/tests/wrappers.test.ts b/packages/ai/tests/wrappers.test.ts index f5cedea..7510ec4 100644 --- a/packages/ai/tests/wrappers.test.ts +++ b/packages/ai/tests/wrappers.test.ts @@ -44,7 +44,8 @@ test("media wrappers preserve structural function types and record metadata with .images[0]!.uint8Array, ).toBe(bytes); expect(spans).toHaveLength(1); - expect(spans[0]!.attributes).toMatchObject({ + // An exact map: an attribute that carries the bytes in any encoding fails here. + expect(spans[0]!.attributes).toEqual({ "gen_ai.operation.name": "generate_content", "gen_ai.output.type": "image", "gen_ai.provider.name": "openai", @@ -54,7 +55,6 @@ test("media wrappers preserve structural function types and record metadata with "gen_ai.usage.image.output_tokens": 2, "td.ai.output.image_count": 1, }); - expect(JSON.stringify(spans[0]!.attributes)).not.toContain("1,2,3"); }); test("media wrappers preserve rejection and synchronous throw identity", async () => { @@ -143,6 +143,11 @@ test("throwing telemetry overrides do not replace provider results or errors", a await Promise.resolve(); expect(durationAttempts).toBe(1); expect(spans).toHaveLength(1); + expect(telemetryErrors.map((error) => (error as Error).message)).toEqual([ + "tokens failed", + "duration failed", + ]); + telemetryErrors.length = 0; const providerError = new Error("provider failed"); expect(() => @@ -154,7 +159,8 @@ test("throwing telemetry overrides do not replace provider results or errors", a overrides, )(), ).toThrow(providerError); - expect(telemetryErrors.length).toBeGreaterThan(0); + expect(durationAttempts).toBe(2); + expect(telemetryErrors.map((error) => (error as Error).message)).toEqual(["duration failed"]); }); test("throwing telemetry setup and one-shot input getters preserve exact provider outcomes", () => { diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index bc1cd00..cc10ede 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -551,40 +551,6 @@ test("messages.create streaming preserves events and records aggregated text usa ).toBe(true); }); -test("stream capture is bounded without changing delivered events", async () => { - const spans = setupSpans(); - - const events: JsonRecord[] = [ - { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, - ...Array.from({ length: 80 }, () => ({ - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "x".repeat(1024) }, - })), - { type: "message_stop" }, - ]; - - const fake = createFakeFetch(namedSseResponse(events)); - - const stream = await clientWith(fake.fetch).messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [], - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); - - const output = jsonAttr }>>( - span, - "gen_ai.output.messages", - ); - - expect(output[0]!.content[0]!.text.length).toBeLessThan(48 * 1024); -}); - test("stream capture retains no output when output capture is disabled", async () => { const spans = setupSpans(undefined, false); const capture = vi.spyOn(telemetrySdk, "boundedCaptureDetails"); @@ -642,44 +608,6 @@ test("wrapAnthropic observes a CommonJS client subclass stream", async () => { ]); }); -test("messages.create streaming observes duck-typed stream responses", async () => { - const spans = setupSpans(); - - class AlternateStream implements AsyncIterable { - private abortController = new AbortController(); - controller = { - abort: (reason?: unknown) => this.abortController.abort(reason), - signal: this.abortController.signal, - }; - - async *[Symbol.asyncIterator](): AsyncIterator { - for (const event of streamEvents()) yield event; - } - } - - const client = wrapAnthropic({ - messages: { - create: (_params: JsonValue) => Promise.resolve(new AlternateStream()), - }, - }); - - const stream = await client.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Say hello" }], - stream: true, - }); - - await collectStream(stream); - - const span = await exportedSpan(spans); - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: [{ type: "text", text: "Hello world" }] }, - ]); - expect(span.attributes["gen_ai.usage.input_tokens"]).toBe(9); - expect(span.attributes["gen_ai.usage.output_tokens"]).toBe(2); -}); - test("message timestamps precede telemetry mapping", async () => { const spans = setupSpans(); let now = 0; @@ -1026,48 +954,6 @@ test("messages.create streaming keeps telemetry on a core span without a capture ]); }); -test("messages.create streaming retains output when the core span lacks chunk recording", async () => { - const spans = setupSpans(); - const startSpan = telemetrySdk.startSpan; - vi.spyOn(telemetrySdk, "startSpan").mockImplementation((...args) => { - const legacySpan = { ...startSpan(...args) }; - Reflect.deleteProperty(legacySpan, "recordOutputChunk"); - - return legacySpan; - }); - - const events = [ - { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "delivered" }, - }, - { type: "message_stop" }, - ]; - - const source = { - controller: new AbortController(), - async *[Symbol.asyncIterator]() { - yield* events; - }, - }; - - const client = wrapAnthropic({ messages: { create: async (_params: unknown) => source } }); - - const stream = await client.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [], - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - expect(jsonAttr(await exportedSpan(spans), "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: [{ type: "text", text: "delivered" }] }, - ]); -}); - test("messages.create streaming leaves the configured attribute limit to the core", async () => { const spans = setupSpans(undefined, true, undefined, { maxAttributeLength: 128 }); @@ -1211,30 +1097,13 @@ test("uninstrumentAnthropic preserves a method installed after its own wrapper", } }); -test("wrapAnthropic keeps a client instrumented after global instrumentation is removed", async () => { - const spans = setupSpans(); - instrumentAnthropic(); - const fake = createFakeFetch(jsonResponse(messagePayload({ id: "msg_instance_wrapped" }))); - const client = wrapAnthropic(new Anthropic({ apiKey: "test", fetch: fake.fetch, maxRetries: 0 })); - uninstrumentAnthropic(); - - await client.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Hi wrapped" }], - }); - - const span = await exportedSpan(spans); - expect(fake.requests).toHaveLength(1); - expect(span.attributes["gen_ai.request.model"]).toBe("claude-sonnet-4-6"); -}); - test("instrumentAnthropic and wrapAnthropic together record one span per call", async () => { const spans = setupSpans(); instrumentAnthropic(); const fake = createFakeFetch( jsonResponse(messagePayload({ id: "msg_global_wrapped" })), + jsonResponse(messagePayload({ id: "msg_beta_both" })), jsonResponse(messagePayload({ id: "msg_instance_wrapped_after_global" })), ); @@ -1245,10 +1114,18 @@ test("instrumentAnthropic and wrapAnthropic together record one span per call", max_tokens: 64, messages: [{ role: "user", content: "Hi wrapped" }], }); + await client.beta.messages.create({ + model: "claude-sonnet-4-6", + max_tokens: 64, + messages: [{ role: "user", content: "Hi beta" }], + }); - const activeSpans = await finishedSpans(spans, 1); - expect(fake.requests).toHaveLength(1); - expect(activeSpans[0]!.attributes["gen_ai.request.model"]).toBe("claude-sonnet-4-6"); + const activeSpans = await finishedSpans(spans, 2); + expect(fake.requests).toHaveLength(2); + expect(activeSpans.map((span) => span.attributes["gen_ai.response.id"])).toEqual([ + "msg_global_wrapped", + "msg_beta_both", + ]); uninstrumentAnthropic(); await client.messages.create({ @@ -1257,9 +1134,9 @@ test("instrumentAnthropic and wrapAnthropic together record one span per call", messages: [{ role: "user", content: "Hi after uninstrument" }], }); - const allSpans = await finishedSpans(spans, 2); - expect(fake.requests).toHaveLength(2); - expect(allSpans[1]!.attributes["gen_ai.request.model"]).toBe("claude-sonnet-4-6"); + const allSpans = await finishedSpans(spans, 3); + expect(fake.requests).toHaveLength(3); + expect(allSpans[2]!.attributes["gen_ai.response.id"]).toBe("msg_instance_wrapped_after_global"); }); const providerClients = { @@ -1298,25 +1175,6 @@ test.each<{ client: keyof typeof providerClients; subclass: boolean; expected: s }, ); -test("wrapAnthropic records Bedrock provider names from client constructor names", async () => { - const spans = setupSpans(); - - class AnthropicBedrock { - messages = { - create: async (_params: JsonValue) => messagePayload({ id: "msg_bedrock" }), - }; - } - - await wrapAnthropic(new AnthropicBedrock()).messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Hi" }], - }); - - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.provider.name"]).toBe("aws.bedrock"); -}); - test("beta.messages.create records a generation span for the beta endpoint", async () => { const spans = setupSpans(); const messages = [{ role: "user" as const, content: "Say hello" }]; @@ -1461,22 +1319,6 @@ test("instrumentAnthropic covers beta.messages and uninstrumentAnthropic restore expect(spans.getFinishedSpans()).toHaveLength(1); }); -test("instrumentAnthropic and wrapAnthropic together record one beta span per call", async () => { - const spans = setupSpans(); - instrumentAnthropic(); - const fake = createFakeFetch(jsonResponse(messagePayload({ id: "msg_beta_both" }))); - const client = wrapAnthropic(new Anthropic({ apiKey: "test", fetch: fake.fetch, maxRetries: 0 })); - - await client.beta.messages.create({ - model: "claude-sonnet-4-6", - max_tokens: 64, - messages: [{ role: "user", content: "Hi" }], - }); - - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.id"]).toBe("msg_beta_both"); -}); - function betaStreamEvents(...blocks: [JsonRecord, JsonRecord[]][]): JsonRecord[] { const events: JsonRecord[] = [ { @@ -1543,33 +1385,28 @@ function fallbackStream(iterations: JsonRecord[]): JsonRecord[] { return events; } -test.each([false, true])( - "stream output over the encoded limit keeps a bounded prefix (masked: %s)", - async (masked) => { - const spans = setupSpans(undefined, true, masked ? (value) => value : undefined); - - const events = betaStreamEvents( - [{ type: "text", text: "" }, [{ type: "text_delta", text: "first" }]], - [{ type: "text", text: "" }, [{ type: "text_delta", text: '"'.repeat(30_000) }]], - ); - - const span = await streamedBetaSpan(events, spans); +test("a stream delta over the encoded limit is dropped and earlier output is kept", async () => { + const spans = setupSpans(); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); + const events = betaStreamEvents( + [{ type: "text", text: "" }, [{ type: "text_delta", text: "first" }]], + [{ type: "text", text: "" }, [{ type: "text_delta", text: '"'.repeat(30_000) }]], + ); - if (masked) expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); - else { - const output = jsonAttr }>>( - span, - "gen_ai.output.messages", - ); + const span = await streamedBetaSpan(events, spans); - expect(String(span.attributes["gen_ai.output.messages"])).not.toContain("[truncated]"); - expect(output[0]?.content[0]).toEqual({ type: "text", text: "first" }); - expect('"'.repeat(30_000).startsWith(output[0]?.content[1]?.text ?? "")).toBe(true); - } - }, -); + expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); + expect(String(span.attributes["gen_ai.output.messages"])).not.toContain("[truncated]"); + expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ + { + role: "assistant", + content: [ + { type: "text", text: "first" }, + { type: "text", text: "" }, + ], + }, + ]); +}); test.each([ [400, true], @@ -1595,7 +1432,7 @@ test.each([ }, ); -test("deltas that fill the stream limit keep the text that was accepted", async () => { +test("deltas that fill the stream limit keep the accepted text and deliver every event", async () => { const spans = setupSpans(); const events = betaStreamEvents([ @@ -1606,7 +1443,17 @@ test("deltas that fill the stream limit keep the text that was accepted", async ], ]); - const span = await streamedBetaSpan(events, spans); + const stream = await clientWith( + createFakeFetch(namedSseResponse(events)).fetch, + ).beta.messages.create({ + model: "claude-opus-5", + max_tokens: 64, + messages: [{ role: "user", content: "Hi" }], + stream: true, + }); + + expect(await collectStream(stream)).toEqual(events); + const span = await exportedSpan(spans); const output = jsonAttr }>>( span, @@ -1698,18 +1545,6 @@ const servedIteration = { output_tokens: 2, }; -test("beta streams record the fallback model when a fallback_message iteration served it", async () => { - const span = await streamedBetaSpan(fallbackStream([declinedIteration, servedIteration])); - - expect(span.attributes["gen_ai.response.model"]).toBe("claude-opus-4-8"); -}); - -test("beta streams keep the requested model when the fallback request failed", async () => { - const span = await streamedBetaSpan(fallbackStream([declinedIteration])); - - expect(span.attributes["gen_ai.response.model"]).toBe("claude-opus-5"); -}); - const advisorIteration = { ...declinedIteration, type: "advisor_message", @@ -1798,6 +1633,12 @@ test("fallback metadata overflow preserves complete masked output", async () => }); test.each<{ name: string; iterations: JsonRecord[]; expected: string }>([ + { + name: "fallback served", + iterations: [declinedIteration, servedIteration], + expected: "claude-opus-4-8", + }, + { name: "fallback failed", iterations: [declinedIteration], expected: "claude-opus-5" }, { name: "advisor without fallback", iterations: [declinedIteration, advisorIteration], @@ -1818,7 +1659,7 @@ test.each<{ name: string; iterations: JsonRecord[]; expected: string }>([ expected: "claude-sonnet-5", }, ])( - "beta streams ignore advisor iterations for the served model: $name", + "beta streams take the served model from the last fallback_message iteration: $name", async ({ iterations, expected }) => { const span = await streamedBetaSpan(fallbackStream(iterations)); @@ -2299,6 +2140,7 @@ async function usageSpan( }); const { cache_creation: _split, ...deltaUsage } = finalUsage; + const abortController = new AbortController(); const events = [ { @@ -2310,8 +2152,12 @@ async function usageSpan( { type: "message_stop" }, ]; + // A plain { abort, signal } controller: stream detection must not require an AbortController. const source = { - controller: new AbortController(), + controller: { + abort: (reason?: unknown) => abortController.abort(reason), + signal: abortController.signal, + }, async *[Symbol.asyncIterator]() { yield* events; }, diff --git a/packages/bedrock/src/internal.ts b/packages/bedrock/src/internal.ts index 5a231fc..104f595 100644 --- a/packages/bedrock/src/internal.ts +++ b/packages/bedrock/src/internal.ts @@ -351,7 +351,7 @@ export function parseJson(value: T): JsonValue { } // Retention uses fixed bounds; the core applies maxAttributeLength after the mask. -export const STREAM_CAPTURE_MAX_BYTES = 48 * 1024; +const STREAM_CAPTURE_MAX_BYTES = 48 * 1024; export const STREAM_CAPTURE_MAX_ITEMS = 1_000; const CAPTURE_TRUNCATED_ATTRIBUTE = "telemetry.dev.capture.truncated"; diff --git a/packages/bedrock/tests/agents.test.ts b/packages/bedrock/tests/agents.test.ts index 07bcc4d..de39030 100644 --- a/packages/bedrock/tests/agents.test.ts +++ b/packages/bedrock/tests/agents.test.ts @@ -17,7 +17,16 @@ import { uninstrumentBedrockAgents, wrapBedrockAgents, } from "../src/agents.ts"; -import { FakeClient, bytes, collect, jsonAttr, setup, streamOf, teardown } from "./helpers.ts"; +import { + FakeClient, + bytes, + collect, + jsonAttr, + jsonRequestHandler, + setup, + streamOf, + teardown, +} from "./helpers.ts"; afterEach(async () => { await teardown(); @@ -1021,50 +1030,63 @@ test("modeled stream errors from Agent, RAG, and Flow mark spans as errors", asy expect(jsonAttr(flowSpan, "gen_ai.output.messages")).toEqual([{ document: { value: 1 } }]); }); -test("global agents instrumentation is idempotent", async () => { +test("global agents instrumentation patches the real client once and restores its inherited send", async () => { const spans = setup(); - - const originalDescriptor = Object.getOwnPropertyDescriptor( - BedrockAgentRuntimeClient.prototype, - "send", - ); - - const fakeSend = async () => ({ - completion: streamOf([{ chunk: { bytes: bytes("global") } }]), - $metadata: { requestId: "agent-global-1" }, + const proto = BedrockAgentRuntimeClient.prototype; + const command = new RetrieveCommand({ knowledgeBaseId: "kb-1", retrievalQuery: { text: "q" } }); + + const client = new BedrockAgentRuntimeClient({ + region: "us-east-1", + credentials: { accessKeyId: "test", secretAccessKey: "test" }, + maxAttempts: 1, + requestHandler: jsonRequestHandler("agent-global-1", { + retrievalResults: [{ content: { text: "doc" } }], + }), }); - Object.defineProperty(BedrockAgentRuntimeClient.prototype, "send", { - configurable: true, - writable: true, - value: fakeSend, - }); + // AWS clients inherit send from the Smithy base class. + expect(Object.hasOwn(proto, "send")).toBe(false); try { instrumentBedrockAgents(); instrumentBedrockAgents(); - const client = new BedrockAgentRuntimeClient({ - region: "us-east-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - }); + await client.send(command); + expect(spans.getFinishedSpans()).toHaveLength(1); + expect(spans.getFinishedSpans()[0]!.attributes["gen_ai.response.id"]).toBe("agent-global-1"); - const response = (await client.send( - new InvokeAgentCommand({ agentId: "agent-1", agentAliasId: "alias", sessionId: "sess" }), - )) as { completion: AsyncIterable }; + const telemetryWrapper = Object.getOwnPropertyDescriptor(proto, "send")?.value; + const laterWrapper = async () => ({ retrievalResults: [] }); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: laterWrapper, + }); + uninstrumentBedrockAgents(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(laterWrapper); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: telemetryWrapper, + }); + uninstrumentBedrockAgents(); + expect(Object.hasOwn(proto, "send")).toBe(false); - await collect(response.completion); + await client.send(command); expect(spans.getFinishedSpans()).toHaveLength(1); - expect(spans.getFinishedSpans()[0]!.attributes["gen_ai.response.id"]).toBe("agent-global-1"); + + const priorOwner = async () => ({ retrievalResults: [] }); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: priorOwner, + }); + instrumentBedrockAgents(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).not.toBe(priorOwner); uninstrumentBedrockAgents(); - expect( - Object.getOwnPropertyDescriptor(BedrockAgentRuntimeClient.prototype, "send")?.value, - ).toBe(fakeSend); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(priorOwner); } finally { uninstrumentBedrockAgents(); - - if (originalDescriptor) { - Object.defineProperty(BedrockAgentRuntimeClient.prototype, "send", originalDescriptor); - } + Reflect.deleteProperty(proto, "send"); } }); diff --git a/packages/bedrock/tests/bedrock.test.ts b/packages/bedrock/tests/bedrock.test.ts index 40b217d..8e94b8f 100644 --- a/packages/bedrock/tests/bedrock.test.ts +++ b/packages/bedrock/tests/bedrock.test.ts @@ -12,12 +12,21 @@ import { afterEach, expect, test, vi } from "vitest"; import type { MaskFn } from "@telemetry-dev/sdk"; import { instrumentBedrock, uninstrumentBedrock, wrapBedrock } from "../src/index.ts"; -import { STREAM_CAPTURE_MAX_BYTES } from "../src/internal.ts"; -import { FakeClient, bytes, collect, jsonAttr, setup, streamOf, teardown } from "./helpers.ts"; +import { + FakeClient, + bytes, + collect, + jsonAttr, + jsonRequestHandler, + setup, + streamOf, + teardown, +} from "./helpers.ts"; afterEach(async () => { uninstrumentBedrock(); await teardown(); + vi.restoreAllMocks(); }); test("Converse captures normalized messages, usage, metadata, request id, and sampling", async () => { @@ -267,22 +276,15 @@ test("masked ConverseStream early break withholds partial output", async () => { expect(seen.some((value) => value.includes("partial secret"))).toBe(false); }); -test.each([ - [false, 0], - [false, 100], - [false, 1024 * 1024], - [true, 0], - [true, 100], - [true, 1024 * 1024], -])( - "ConverseStream past the fixed bound is flagged (masked %s, maxAttributeLength %i)", - async (masked, maxAttributeLength) => { +test.each([false, true])( + "ConverseStream past the fixed bound is flagged even when the cap is larger (masked %s)", + async (masked) => { const seen: string[] = []; - const options: NonNullable[0]> = { maxAttributeLength }; - if (masked) options.mask = recordingMask(seen); - - const spans = setup(options); + const spans = setup({ + maxAttributeLength: 1024 * 1024, + ...(masked ? { mask: recordingMask(seen) } : {}), + }); await streamedConverse(textStream("x".repeat(100), 1_100)); @@ -293,15 +295,11 @@ test.each([ if (masked) { expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); expect(seen.some((value) => value.includes("x".repeat(100)))).toBe(false); - } else if (maxAttributeLength > STREAM_CAPTURE_MAX_BYTES) { + } else { const retained = jsonAttr(span, "gen_ai.output.messages")[0].parts[0].content as string; expect(retained.length).toBeGreaterThan(0); - expect(retained.length).toBeLessThan(1_100 * 100); + expect(retained.length).toBeLessThanOrEqual(48 * 1024); expect(retained).toBe("x".repeat(retained.length)); - } else { - expect(String(span.attributes["gen_ai.output.messages"] ?? "").length).toBeLessThanOrEqual( - maxAttributeLength, - ); } }, ); @@ -462,6 +460,17 @@ test("InvokeModel captures provider-native bodies and embedding spans", async () ); const [chat, embedding] = spans.getFinishedSpans(); + expect(jsonAttr(chat, "gen_ai.input.messages")).toEqual({ + messages: [], + max_tokens: 10, + top_k: 3, + }); + expect(jsonAttr(chat, "gen_ai.output.messages")).toEqual({ + content: [{ type: "text", text: "answer" }], + usage: { input_tokens: 9, output_tokens: 4, cache_read_input_tokens: 1 }, + stop_reason: "end_turn", + }); + expect(chat.attributes["gen_ai.usage.input_tokens"]).toBe(9 + 1); expect(chat.attributes["gen_ai.request.top_k"]).toBe(3); expect(chat.attributes["gen_ai.usage.cache_read.input_tokens"]).toBe(1); expect(chat.attributes["gen_ai.response.id"]).toBe("invoke-1"); @@ -833,7 +842,6 @@ test("InvokeModelWithResponseStream parses each payload once and delivers malfor expect(await collect(response.body)).toEqual(events); expect(parse.mock.calls.length - before).toBe(4); - parse.mockRestore(); const [span] = spans.getFinishedSpans(); expect(span).toMatchObject({ [Symbol.for("telemetry.dev.outputChunkHistogram")]: { count: 2 }, @@ -996,7 +1004,7 @@ test("ApplyGuardrail captures action metadata", async () => { expect(span.attributes["td.metadata.guardrail_action_reason"]).toBe("blocked"); }); -test("errors, fail-open, idempotency, and prototype restore behave", async () => { +test("send errors end one error span, double wrapping traces once, and shutdown passes through", async () => { const spans = setup(); const error = Object.assign(new Error("throttled"), { @@ -1017,11 +1025,10 @@ test("errors, fail-open, idempotency, and prototype restore behave", async () => await teardown(); const disabled = wrapBedrock(new FakeClient([{ output: { message: { content: [] } } }])); - await disabled.send(new ConverseCommand({ modelId: "m", messages: [] })); + await expect(disabled.send(new ConverseCommand({ modelId: "m", messages: [] }))).resolves.toEqual( + { output: { message: { content: [] } } }, + ); expect(disabled.calls).toHaveLength(1); - - instrumentBedrock(); - uninstrumentBedrock(); }); test("callback-style send creates a span while returning undefined", () => { @@ -1066,67 +1073,66 @@ test("callback-style send creates a span while returning undefined", () => { expect(jsonAttr(span, "gen_ai.output.messages")[0].parts[0].content).toBe("ok"); }); -test("global instrumentation is idempotent", async () => { +test("global instrumentation patches the real client once and restores its inherited send", async () => { const spans = setup(); - - const originalDescriptor = Object.getOwnPropertyDescriptor( - BedrockRuntimeClient.prototype, - "send", - ); - - const fakeSend = async () => ({ - output: { message: { content: [] } }, - $metadata: { requestId: "global-1" }, + const proto = BedrockRuntimeClient.prototype; + const command = new ConverseCommand({ modelId: "m", messages: [] }); + + const client = new BedrockRuntimeClient({ + region: "us-east-1", + credentials: { accessKeyId: "test", secretAccessKey: "test" }, + maxAttempts: 1, + requestHandler: jsonRequestHandler("global-1", { + output: { message: { role: "assistant", content: [{ text: "ok" }] } }, + stopReason: "end_turn", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: { latencyMs: 1 }, + }), }); - Object.defineProperty(BedrockRuntimeClient.prototype, "send", { - configurable: true, - writable: true, - value: fakeSend, - }); + // AWS clients inherit send from the Smithy base class. + expect(Object.hasOwn(proto, "send")).toBe(false); try { instrumentBedrock(); instrumentBedrock(); - const client = new BedrockRuntimeClient({ - region: "us-east-1", - credentials: { accessKeyId: "test", secretAccessKey: "test" }, - }); - - await client.send(new ConverseCommand({ modelId: "m", messages: [] })); + await client.send(command); expect(spans.getFinishedSpans()).toHaveLength(1); expect(spans.getFinishedSpans()[0]!.attributes["gen_ai.response.id"]).toBe("global-1"); - const telemetryWrapper = Object.getOwnPropertyDescriptor( - BedrockRuntimeClient.prototype, - "send", - )?.value; - + const telemetryWrapper = Object.getOwnPropertyDescriptor(proto, "send")?.value; const laterWrapper = async () => ({ output: { message: { content: [] } } }); - Object.defineProperty(BedrockRuntimeClient.prototype, "send", { + Object.defineProperty(proto, "send", { configurable: true, writable: true, value: laterWrapper, }); uninstrumentBedrock(); - expect(Object.getOwnPropertyDescriptor(BedrockRuntimeClient.prototype, "send")?.value).toBe( - laterWrapper, - ); - Object.defineProperty(BedrockRuntimeClient.prototype, "send", { + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(laterWrapper); + Object.defineProperty(proto, "send", { configurable: true, writable: true, value: telemetryWrapper, }); uninstrumentBedrock(); - expect(Object.getOwnPropertyDescriptor(BedrockRuntimeClient.prototype, "send")?.value).toBe( - fakeSend, - ); + expect(Object.hasOwn(proto, "send")).toBe(false); + + await client.send(command); + expect(spans.getFinishedSpans()).toHaveLength(1); + + const priorOwner = async () => ({ output: { message: { content: [] } } }); + Object.defineProperty(proto, "send", { + configurable: true, + writable: true, + value: priorOwner, + }); + instrumentBedrock(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).not.toBe(priorOwner); + uninstrumentBedrock(); + expect(Object.getOwnPropertyDescriptor(proto, "send")?.value).toBe(priorOwner); } finally { uninstrumentBedrock(); - - if (originalDescriptor) { - Object.defineProperty(BedrockRuntimeClient.prototype, "send", originalDescriptor); - } + Reflect.deleteProperty(proto, "send"); } }); diff --git a/packages/bedrock/tests/helpers.ts b/packages/bedrock/tests/helpers.ts index 6ebc2d0..5d8ed86 100644 --- a/packages/bedrock/tests/helpers.ts +++ b/packages/bedrock/tests/helpers.ts @@ -4,6 +4,7 @@ import { type ReadableSpan, type Sampler, } from "@opentelemetry/sdk-trace-base"; +import { HttpResponse } from "@smithy/protocol-http"; import { init, shutdown, type MaskFn } from "@telemetry-dev/sdk"; @@ -62,6 +63,19 @@ export class FakeClient { } } +/** A Smithy request handler that answers every request with one JSON body, so a real client runs its own send. */ +export function jsonRequestHandler(requestId: string, body: unknown) { + return { + handle: async () => ({ + response: new HttpResponse({ + statusCode: 200, + headers: { "content-type": "application/json", "x-amzn-requestid": requestId }, + body: new TextEncoder().encode(JSON.stringify(body)), + }), + }), + }; +} + export async function collect(iterable: AsyncIterable, limit?: number): Promise { const out: T[] = []; diff --git a/packages/cursor/tests/cursor.test.ts b/packages/cursor/tests/cursor.test.ts index bc243f6..080df60 100644 --- a/packages/cursor/tests/cursor.test.ts +++ b/packages/cursor/tests/cursor.test.ts @@ -162,18 +162,6 @@ describe("turn lifecycle", () => { expect(agent!.attributes["error.type"]).toBe("AgentError"); }); - test("a second beforeSubmitPrompt ends the stale turn as incomplete", async () => { - const { spans, telemetry } = makeHarness(); - telemetry.handle(event("beforeSubmitPrompt", { prompt: "first" })); - telemetry.handle(event("beforeSubmitPrompt", { prompt: "second", generation_id: "gen-2" })); - telemetry.handle(event("stop", { status: "completed", generation_id: "gen-2" })); - await flush(); - - const agents = agentSpans(spans, "cursor"); - expect(agents).toHaveLength(2); - expect(agents[0]!.attributes["gen_ai.response.finish_reasons"]).toEqual(["incomplete"]); - }); - test("generation_id drift within a turn does not split it", async () => { const { spans, telemetry } = makeHarness(); telemetry.handle(event("beforeSubmitPrompt", { prompt: "one turn" })); @@ -836,21 +824,6 @@ describe("titles", () => { } }); - test("a missing title falls back to the prompt as the span name", async () => { - const chats = mkdtempSync(join(tmpdir(), "cursor-chats-")); - const { spans, telemetry } = makeHarness({ chatsDir: chats }); - - try { - telemetry.handle(event("beforeSubmitPrompt", { prompt: "x" })); - telemetry.handle(event("stop", { status: "completed" })); - await flush(); - - expect(finished(spans).map((s) => s.name)).toContain("x"); - } finally { - rmSync(chats, { recursive: true, force: true }); - } - }); - test.each([ { captureInput: true, @@ -999,24 +972,34 @@ describe("settle", () => { }); test("malformed events never throw", () => { - const { telemetry } = makeHarness(); + const onError = vi.fn(); + const { telemetry } = makeHarness({ onError }); + const failure = new Error("unreadable hook payload"); const eventRecord = { hook_event_name: "unknownEvent", nested: { arbitrary: new Date() }, }; + const unreadable = { + get hook_event_name(): string { + throw failure; + }, + }; + expectTypeOf().parameter(0).toEqualTypeOf>(); telemetry.handle({}); telemetry.handle({ hook_event_name: "postToolUse" }); telemetry.handle({ hook_event_name: "unknownEvent", conversation_id: "conv-1" }); telemetry.handle(eventRecord); + expect(() => telemetry.handle(unreadable)).not.toThrow(); + expect(onError).toHaveBeenCalledWith(failure); expect(telemetry.open()).toBe(false); }); }); describe("fileConfig", () => { - test("a missing file yields defaults; an unreadable or non-object file disables telemetry", () => { + test("a missing file yields defaults and a non-string value is ignored", () => { const home = mkdtempSync(join(tmpdir(), "cursor-cfg-")); const restoreHome = setHome(home); const file = join(home, ".cursor", "telemetry-dev.json"); @@ -1025,11 +1008,6 @@ describe("fileConfig", () => { expect(fileConfig()).toEqual({}); mkdirSync(join(home, ".cursor")); - for (const content of ["{not json", '["captureInput", false]', '"text"', "null"]) { - writeFileSync(file, content); - expect(fileConfig(), content).toEqual({ enabled: false }); - } - // A non-string value is ignored rather than coerced, which could throw. vi.stubEnv("OTEL_SERVICE_NAME", undefined); vi.stubEnv("TELEMETRY_DEV_ENVIRONMENT", undefined); @@ -1050,6 +1028,8 @@ describe("fileConfig", () => { { content: '{"enabled":false}', exported: false }, { content: "{not json", exported: false }, { content: '["captureOutput", false]', exported: false }, + { content: '"text"', exported: false }, + { content: "null", exported: false }, // Creating symlinks on Windows needs extra privileges. ...(process.platform === "win32" ? [] : [{ content: "dangling symlink", exported: false }]), ])( diff --git a/packages/eve/tests/eve.test.ts b/packages/eve/tests/eve.test.ts index b97fc5e..d91ed42 100644 --- a/packages/eve/tests/eve.test.ts +++ b/packages/eve/tests/eve.test.ts @@ -309,6 +309,7 @@ function onlyLog(exporters: Exporters) { } afterEach(async () => { + vi.useRealTimers(); vi.unstubAllGlobals(); vi.restoreAllMocks(); @@ -406,11 +407,20 @@ test("instrumentation puts every ai.eve.turn of a session in the session trace", } }); -test("instrumentation captureInput:false maps to eve recordInputs:false", () => { - const definition = telemetryDevInstrumentation(telemetryOptions({ captureInput: false })); +test.each([ + { capture: { captureInput: false }, recordInputs: false, recordOutputs: true }, + { capture: { captureOutput: false }, recordInputs: true, recordOutputs: false }, +])( + "instrumentation maps $capture to eve recordInputs and recordOutputs", + ({ capture, recordInputs, recordOutputs }) => { + const definition = telemetryDevInstrumentation(telemetryOptions(capture)); - expect(definition.recordInputs).toBe(false); -}); + expect([definition.recordInputs, definition.recordOutputs]).toEqual([ + recordInputs, + recordOutputs, + ]); + }, +); test("provider integration forwards content capture flags", () => { const integration = telemetryDevOtelIntegration({ @@ -952,17 +962,31 @@ test.each([ test("wrapped eve client puts every turn of one session in the session trace", async () => { const exporters = makeExporters(); - stubFetchSequence([ + + const responses = [ jsonResponse({ sessionId: "session-1" }), ndjsonResponse(successfulTurnEvents()), jsonResponse({ sessionId: "session-1" }), ndjsonResponse(successfulTurnEvents()), - ]); + ]; + + // Each request takes a minute on a frozen clock: a span that started when its + // POST resolved would start a minute after its turn was sent. + vi.useFakeTimers({ now: Date.parse("2026-01-02T03:04:05.000Z"), toFake: ["Date"] }); + vi.stubGlobal( + "fetch", + vi.fn(async () => { + vi.setSystemTime(Date.now() + 60_000); + + return responses.shift()!; + }), + ); const client = wrapClient(exporters); - const before = Date.now(); + const sentAt = [Date.now()]; const { response, session } = await client.sessions.create({ message: "first" }); await response.result(); + sentAt.push(Date.now()); await (await session.send("second")).result(); await flush(); @@ -976,10 +1000,9 @@ test("wrapped eve client puts every turn of one session in the session trace", a expect(span.spanContext().traceId).toBe(expected.traceId); expect(span.parentSpanContext?.spanId).toBe(expected.spanId); expect(span.attributes["gen_ai.conversation.id"]).toBe("session-1"); - // The span keeps the pre-send start time even though it is created after the response. - expect(ms(span.startTime)).toBeGreaterThanOrEqual(before); - expect(ms(span.startTime)).toBeLessThanOrEqual(ms(span.endTime)); } + + expect(spans.map((span) => ms(span.startTime))).toEqual(sentAt); }); test("wrapped eve client marks streams without terminal events as errored spans", async () => { diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index 1f5d07b..d1d291a 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -1820,6 +1820,7 @@ test.each([ const span = await exportedSpan(spans); expect(span.attributes["google_genai.response.block_reason"]).toBe("SAFETY"); + expect(span.status.code).toBe(SPAN_STATUS_UNSET); if (capture === "on") { expect(span.attributes["google_genai.response.block_reason_message"]).toBe( @@ -2516,32 +2517,6 @@ test("non-Error stream setup failures end spans and preserve the thrown values", ).toEqual(["stream setup rejected", "[object Object]"]); }); -test("blocked prompt records block attrs without output", async () => { - const spans = setupSpans(); - - const fake = createFakeFetch( - jsonResponse({ - promptFeedback: { - blockReason: "SAFETY", - blockReasonMessage: "blocked for safety", - safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", probability: "HIGH" }], - }, - }), - ); - - vi.stubGlobal("fetch", fake.fetch); - const client = clientWith(); - await client.models.generateContent({ model: "gemini-2.5-flash", contents: "blocked" }); - const span = await exportedSpan(spans); - expect(span.status.code).toBe(SPAN_STATUS_UNSET); - expect(span.attributes["google_genai.response.block_reason"]).toBe("SAFETY"); - expect(span.attributes["google_genai.response.block_reason_message"]).toBe("blocked for safety"); - expect(jsonAttr(span, "google_genai.response.prompt_safety_ratings")).toEqual([ - { category: "HARM_CATEGORY_DANGEROUS_CONTENT", probability: "HIGH" }, - ]); - expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); -}); - test("embedContent maps embedding span fields without output", async () => { const spans = setupSpans(); @@ -2704,7 +2679,7 @@ test("wrapped calls fail open when telemetry is not initialized", async () => { expect(response.text).toBe("ok"); }); -test("usage modality details map text, image, and audio without video attributes", async () => { +test("usage modality details map text, image, and audio tokens and ignore other modalities", async () => { const spans = setupSpans(); const client = wrapGoogleGenAI({ @@ -2737,9 +2712,6 @@ test("usage modality details map text, image, and audio without video attributes expect(span.attributes["gen_ai.usage.text.cache_read.input_tokens"]).toBe(5); expect(span.attributes["gen_ai.usage.image.cache_read.input_tokens"]).toBe(6); expect(span.attributes["gen_ai.usage.audio.cache_read.input_tokens"]).toBe(7); - expect( - Object.keys(span.attributes).some((key) => key.includes("video") && key.includes("usage")), - ).toBe(false); }); test.each(["generateImages", "editImage", "upscaleImage"] as const)( @@ -2775,7 +2747,6 @@ test.each(["generateImages", "editImage", "upscaleImage"] as const)( expect(messagesAttr(span, "gen_ai.output.messages")).toEqual([ { type: "image", uri: "gs://bucket/output.png" }, ]); - expect(span.attributes["google_genai.response.image_uris"]).toBeUndefined(); expect(JSON.stringify(span.attributes)).not.toContain(binary); }, ); @@ -2857,7 +2828,11 @@ test("generateVideos traces submit and tracked polling idempotently", async () = expect(messagesAttr(exported[1]!, "gen_ai.output.messages")).toEqual([ { type: "video", uri: "gs://bucket/video.mp4" }, ]); - expect(exported[1]!.attributes["google_genai.response.video_uris"]).toBeUndefined(); + expect( + Object.keys(exported[1]!.attributes).filter((key) => + String(exported[1]!.attributes[key]).includes("gs://bucket/video.mp4"), + ), + ).toEqual(["gen_ai.output.messages"]); expect(exported.every((span) => span.attributes["gen_ai.output.type"] === "video")).toBe(true); }); @@ -3094,30 +3069,6 @@ test.each(["get", "getVideosOperation"] as const)( }, ); -test("media methods preserve rejected and synchronously thrown failures", async () => { - const spans = setupSpans(); - const rejected = new Error("image rejected"); - const thrown = new Error("video threw"); - - const client = wrapGoogleGenAI({ - models: { - generateImages(_params: unknown) { - return Promise.reject(rejected); - }, - generateVideos(_params: unknown) { - throw thrown; - }, - }, - }); - - await expect(client.models.generateImages({ model: "imagen", prompt: "fail" })).rejects.toBe( - rejected, - ); - expect(() => client.models.generateVideos({ model: "veo", prompt: "fail" })).toThrow(thrown); - const exported = await finishedSpans(spans, 2); - expect(exported.every((span) => span.status.code === SPAN_STATUS_ERROR)).toBe(true); -}); - test("capture traversal is bounded, cycle-aware, binary-safe, and preserves provider outcomes", async () => { const spans = setupSpans(); const shared = { text: "shared" }; @@ -3157,6 +3108,8 @@ test("capture traversal is bounded, cycle-aware, binary-safe, and preserves prov const span = await exportedSpan(spans); expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); expect(String(span.attributes["gen_ai.input.messages"])).not.toContain("SECRET_BINARY"); + expect(String(span.attributes["gen_ai.input.messages"])).toContain('"shared"'); + expect(String(span.attributes["gen_ai.output.messages"])).toContain('"shared"'); expect(String(span.attributes["gen_ai.input.messages"]).length).toBeLessThan(65_536); expect(String(span.attributes["gen_ai.output.messages"]).length).toBeLessThan(65_536); }); diff --git a/packages/mcp/tests/mcp.test.ts b/packages/mcp/tests/mcp.test.ts index 40757ed..bd0e590 100644 --- a/packages/mcp/tests/mcp.test.ts +++ b/packages/mcp/tests/mcp.test.ts @@ -251,7 +251,12 @@ test("completes responses before a per-request HTTP transport closes", async () await handler.close(); }); -test("applies asymmetric JSON-RPC error status rules", async () => { +test.each([ + { code: -32602, serverFault: false }, + { code: -32021, serverFault: false }, + { code: -32022, serverFault: false }, + { code: -32603, serverFault: true }, +])("applies asymmetric JSON-RPC error status rules to $code", async ({ code, serverFault }) => { const spans = setup(); const [client, server] = InMemoryTransport.createLinkedPair().map((transport) => @@ -260,60 +265,24 @@ test("applies asymmetric JSON-RPC error status rules", async () => { client.onmessage = () => undefined; server.onmessage = () => { - void server.send({ - jsonrpc: "2.0", - id: 7, - error: { code: -32602, message: "Invalid params" }, - }); + void server.send({ jsonrpc: "2.0", id: 7, error: { code, message: "JSON-RPC error" } }); }; await client.start(); await server.start(); - await client.send({ jsonrpc: "2.0", id: 7, method: "prompts/get", params: { name: "review" } }); + await client.send({ jsonrpc: "2.0", id: 7, method: "ping" }); const finished = await exported(spans, 2); const clientSpan = finished.find((span) => span.kind === SpanKind.CLIENT)!; const serverSpan = finished.find((span) => span.kind === SpanKind.SERVER)!; - expect(clientSpan.attributes["rpc.response.status_code"]).toBe("-32602"); - expect(clientSpan.attributes["error.type"]).toBe("-32602"); + expect(clientSpan.attributes["rpc.response.status_code"]).toBe(String(code)); + expect(serverSpan.attributes["rpc.response.status_code"]).toBe(String(code)); + expect(clientSpan.attributes["error.type"]).toBe(String(code)); expect(clientSpan.status.code).toBe(SpanStatusCode.ERROR); - expect(serverSpan.attributes["rpc.response.status_code"]).toBe("-32602"); - expect(serverSpan.attributes["error.type"]).toBeUndefined(); - expect(serverSpan.status.code).toBe(SpanStatusCode.UNSET); + expect(serverSpan.attributes["error.type"]).toBe(serverFault ? String(code) : undefined); + expect(serverSpan.status.code).toBe(serverFault ? SpanStatusCode.ERROR : SpanStatusCode.UNSET); }); -test.each([-32021, -32022])( - "treats MCP v2 caller error %i as a client failure only", - async (code) => { - const spans = setup(); - - const [client, server] = InMemoryTransport.createLinkedPair().map((transport) => - instrumentMcpTransport(transport), - ); - - client.onmessage = () => undefined; - server.onmessage = () => { - void server.send({ - jsonrpc: "2.0", - id: 8, - error: { code, message: "Client compatibility error" }, - }); - }; - - await client.start(); - await server.start(); - await client.send({ jsonrpc: "2.0", id: 8, method: "ping" }); - - const finished = await exported(spans, 2); - const clientSpan = finished.find((span) => span.kind === SpanKind.CLIENT)!; - const serverSpan = finished.find((span) => span.kind === SpanKind.SERVER)!; - expect(clientSpan.attributes["error.type"]).toBe(String(code)); - expect(clientSpan.status.code).toBe(SpanStatusCode.ERROR); - expect(serverSpan.attributes["error.type"]).toBeUndefined(); - expect(serverSpan.status.code).toBe(SpanStatusCode.UNSET); - }, -); - test("marks CallToolResult errors and closes pending requests", async () => { const spans = setup(); @@ -348,12 +317,14 @@ test("marks CallToolResult errors and closes pending requests", async () => { }; const transport = instrumentMcpTransport(rawTransport); - transport.onclose = () => undefined; + const hostClose = vi.fn(); + transport.onclose = hostClose; await transport.send({ jsonrpc: "2.0", id: 10, method: "ping" } as never); transport.onclose?.(); const [closed] = await exported(closeSpans, 1); expect(closed?.attributes["error.type"]).toBe("connection_error"); expect(closed?.status.code).toBe(SpanStatusCode.ERROR); + expect(hostClose).toHaveBeenCalledOnce(); }); test("ends an outgoing request span when send fails", async () => { diff --git a/packages/omp/tests/omp.test.ts b/packages/omp/tests/omp.test.ts index c2ecc94..efddf8d 100644 --- a/packages/omp/tests/omp.test.ts +++ b/packages/omp/tests/omp.test.ts @@ -393,6 +393,7 @@ test("tool execution produces an execute_tool span with arguments and result", a "gen_ai.operation.name": "execute_tool", "gen_ai.tool.call.arguments": JSON.stringify({ command: "ls" }), "gen_ai.tool.call.id": "call-1", + "gen_ai.tool.call.result": JSON.stringify({ content: [{ type: "text", text: "README.md" }] }), "gen_ai.tool.name": "bash", }); expect(span.status.code).not.toBe(SpanStatusCode.ERROR); @@ -542,14 +543,17 @@ test("malformed events never throw and report through onError", async () => { expect(harness.exporters.spans.getFinishedSpans().length).toBeGreaterThan(0); }); -test("events for a different session carry that session's conversation id", async () => { +test("each event reads the current session id", async () => { const harness = makeHarness(); - const other = makeContext("session-2"); - await harness.emit({ type: "session_start" }, other); + await harness.emit({ type: "session_start" }, makeContext("session-2")); + await harness.emit({ type: "session_shutdown" }); await flush(); - const record = harness.exporters.logs.getFinishedLogRecords()[0]; - expect(record?.attributes["gen_ai.conversation.id"]).toBe("session-2"); + const records = harness.exporters.logs.getFinishedLogRecords(); + expect(records.map((record) => record.attributes["gen_ai.conversation.id"])).toEqual([ + "session-2", + "session-1", + ]); }); test("tool spans nest under the chat span that issued the tool call", async () => { diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 81fdd87..4467aa3 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -35,11 +35,15 @@ test("runs the compatibility suite against the selected OpenAI major", () => { expect(VERSION).toBe(process.env.OPENAI_SDK_VERSION === "6" ? "6.45.0" : "7.18.0"); }); -test("package exports resolve to built files inside the package", () => { - expect(packageJson.exports["."].default).toBe("./dist/index.mjs"); - expect(packageJson.publishConfig.exports["."].types).toBe("./dist/index.d.mts"); - expect(packageJson.publishConfig.exports["."].import).toBe("./dist/index.mjs"); - expect(packageJson.publishConfig.exports["."].default).toBe("./dist/index.mjs"); +test("published exports resolve types first and point only at shipped dist files", () => { + const published = packageJson.publishConfig.exports["."]; + + expect(packageJson.files).toContain("dist"); + expect(Object.keys(published)).toEqual(["types", "import", "default"]); + expect(published.types).toMatch(/\.d\.mts$/); + + for (const target of [packageJson.exports["."].default, ...Object.values(published)]) + expect(target).toMatch(/^\.\/dist\/[^/]+$/); }); test.each(["", "8"])("rejects unsupported OpenAI SDK selector %j", async (version) => { @@ -601,18 +605,18 @@ test("chat completions preserve tool-call messages in output", async () => { expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["tool_calls"]); }); -test("chat completions capture every returned choice", async () => { +test("chat completions capture every returned choice with its finish reason", async () => { const spans = setupSpans(); const fake = createFakeFetch( jsonResponse({ - id: "chatcmpl_two", + id: "chatcmpl_multi_finish", object: "chat.completion", created: 1, model: "gpt-4o-2024-11-20", choices: [ { index: 0, message: { role: "assistant", content: "First" }, finish_reason: "stop" }, - { index: 1, message: { role: "assistant", content: "Second" }, finish_reason: "stop" }, + { index: 1, message: { role: "assistant", content: "Second" }, finish_reason: "length" }, ], usage: { prompt_tokens: 10, completion_tokens: 8, total_tokens: 18 }, }), @@ -620,69 +624,16 @@ test("chat completions capture every returned choice", async () => { await clientWith(fake.fetch).chat.completions.create({ model: "gpt-4o", - messages: [{ role: "user", content: "Give two options" }], + messages: [{ role: "user", content: "Two answers" }], n: 2, }); const span = await exportedSpan(spans); - expect(jsonAttr(span, "gen_ai.output.messages")).toHaveLength(2); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ { role: "assistant", content: "First" }, { role: "assistant", content: "Second" }, ]); -}); - -test("chat completions record finish reasons for single and multi-choice responses", async () => { - const spans = setupSpans(); - - const fake = createFakeFetch( - jsonResponse({ - id: "chatcmpl_single_finish", - object: "chat.completion", - created: 1, - model: "gpt-4o-2024-11-20", - choices: [ - { index: 0, message: { role: "assistant", content: "Single" }, finish_reason: "stop" }, - ], - usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 }, - }), - jsonResponse({ - id: "chatcmpl_multi_finish", - object: "chat.completion", - created: 1, - model: "gpt-4o-2024-11-20", - choices: [ - { index: 0, message: { role: "assistant", content: "First" }, finish_reason: "stop" }, - { index: 1, message: { role: "assistant", content: "Second" }, finish_reason: "length" }, - ], - usage: { prompt_tokens: 10, completion_tokens: 8, total_tokens: 18 }, - }), - ); - - const client = clientWith(fake.fetch); - - await client.chat.completions.create({ - model: "gpt-4o", - messages: [{ role: "user", content: "One answer" }], - }); - await client.chat.completions.create({ - model: "gpt-4o", - messages: [{ role: "user", content: "Two answers" }], - n: 2, - }); - - const finished = await finishedSpans(spans, 2); - - const single = finished.find( - (span) => span.attributes["gen_ai.response.id"] === "chatcmpl_single_finish", - ); - - const multi = finished.find( - (span) => span.attributes["gen_ai.response.id"] === "chatcmpl_multi_finish", - ); - - expect(single?.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); - expect(multi?.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); + expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); }); test("chunk timing preserves delivery", async () => { @@ -907,7 +858,7 @@ test("chat streams record finish reasons for every choice", async () => { expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); }); -test("chat streams leave the request unchanged and capture no usage by default", async () => { +test("chat streams send the request unchanged by default", async () => { const spans = setupSpans(); const fake = createFakeFetch( @@ -946,7 +897,6 @@ test("chat streams leave the request unchanged and capture no usage by default", expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ { role: "assistant", content: "Hello" }, ]); - expect(span.attributes["gen_ai.usage.total_tokens"]).toBeUndefined(); }); test("chat streams bound retained content without changing events or usage", async () => { @@ -1090,42 +1040,6 @@ test.each(["chat", "responses", "transcription"] as const)( }, ); -test("chat streams retain output when the core span lacks chunk recording", async () => { - const spans = setupSpans(); - const startSpan = telemetrySdk.startSpan; - vi.spyOn(telemetrySdk, "startSpan").mockImplementation((...args) => { - const legacySpan = { ...startSpan(...args) }; - Reflect.deleteProperty(legacySpan, "recordOutputChunk"); - - return legacySpan; - }); - - const events = [ - { choices: [{ index: 0, delta: { content: "delivered" }, finish_reason: "stop" }] }, - ]; - - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create: async (_params: unknown) => source } }, - responses: { create() {} }, - embeddings: { create() {} }, - }); - - const stream = await client.chat.completions.create({ - model: "gpt-4o", - messages: [], - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - expect(jsonAttr(await exportedSpan(spans), "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: "delivered" }, - ]); -}); - test("chat streams bound retained tool arguments", async () => { const spans = setupSpans(); const states = observeChatChoiceStates(); @@ -2517,7 +2431,6 @@ test("chat streams reclaim finish-reason items when replacing a choice", async ( test("chat streams bound retained choice states", async () => { const spans = setupSpans(); - const states = observeChatChoiceStates(); const event = { choices: Array.from({ length: 500 }, (_, index) => ({ @@ -2544,7 +2457,6 @@ test("chat streams bound retained choice states", async () => { const iterator = stream[Symbol.asyncIterator](); expect((await iterator.next()).value).toBe(event); - expect(states().size).toBe(499); expect(await iterator.next()).toEqual({ done: true, value: undefined }); const span = await exportedSpan(spans); const output = jsonAttr(span, "gen_ai.output.messages"); @@ -2785,83 +2697,6 @@ test("chat streams inject include_usage when opted in and hide only the syntheti ).toBe("number"); }); -test.each([true, false])( - "nonterminal chat stream close marks capture incomplete with output capture=%s", - async (captureOutput) => { - const spans = setupSpans(undefined, { captureOutput }); - - const event = { - choices: [{ index: 0, delta: { role: "assistant", content: "partial" } }], - }; - - const source = new Stream(async function* () { - yield event; - yield { choices: [{ index: 0, delta: { content: "unread" }, finish_reason: "stop" }] }; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create: async (_params: unknown) => source } }, - responses: { create() {} }, - embeddings: { create() {} }, - }); - - const stream = await client.chat.completions.create({ - model: "gpt-4o", - messages: [], - stream: true, - }); - - for await (const chunk of stream) { - expect(chunk).toBe(event); - break; - } - - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.finish_reasons"]).toBeUndefined(); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); - - if (captureOutput) - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: "partial" }, - ]); - else expect(span.attributes["gen_ai.output.messages"]).toBeUndefined(); - }, -); - -test("nonterminal chat stream errors omit finish reasons and mark capture incomplete", async () => { - const spans = setupSpans(); - const failure = new Error("stream failed before terminal chunk"); - - const event = { - choices: [{ index: 0, delta: { role: "assistant", content: "partial" } }], - }; - - const source = new Stream(async function* () { - yield event; - throw failure; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create: async (_params: unknown) => source } }, - responses: { create() {} }, - embeddings: { create() {} }, - }); - - const stream = await client.chat.completions.create({ - model: "gpt-4o", - messages: [], - stream: true, - }); - - await expect(collectStream(stream)).rejects.toBe(failure); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.finish_reasons"]).toBeUndefined(); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ - { role: "assistant", content: "partial" }, - ]); -}); - test.each([false, true])( "a post-finish content filter chunk keeps a complete chat stream complete (masked: %s)", async (masked) => { @@ -3325,55 +3160,6 @@ test.each([ ]); }); -test("responses streams end from response.completed terminal event", async () => { - const spans = setupSpans(); - - const completed = { - id: "resp_stream", - object: "response", - status: "completed", - model: "gpt-4.1-2025-04-14", - output: [ - { - id: "msg_stream", - type: "message", - role: "assistant", - content: [{ type: "output_text", text: "Done" }], - }, - ], - usage: { input_tokens: 8, output_tokens: 2, total_tokens: 10 }, - }; - - const fake = createFakeFetch( - sseResponse([ - { type: "response.created", response: { id: "resp_stream", status: "in_progress" } }, - { type: "response.completed", response: completed }, - ]), - ); - - const stream = await clientWith(fake.fetch).responses.create({ - model: "gpt-4.1", - input: "Finish", - stream: true, - }); - - const events = await collectStream(stream); - - expect(events).toHaveLength(2); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.response.id"]).toBe("resp_stream"); - expect(span.attributes["gen_ai.response.model"]).toBe("gpt-4.1-2025-04-14"); - expect(jsonAttr(span, "gen_ai.output.messages")).toEqual(completed.output); - expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); - expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(10); - expect( - Number(span.attributes["gen_ai.response.time_to_first_chunk"]) === - span.attributes["gen_ai.response.time_to_first_chunk"] - ? "number" - : "other", - ).toBe("number"); -}); - test("responses streams retain the last fitting output when the terminal snapshot truncates", async () => { const spans = setupSpans(); @@ -3893,6 +3679,8 @@ test("responses streams end from terminal event before reading the stream sentin const span = await exportedSpan(spans); expect(span.attributes["gen_ai.response.id"]).toBe("resp_stream_terminal"); + expect(span.attributes["gen_ai.response.model"]).toBe("gpt-4.1-2025-04-14"); + expect(span.attributes["gen_ai.response.time_to_first_chunk"]).toEqual(expect.any(Number)); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual(completed.output); expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(10); @@ -4087,21 +3875,12 @@ test("global instrumentation records responses and embeddings with wrapOpenAI pa expect(responseSpan?.attributes["gen_ai.provider.name"]).toBe("openai"); expect(responseSpan?.attributes["gen_ai.operation.name"]).toBe("chat"); expect(responseSpan?.attributes["gen_ai.system_instructions"]).toBe("You must never tell jokes"); - expect(responseSpan?.attributes["gen_ai.input.messages"]).toBe("Tell me a joke"); expect(jsonAttr(responseSpan!, "gen_ai.output.messages")).toEqual(output); - expect(responseSpan?.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop"]); - expect(responseSpan?.attributes["gen_ai.usage.cache_read.input_tokens"]).toBe(4); - expect(responseSpan?.attributes["gen_ai.usage.reasoning.output_tokens"]).toBe(1); - expect(responseSpan?.attributes["gen_ai.request.max_tokens"]).toBe(50); const embeddingSpan = finished.find((span) => span.name === "embeddings text-embedding-3-small"); expect(embeddingSpan?.attributes["gen_ai.provider.name"]).toBe("openai"); expect(embeddingSpan?.attributes["gen_ai.operation.name"]).toBe("embeddings"); - expect(embeddingSpan?.attributes["gen_ai.request.model"]).toBe("text-embedding-3-small"); - expect(embeddingSpan?.attributes["gen_ai.response.model"]).toBe("text-embedding-3-small"); - expect(embeddingSpan?.attributes["gen_ai.input.messages"]).toBe("OpenTelemetry traces"); expect(embeddingSpan?.attributes["gen_ai.usage.input_tokens"]).toBe(6); - expect(embeddingSpan?.attributes["gen_ai.usage.total_tokens"]).toBe(6); expect(embeddingSpan?.attributes["gen_ai.output.messages"]).toBeUndefined(); }); @@ -4675,7 +4454,10 @@ test("images generate maps modality usage without capturing image bytes", async expect(span.attributes["gen_ai.usage.image.cache_read.input_tokens"]).toBe(1); expect(span.attributes["gen_ai.usage.audio.cache_read.input_tokens"]).toBe(0); expect(span.attributes["gen_ai.usage.image.output_tokens"]).toBe(12); - expect(String(span.attributes["gen_ai.output.messages"])).not.toContain("secret-image-bytes"); + expect(jsonAttr(span, "gen_ai.output.messages")).toEqual({ + count: 1, + revised_prompts: ["A revised prompt"], + }); }); test.each([true, false])( @@ -4783,32 +4565,6 @@ test("streaming transcription bounds captured output while yielding every event" expect(span.attributes["gen_ai.usage.text.output_tokens"]).toBe(5); }); -test("streaming transcription retains no more than the transcript ceiling", async () => { - const spans = setupSpans(); - const events = [{ type: "transcript.text.delta", delta: "x".repeat(70_000) }]; - - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); - - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("x".repeat(65_534)); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBe(true); -}); - test("streaming transcription omits a bounded prefix that a mask cannot evaluate safely", async () => { const spans = setupSpans(undefined, { mask: (value) => (String(value).includes("SECRET") ? "[REDACTED]" : value), @@ -4950,94 +4706,73 @@ test.each([ }, ); -test("a complete masked transcription stream keeps its output and is not flagged", async () => { - const spans = setupSpans(undefined, { mask: (value) => value }); - - const events = [ - { type: "transcript.text.delta", delta: "partial" }, - { type: "transcript.text.done", text: "complete transcript" }, - ]; - - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); - - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); - - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); - - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("complete transcript"); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBeUndefined(); -}); - -test("streaming transcription replaces truncated deltas with complete terminal text", async () => { - const spans = setupSpans(); +test.each([false, true])( + "streaming transcription replaces truncated deltas with complete terminal text (masked: %s)", + async (masked) => { + const spans = setupSpans(undefined, masked ? { mask: (value) => value } : {}); - const events = [ - { type: "transcript.text.delta", delta: "x".repeat(70_000) }, - { type: "transcript.text.done", text: "complete transcript" }, - ]; + const events = [ + { type: "transcript.text.delta", delta: "x".repeat(70_000) }, + { type: "transcript.text.done", text: "complete transcript" }, + ]; - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); + const source = new Stream(async function* () { + yield* events; + }, new AbortController()); - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); + const client = wrapOpenAI({ + chat: { completions: { create() {} } }, + responses: { create() {} }, + embeddings: { create() {} }, + audio: { transcriptions: { create: async (_params: unknown) => source } }, + }); - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); + const stream = await client.audio!.transcriptions!.create({ + model: "gpt-4o-transcribe", + stream: true, + }); - expect(await collectStream(stream)).toEqual(events); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("complete transcript"); - expect(span.attributes["telemetry.dev.capture.truncated"]).toBeUndefined(); -}); + expect(await collectStream(stream)).toEqual(events); + const span = await exportedSpan(spans); + expect(span.attributes["gen_ai.output.messages"]).toBe("complete transcript"); + expect(span.attributes["telemetry.dev.capture.truncated"]).toBeUndefined(); + }, +); -test("streaming transcription captures many small deltas in linear work", async () => { - const spans = setupSpans(); +test.each([ + ["one oversized delta", 1, 70_000], + ["70,000 one-character deltas", 70_000, 1], +])( + "streaming transcription retains at most the transcript ceiling from %s", + async (_case, count, size) => { + const spans = setupSpans(); - const events = Array.from({ length: 70_000 }, () => ({ - type: "transcript.text.delta", - delta: "x", - })); + const events = Array.from({ length: count }, () => ({ + type: "transcript.text.delta", + delta: "x".repeat(size), + })); - const source = new Stream(async function* () { - yield* events; - }, new AbortController()); + const source = new Stream(async function* () { + yield* events; + }, new AbortController()); - const client = wrapOpenAI({ - chat: { completions: { create() {} } }, - responses: { create() {} }, - embeddings: { create() {} }, - audio: { transcriptions: { create: async (_params: unknown) => source } }, - }); + const client = wrapOpenAI({ + chat: { completions: { create() {} } }, + responses: { create() {} }, + embeddings: { create() {} }, + audio: { transcriptions: { create: async (_params: unknown) => source } }, + }); - const stream = await client.audio!.transcriptions!.create({ - model: "gpt-4o-transcribe", - stream: true, - }); + const stream = await client.audio!.transcriptions!.create({ + model: "gpt-4o-transcribe", + stream: true, + }); - expect(await collectStream(stream)).toHaveLength(events.length); - const span = await exportedSpan(spans); - expect(span.attributes["gen_ai.output.messages"]).toBe("x".repeat(65_534)); -}); + expect(await collectStream(stream)).toHaveLength(events.length); + const span = await exportedSpan(spans); + expect(span.attributes["gen_ai.output.messages"]).toBe("x".repeat(65_534)); + }, +); test.each([ ["enabled", true, "hello"], @@ -5656,36 +5391,27 @@ test("realtime wrapper correlates reversed manual responses and ignores automati expect(withMetadataFailure.find((span) => span.status.code === SPAN_STATUS_ERROR)).toBeDefined(); }); -test("realtime wrapper ends active spans on transport errors and close", async () => { +test("realtime close fails pending spans and stops tracing later sends", async () => { const spans = setupSpans(); - const listeners = new Map void>>(); + const sent: unknown[] = []; const connection = { - send(_event?: unknown) {}, - close() {}, - on(event: string, listener: (...args: unknown[]) => void) { - listeners.set(event, [...(listeners.get(event) ?? []), listener]); - }, - off(event: string, listener: (...args: unknown[]) => void) { - listeners.set( - event, - (listeners.get(event) ?? []).filter((candidate) => candidate !== listener), - ); + send(event: unknown) { + sent.push(event); }, + close() {}, + on() {}, + off() {}, }; const wrapped = wrapOpenAIRealtime(connection); - const emit = (event: string, value: unknown) => { - for (const listener of listeners.get(event) ?? []) listener(value); - }; - wrapped.send({ type: "response.create" }); - emit("error", "socket failed"); wrapped.send({ type: "response.create" }); wrapped.close(); wrapped.send({ type: "response.create" }); + expect(sent[2]).toEqual({ type: "response.create" }); const finished = await finishedSpans(spans, 2); expect(finished.every((span) => span.status.code === SPAN_STATUS_ERROR)).toBe(true); expect(spans.getFinishedSpans()).toHaveLength(2); diff --git a/packages/opencode/tests/opencode.test.ts b/packages/opencode/tests/opencode.test.ts index 35bdad0..69851b4 100644 --- a/packages/opencode/tests/opencode.test.ts +++ b/packages/opencode/tests/opencode.test.ts @@ -264,34 +264,6 @@ test.each([ expect(span.endTime[0]).toBe(1_700_000_021); }); -test("terminal session.error marks the agent span when the session settles", async () => { - const harness = await makeHarness(); - await chatMessage(harness.hooks); - - await event(harness.hooks, "session.error", { - sessionID: "session-1", - error: { name: "ApiError", data: { message: "provider overloaded" } }, - }); - await event(harness.hooks, "session.idle", { sessionID: "session-1" }); - await flush(); - - const span = spanByName(harness.exporters, "invoke_agent"); - expect(span.status.code).toBe(SpanStatusCode.ERROR); - expect(span.attributes["error.type"]).toBe("ApiError"); - - const records = harness.exporters.logs.getFinishedLogRecords(); - expect(records).toHaveLength(1); - expect(records[0]?.body).toBe("Session error"); - expect(records[0]?.eventName).toBe("session.error"); - expect(records[0]?.severityText).toBe("ERROR"); - expect(records[0]?.attributes).toMatchObject({ - "gen_ai.agent.name": "opencode", - "gen_ai.conversation.id": "session-1", - "opencode.error.message": "provider overloaded", - "opencode.error.name": "ApiError", - }); -}); - test("dispose ends every dangling agent span as incomplete and flushes", async () => { const harness = await makeHarness(); await chatMessage(harness.hooks, "session-1"); @@ -307,23 +279,6 @@ test("dispose ends every dangling agent span as incomplete and flushes", async ( } }); -test("dispose applies a deferred session.error instead of ending the span as incomplete", async () => { - const harness = await makeHarness(); - await chatMessage(harness.hooks); - await event(harness.hooks, "session.error", { - sessionID: "session-1", - error: { name: "ApiError", data: { message: "provider overloaded" } }, - }); - - // The plugin is disposed before the session ever goes idle. - await harness.hooks.dispose?.(); - - const span = spanByName(harness.exporters, "invoke_agent"); - expect(span.status.code).toBe(SpanStatusCode.ERROR); - expect(span.attributes["error.type"]).toBe("ApiError"); - expect(span.attributes["gen_ai.response.finish_reasons"]).toBeUndefined(); -}); - describe.each([ { name: "default capture", @@ -382,10 +337,19 @@ describe.each([ const stack = String(exception?.["exception.stacktrace"]); expect(stack.startsWith(`UnknownError: ${message}\n`), name).toBe(true); expect(stack.includes("private-attachment-listing"), name).toBe(message !== "session error"); + expect(span.attributes["gen_ai.response.finish_reasons"], name).toBeUndefined(); } const records = harness.exporters.logs.getFinishedLogRecords(); - expect(records[0]?.attributes["opencode.error.name"]).toBe("UnknownError"); + expect(records).toHaveLength(1); + expect(records[0]?.body).toBe("Session error"); + expect(records[0]?.eventName).toBe("session.error"); + expect(records[0]?.severityText).toBe("ERROR"); + expect(records[0]?.attributes).toMatchObject({ + "gen_ai.agent.name": "opencode", + "gen_ai.conversation.id": "session-1", + "opencode.error.name": "UnknownError", + }); expect(records[0]?.attributes["opencode.error.message"]).toBe(logMessage); }); }); diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index dcfa465..1296de0 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -2081,47 +2081,38 @@ test("multi-choice chat streams reconstruct every choice", async () => { expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["stop", "length"]); }); -test("non-2xx chat responses end one error span and propagate the failure", async () => { +test.each([ + { + failure: "non-2xx responses", + fetcher: (): Fetcher => + createFakeFetcher( + new Response(JSON.stringify({ error: { code: 500, message: "provider exploded" } }), { + status: 500, + headers: { "content-type": "application/json" }, + }), + ).fetcher, + message: "provider exploded", + }, + { + failure: "rejected transport requests", + fetcher: (): Fetcher => () => Promise.reject(new Error("network down")), + message: "network down", + }, +])("chat $failure end one error span and propagate the failure", async ({ fetcher, message }) => { const spans = setupSpans(); - const fake = createFakeFetcher( - new Response(JSON.stringify({ error: { code: 500, message: "provider exploded" } }), { - status: 500, - headers: { "content-type": "application/json" }, - }), - ); - await expect( - clientWith(fake.fetcher).chat.send({ + clientWith(fetcher()).chat.send({ chatRequest: { model: "openai/gpt-4o", messages: [{ role: "user", content: "Boom" }], }, }), - ).rejects.toThrow(); - - const span = await exportedSpan(spans); - expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.events[0]?.name).toBe("exception"); -}); - -test("rejected transport requests end one error span and rethrow", async () => { - const spans = setupSpans(); - const failure = new Error("network down"); - const fetcher: Fetcher = () => Promise.reject(failure); - - await expect( - clientWith(fetcher).chat.send({ - chatRequest: { - model: "openai/gpt-4o", - messages: [{ role: "user", content: "Offline" }], - }, - }), - ).rejects.toThrow("network down"); + ).rejects.toThrow(message); const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.events[0]?.attributes?.["exception.message"]).toContain("network down"); + expect(span.events[0]?.attributes?.["exception.message"]).toContain(message); }); test("mid-stream body failures end one error span with partial output", async () => { diff --git a/packages/otel/src/transport.ts b/packages/otel/src/transport.ts index 4c684ed..2fde519 100644 --- a/packages/otel/src/transport.ts +++ b/packages/otel/src/transport.ts @@ -250,7 +250,6 @@ async function postSerialized( target: OtlpTarget, transport: Transport, label: string, - post: typeof postOtlp = postOtlp, signal?: AbortSignal, ): Promise { const { body: finalBody, contentEncoding } = await maybeGzip(body); @@ -259,7 +258,7 @@ async function postSerialized( ? { ...target.headers, "content-encoding": contentEncoding } : target.headers; - const res = await post({ + const res = await postOtlp({ fetchImpl: transport.fetchImpl, url: target.url, headers, @@ -387,7 +386,7 @@ function createOtlpBatchSender( return; } - await postSerialized(body, target, transport, label, postOtlp, signal); + await postSerialized(body, target, transport, label, signal); }; return send; @@ -446,7 +445,7 @@ export function createMetricExporter(target: OtlpTarget, transport: Transport): : undefined; if (body !== undefined && body.byteLength > 0) { - await postSerialized(body, target, transport, "metric", postOtlp, signal); + await postSerialized(body, target, transport, "metric", signal); } }, resultCallback); }, diff --git a/packages/otel/tests/otel-byo.test.ts b/packages/otel/tests/otel-byo.test.ts index 814d1b6..346d581 100644 --- a/packages/otel/tests/otel-byo.test.ts +++ b/packages/otel/tests/otel-byo.test.ts @@ -7,6 +7,7 @@ import { trace, TraceFlags, } from "@opentelemetry/api"; +import { ExportResultCode } from "@opentelemetry/core"; import { AlwaysOffSampler, AlwaysOnSampler, @@ -99,57 +100,54 @@ test("spanFilter narrows what gets exported", async () => { }); test("without an api key or exporter override the processor is a silent no-op", async () => { - const previous = process.env.TELEMETRY_DEV_API_KEY; - delete process.env.TELEMETRY_DEV_API_KEY; + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + const urls: string[] = []; + + const fetchImpl: typeof fetch = (input) => { + urls.push(input instanceof URL ? input.href : input instanceof Request ? input.url : input); + + return Promise.resolve(new Response(null, { status: 200 })); + }; + + for (const apiKey of [undefined, "td_live_x"]) { + const processor = new TelemetrySpanProcessor({ + apiKey, + baseUrl: "https://ingest.example", + fetch: fetchImpl, + }); - try { - const processor = new TelemetrySpanProcessor(); const provider = new BasicTracerProvider({ spanProcessors: [processor] }); - provider.getTracer("x").startSpan("lost").end(); + provider.getTracer("x").startSpan("span").end(); await processor.forceFlush(); await processor.shutdown(); - } finally { - if (previous !== undefined) process.env.TELEMETRY_DEV_API_KEY = previous; + expect(urls).toEqual(apiKey ? ["https://ingest.example/v1/traces"] : []); } }); -test("forceFlush and shutdown resolve", async () => { - const { processor } = byoSetup(); - await expect(processor.forceFlush()).resolves.toBeUndefined(); - await expect(processor.shutdown()).resolves.toBeUndefined(); -}); - test("createTelemetrySpanExporter is a no-op without a key and real otherwise", async () => { - const previous = process.env.TELEMETRY_DEV_API_KEY; - delete process.env.TELEMETRY_DEV_API_KEY; - - try { - const noop = createTelemetrySpanExporter(); - - const code = await new Promise((resolve) => - noop.export([], (result) => resolve(result.code)), - ); + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + const calls: Array<{ url: string; method?: string }> = []; - expect(code).toBe(0); - await noop.shutdown(); - - const calls: Array<{ url: string; method?: string }> = []; + const fetchImpl: typeof fetch = (input, init) => { + calls.push({ + url: input instanceof URL ? input.href : input instanceof Request ? input.url : input, + method: init?.method, + }); - const fetchImpl: typeof fetch = (input, init) => { - calls.push({ - url: input instanceof URL ? input.href : input instanceof Request ? input.url : input, - method: init?.method, - }); - - return Promise.resolve(new Response(null, { status: 200 })); - }; + return Promise.resolve(new Response(null, { status: 200 })); + }; + for (const apiKey of [undefined, "td_live_x"]) { const exporter = createTelemetrySpanExporter({ - apiKey: "td_live_x", + apiKey, baseUrl: "https://ingest.example/", fetch: fetchImpl, }); + expect( + await new Promise((resolve) => exporter.export([], (result) => resolve(result.code))), + ).toBe(ExportResultCode.SUCCESS); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)], }); @@ -157,12 +155,12 @@ test("createTelemetrySpanExporter is a no-op without a key and real otherwise", try { provider.getTracer("public-exporter").startSpan("exported").end(); await provider.forceFlush(); - expect(calls).toEqual([{ url: "https://ingest.example/v1/traces", method: "POST" }]); + expect(calls).toEqual( + apiKey ? [{ url: "https://ingest.example/v1/traces", method: "POST" }] : [], + ); } finally { await provider.shutdown(); } - } finally { - if (previous !== undefined) process.env.TELEMETRY_DEV_API_KEY = previous; } }); @@ -245,19 +243,6 @@ test("metrics are skipped without an api key even when enabled", async () => { } }); -test("global tracer interop: registered provider receives raw OTel API spans", async () => { - const { exporter, processor, provider } = byoSetup(); - trace.setGlobalTracerProvider(provider); - - try { - trace.getTracer("third-party").startSpan("via-global").end(); - await processor.forceFlush(); - expect(exporter.getFinishedSpans().map((s) => s.name)).toEqual(["via-global"]); - } finally { - trace.disable(); - } -}); - test("sha256 matches the FIPS vector", () => { const hex = Array.from(sha256(new TextEncoder().encode("abc")), (b) => b.toString(16).padStart(2, "0"), diff --git a/packages/otel/tests/transport.test.ts b/packages/otel/tests/transport.test.ts index 0e07496..a93e1ee 100644 --- a/packages/otel/tests/transport.test.ts +++ b/packages/otel/tests/transport.test.ts @@ -27,9 +27,7 @@ import { createMetricExporter, createLogExporter, createTraceExporter, - maybeGzip, otlpHeaders, - postOtlp, } from "../src/transport.ts"; interface Call { @@ -76,6 +74,8 @@ function makeSpans(count: number, attrBytes = 0): ReadableSpan[] { return exporter.getFinishedSpans(); } +const target = { url: "https://ingest.example/v1/telemetry", headers: otlpHeaders("td_live_x") }; + const exportSpans = (exporter: SpanExporter, spans: ReadableSpan[]): Promise => new Promise((resolve) => exporter.export(spans, (result) => resolve(result.code))); @@ -152,20 +152,6 @@ test("does not retry non-retryable statuses and reports the failure", async () = expect(errors).toHaveLength(1); }); -test("gives up after exhausting retries", async () => { - const { calls, fetchImpl } = makeFetch([503, 503, 503]); - - const res = await postOtlp({ - fetchImpl, - url: "https://ingest.example/v1/traces", - headers: {}, - body: new Uint8Array([1]), - }); - - expect(res.status).toBe(503); - expect(calls).toHaveLength(3); -}); - test("exhausted retries surface as a failed export with onError", async () => { const errors: unknown[] = []; const { calls, fetchImpl } = makeFetch([503, 503, 503]); @@ -204,47 +190,37 @@ describe("retry-after", () => { vi.useRealTimers(); }); - const startPost = (statuses: number[], headers: Record) => { + const startExport = (statuses: number[], headers: Record) => { const { calls, fetchImpl } = makeFetch(statuses, headers); - const pending = postOtlp({ - fetchImpl, - url: "https://ingest.example/v1/traces", - headers: {}, - body: new Uint8Array([1]), - }); - - return { calls, pending }; + return { + calls, + pending: exportSpans(createTraceExporter(target, { fetchImpl }), makeSpans(1)), + }; }; test("sets the delay before the next attempt", async () => { - const { calls, pending } = startPost([503, 200], { "retry-after": "1" }); + const { calls, pending } = startExport([503, 200], { "retry-after": "1" }); await vi.advanceTimersByTimeAsync(999); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test("waits when retry-after fits under the cap", async () => { - const { calls, pending } = startPost([503, 200], { "retry-after": "2" }); + const { calls, pending } = startExport([503, 200], { "retry-after": "2" }); await vi.advanceTimersByTimeAsync(1999); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); - }); - - test("gives up at once when retry-after exceeds the cap", async () => { - const { calls, pending } = startPost([429, 200], { "retry-after": "60" }); - expect((await pending).status).toBe(429); - expect(calls).toHaveLength(1); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test("an HTTP date sets the delay relative to now", async () => { vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); - const { calls, pending } = startPost([503, 200], { + const { calls, pending } = startExport([503, 200], { "retry-after": new Date(Date.now() + 1000).toUTCString(), }); @@ -252,27 +228,27 @@ describe("retry-after", () => { expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test.each(["Thursday, 01-Jan-26 00:00:01 GMT", "Thu Jan 1 00:00:01 2026"])( "accepts obsolete HTTP date %s", async (retryAfter) => { vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); - const { calls, pending } = startPost([503, 200], { "retry-after": retryAfter }); + const { calls, pending } = startExport([503, 200], { "retry-after": retryAfter }); await vi.advanceTimersByTimeAsync(999); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }, ); test("falls back to the fixed schedule when unparseable", async () => { - const { calls, pending } = startPost([503, 200], { "retry-after": "soon" }); + const { calls, pending } = startExport([503, 200], { "retry-after": "soon" }); await vi.advanceTimersByTimeAsync(100); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); test.each([ @@ -288,35 +264,17 @@ describe("retry-after", () => { "January 1, 2026 00:00:01 GMT", ])("falls back to the fixed schedule for invalid value %s", async (retryAfter) => { vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); - const { calls, pending } = startPost([503, 200], { "retry-after": retryAfter }); + const { calls, pending } = startExport([503, 200], { "retry-after": retryAfter }); await vi.advanceTimersByTimeAsync(99); expect(calls).toHaveLength(1); await vi.advanceTimersByTimeAsync(1); expect(calls).toHaveLength(2); - expect((await pending).status).toBe(200); + expect(await pending).toBe(ExportResultCode.SUCCESS); }); }); -test("a missing CompressionStream ships large bodies uncompressed", async () => { - vi.stubGlobal("CompressionStream", undefined); - - try { - const original = new TextEncoder().encode("a".repeat(5000)); - const { body, contentEncoding } = await maybeGzip(original); - expect(contentEncoding).toBeUndefined(); - expect(body).toBe(original); - } finally { - vi.unstubAllGlobals(); - } -}); - -test("bodies above the threshold are gzipped and round-trip", async () => { - const original = new TextEncoder().encode("a".repeat(5000)); - const { body, contentEncoding } = await maybeGzip(original); - expect(contentEncoding).toBe("gzip"); - expect(body.byteLength).toBeLessThan(original.byteLength); - - const decompressed = new Uint8Array( +const gunzip = async (body: Uint8Array) => + new Uint8Array( await new Response( new Blob([body as Uint8Array]) .stream() @@ -324,27 +282,37 @@ test("bodies above the threshold are gzipped and round-trip", async () => { ).arrayBuffer(), ); - expect(decompressed).toEqual(original); -}); - -test("small bodies are not compressed", async () => { - const original = new TextEncoder().encode("tiny"); - const { body, contentEncoding } = await maybeGzip(original); - expect(contentEncoding).toBeUndefined(); - expect(body).toBe(original); -}); - -test("exported batches send a content-encoding gzip header for large payloads", async () => { - const { calls, fetchImpl } = makeFetch([200]); - - const exporter = createTraceExporter( - { url: "https://ingest.example/v1/traces", headers: otlpHeaders("td_live_x") }, - { fetchImpl }, - ); +test.each([ + { name: "large", attrBytes: 10_000, compressionStream: true, encoding: "gzip" }, + { name: "small", attrBytes: 0, compressionStream: true, encoding: undefined }, + { + name: "large without CompressionStream", + attrBytes: 10_000, + compressionStream: false, + encoding: undefined, + }, +])( + "$name batches post the serialized request with content-encoding $encoding", + async ({ attrBytes, compressionStream, encoding }) => { + if (!compressionStream) vi.stubGlobal("CompressionStream", undefined); - await exportSpans(exporter, makeSpans(1, 10_000)); - expect(calls[0]!.headers["content-encoding"]).toBe("gzip"); -}); + try { + const { calls, fetchImpl } = makeFetch([200]); + const spans = makeSpans(1, attrBytes); + const serialized = ProtobufTraceSerializer.serializeRequest(spans)!; + expect(await exportSpans(createTraceExporter(target, { fetchImpl }), spans)).toBe( + ExportResultCode.SUCCESS, + ); + expect(calls).toHaveLength(1); + const { headers, body } = calls[0]!; + expect(headers["content-encoding"]).toBe(encoding); + expect(body.byteLength < serialized.byteLength).toBe(encoding === "gzip"); + expect(encoding === "gzip" ? await gunzip(body) : body).toEqual(serialized); + } finally { + vi.unstubAllGlobals(); + } + }, +); test("oversized batches split into multiple POSTs", async () => { const { calls, fetchImpl } = makeFetch([200]); @@ -423,8 +391,6 @@ test("metric exporter exhausts retries for a network failure", async () => { expect(String(errors[0])).toContain("connection reset"); }); -const target = { url: "https://ingest.example/v1/telemetry", headers: otlpHeaders("td_live_x") }; - const lifecycleCases = [ { name: "trace", diff --git a/packages/pi/tests/pi.test.ts b/packages/pi/tests/pi.test.ts index 88f4803..b957b0f 100644 --- a/packages/pi/tests/pi.test.ts +++ b/packages/pi/tests/pi.test.ts @@ -580,7 +580,7 @@ test("failed assistant message marks the chat and agent spans as errors", async expect(agent.attributes["error.type"]).toBe("error"); }); -test("tool executions produce successful and failed execute_tool spans", async () => { +test("a successful tool execution produces an execute_tool span with arguments and result", async () => { const harness = makeHarness(); await harness.emit({ type: "agent_start" }); await harness.emit({ @@ -596,38 +596,22 @@ test("tool executions produce successful and failed execute_tool spans", async ( result: { content: [{ type: "text", text: "README.md" }] }, isError: false, }); - await harness.emit({ - type: "tool_execution_start", - toolCallId: "call-2", - toolName: "bash", - args: { command: "false" }, - }); - await harness.emit({ - type: "tool_execution_end", - toolCallId: "call-2", - toolName: "bash", - result: { content: [{ type: "text", text: "command failed" }] }, - isError: true, - }); await harness.emit({ type: "agent_end", messages: [] }); await harness.emit({ type: "agent_settled" }); await flush(); - const spans = harness.exporters.spans - .getFinishedSpans() - .filter((span) => span.name === "execute_tool bash"); - - expect(spans).toHaveLength(2); - expect(spans[0]?.attributes).toMatchObject({ + const span = spanByName(harness.exporters, "execute_tool bash"); + const agent = spanByName(harness.exporters, "invoke_agent"); + expect(span.parentSpanContext?.spanId).toBe(agent.spanContext().spanId); + expect(span.attributes).toMatchObject({ "gen_ai.conversation.id": "session-1", "gen_ai.operation.name": "execute_tool", "gen_ai.tool.call.arguments": JSON.stringify({ command: "ls" }), "gen_ai.tool.call.id": "call-1", + "gen_ai.tool.call.result": JSON.stringify({ content: [{ type: "text", text: "README.md" }] }), "gen_ai.tool.name": "bash", }); - expect(spans[0]?.status.code).not.toBe(SpanStatusCode.ERROR); - expect(spans[1]?.status.code).toBe(SpanStatusCode.ERROR); - expect(spans[1]?.attributes["error.type"]).toBe("ToolExecutionError"); + expect(span.status.code).not.toBe(SpanStatusCode.ERROR); }); test.each([ diff --git a/packages/sdk/tests/client.test.ts b/packages/sdk/tests/client.test.ts index 1b0e75b..095b4b8 100644 --- a/packages/sdk/tests/client.test.ts +++ b/packages/sdk/tests/client.test.ts @@ -36,7 +36,7 @@ test("enabled:false yields a no-op client even with a key", () => { const spans = new InMemorySpanExporter(); const client = init( - { apiKey: "td_live_test", enabled: false, logLevel: "silent" }, + { apiKey: "td_live_test", enabled: false, exportMode: "immediate", logLevel: "silent" }, { spanExporter: spans }, ); diff --git a/packages/sdk/tests/metrics.test.ts b/packages/sdk/tests/metrics.test.ts index d2266a2..7d70f48 100644 --- a/packages/sdk/tests/metrics.test.ts +++ b/packages/sdk/tests/metrics.test.ts @@ -1,8 +1,4 @@ -import { - AggregationTemporality, - DataPointType, - type HistogramMetricData, -} from "@opentelemetry/sdk-metrics"; +import { DataPointType, type HistogramMetricData } from "@opentelemetry/sdk-metrics"; import { afterEach, expect, test } from "vitest"; import { flush, shutdown, startSpan } from "../src/index.ts"; @@ -203,7 +199,7 @@ test("plain function spans record no metrics", async () => { expect(histogramPoints(batches, "gen_ai.client.token.usage")).toHaveLength(0); }); -test("histograms use DELTA temporality and the GenAI bucket boundaries", async () => { +test("histograms use the GenAI bucket boundaries", async () => { const { batches, exporter } = makeMetricCapture(); setup({}, { metricExporter: exporter }); startSpan("gen", { type: "generation", usage: { inputTokens: 10 } }).end(); @@ -214,7 +210,6 @@ test("histograms use DELTA temporality and the GenAI bucket boundaries", async ( .flatMap((sm) => sm.metrics) .find((m) => m.descriptor.name === "gen_ai.client.token.usage")!; - expect(metric.aggregationTemporality).toBe(AggregationTemporality.DELTA); expect(metric.dataPointType).toBe(DataPointType.HISTOGRAM); const point = metric.dataPoints[0]!; expect( @@ -246,15 +241,24 @@ test("filter-rejected spans record no metrics", async () => { expect(histogramPoints(batches, "gen_ai.client.operation.time_to_first_chunk")).toHaveLength(0); }); -test("quiet intervals collect zero data points", async () => { - const { batches, exporter } = makeMetricCapture(); - setup({}, { metricExporter: exporter }); - await flush(); +test("the metric exporter sends DELTA intervals: idle flushes post nothing and points are not resent", async () => { + const urls: string[] = []; - const pointCount = batches - .flatMap((rm) => rm.scopeMetrics) - .flatMap((sm) => sm.metrics) - .reduce((total, metric) => total + metric.dataPoints.length, 0); + setup({ + baseUrl: "https://ingest.test", + fetch: async (input) => { + urls.push(new Request(input).url); + + return new Response(null, { status: 200 }); + }, + }); - expect(pointCount).toBe(0); + await flush(); + expect(urls).toEqual([]); + startSpan("gen", { type: "generation", usage: { inputTokens: 10 } }).end(); + await flush(); + expect(urls).toEqual(["https://ingest.test/v1/metrics"]); + // A cumulative exporter would resend the duration and token series on every collection. + await flush(); + expect(urls).toEqual(["https://ingest.test/v1/metrics"]); }); diff --git a/packages/sdk/tests/process-session.test.ts b/packages/sdk/tests/process-session.test.ts index 398812e..4b2d562 100644 --- a/packages/sdk/tests/process-session.test.ts +++ b/packages/sdk/tests/process-session.test.ts @@ -59,6 +59,8 @@ test("explicit sessions win and remain consistent on nested spans and logs", asy }, ); await flush(); + expect(spans.getFinishedSpans()).toHaveLength(6); + expect(logs.getFinishedLogRecords()).toHaveLength(3); for (const span of spans.getFinishedSpans()) { expect(span.attributes["gen_ai.conversation.id"]).toBe(span.name.split("-")[0]); diff --git a/packages/sdk/tests/span.test.ts b/packages/sdk/tests/span.test.ts index 76b2e1f..b569db5 100644 --- a/packages/sdk/tests/span.test.ts +++ b/packages/sdk/tests/span.test.ts @@ -387,16 +387,6 @@ test("updateActiveSpan maps fields using the active span's type", async () => { expect(span.attributes["gen_ai.usage.output_tokens"]).toBe(9); }); -test("spanLimits backstop truncates raw attributes set outside the capture funnel", async () => { - const { spans } = setup(); - const handle = startSpan("raw"); - handle.span.setAttribute("raw.dump", "x".repeat(70000)); - handle.end(); - await flush(); - const value = String(spans.getFinishedSpans()[0]!.attributes["raw.dump"]); - expect(value.length).toBe(65536); -}); - test("per-call captureInput:false drops input but keeps output", async () => { const { spans } = setup(); startSpan("private", { diff --git a/packages/tanstack-ai/tests/evaluate.test.ts b/packages/tanstack-ai/tests/evaluate.test.ts index 589b297..5982e05 100644 --- a/packages/tanstack-ai/tests/evaluate.test.ts +++ b/packages/tanstack-ai/tests/evaluate.test.ts @@ -67,50 +67,38 @@ test("decide emits isolated evaluation spans and one set of usage metrics for co const spans = spanBatches.flat(); expect(spans).toHaveLength(2); expect(spans.every((span) => span.kind === SpanKind.CLIENT)).toBe(true); - expect( - spans - .map((span) => String(span.attributes["gen_ai.provider.name"])) - .sort((a, b) => a.localeCompare(b)), - ).toEqual(["first", "second"]); - expect( - spans - .map((span) => String(span.attributes["gen_ai.request.model"])) - .sort((a, b) => a.localeCompare(b)), - ).toEqual(["first-model", "second-model"]); - expect( - spans.map((span) => Number(span.attributes["gen_ai.usage.input_tokens"])).sort((a, b) => a - b), - ).toEqual([7, 11]); - expect(spans.every((span) => span.attributes["gen_ai.output.messages"] === undefined)).toBe(true); - expect(metrics.filter((metric) => metric.type === "duration")).toHaveLength(2); - expect(metrics.filter((metric) => metric.type === "input")).toHaveLength(2); - expect(metrics.filter((metric) => metric.type === "output")).toHaveLength(2); -}); -test.each([ - { provider: "anthropic", input: 10 + 3 + 2 }, - { provider: "openrouter", input: 10 }, -])( - "decide from $provider records input tokens with cache reads and writes once", - async ({ provider, input }) => { - const { middleware, metrics, spanBatches } = capture(); - - const judge = adapter(provider, async () => ({ - ...answer("judge-model", 10, 2), - usage: { - promptTokens: 10, - completionTokens: 2, - totalTokens: 12, - promptTokensDetails: { cachedTokens: 3, cacheWriteTokens: 2 }, - }, - })); + const byProvider = (provider: string) => + spans.find((span) => span.attributes["gen_ai.provider.name"] === provider)!.attributes; - await decide({ adapter: judge, state: "one", questions, middleware: [middleware] }); + expect(byProvider("first")).toMatchObject({ + "gen_ai.request.model": "first-model", + "gen_ai.usage.input_tokens": 11, + "gen_ai.usage.output_tokens": 3, + }); + expect(byProvider("second")).toMatchObject({ + "gen_ai.request.model": "second-model", + "gen_ai.usage.input_tokens": 7, + "gen_ai.usage.output_tokens": 2, + }); + expect(spans.every((span) => span.attributes["gen_ai.output.messages"] === undefined)).toBe(true); - const [span] = spanBatches.flat(); - expect(span?.attributes["gen_ai.usage.input_tokens"]).toBe(input); - expect(metrics.find((metric) => metric.type === "input")?.value).toBe(input); - }, -); + const tokens = (provider: string) => + metrics + .filter((metric) => metric.attributes["gen_ai.provider.name"] === provider) + .map((metric) => [metric.type, metric.value]); + + expect(tokens("first")).toEqual([ + ["duration", expect.any(Number)], + ["input", 11], + ["output", 3], + ]); + expect(tokens("second")).toEqual([ + ["duration", expect.any(Number)], + ["input", 7], + ["output", 2], + ]); +}); test.each([ { error: new TypeError("provider failed"), type: "TypeError", message: "provider failed" }, diff --git a/packages/tanstack-ai/tests/generation.test.ts b/packages/tanstack-ai/tests/generation.test.ts index b5b4ebd..4da2238 100644 --- a/packages/tanstack-ai/tests/generation.test.ts +++ b/packages/tanstack-ai/tests/generation.test.ts @@ -65,7 +65,7 @@ test.each([ id: "e1", model: "text-embedding-3-small", embeddings: [{ index: 0, vector: [0.1, 0.2] }], - usage: usage(8, 0), + usage: usage(8, 0, { promptTokensDetails: { cachedTokens: 4, cacheWriteTokens: 2 } }), }), }), input: "hello", @@ -77,8 +77,10 @@ test.each([ operation: "embeddings", outputType: undefined, responseModel: undefined, + // OpenAI reports promptTokens with cache reads and writes; the span total adds nothing. input: 8, output: 0, + modality: {}, }, { activity: "image", @@ -106,6 +108,11 @@ test.each([ responseModel: "gpt-image-1-2025", input: 12, output: 40, + modality: { + "gen_ai.usage.text.input_tokens": 10, + "gen_ai.usage.image.input_tokens": 2, + "gen_ai.usage.image.output_tokens": 40, + }, }, { activity: "tts", @@ -131,6 +138,7 @@ test.each([ responseModel: "eleven-v3", input: 5, output: 30, + modality: { "gen_ai.usage.audio.output_tokens": 30 }, }, { activity: "transcription", @@ -155,6 +163,7 @@ test.each([ responseModel: "whisper-1", input: 20, output: 4, + modality: { "gen_ai.usage.audio.input_tokens": 20 }, }, { activity: "summarize", @@ -165,7 +174,9 @@ test.each([ id: "m1", model: "claude-resolved", summary: "secret summary", - usage: usage(100, 10, { promptTokensDetails: { cachedTokens: 30 } }), + usage: usage(100, 10, { + promptTokensDetails: { cachedTokens: 30, cacheWriteTokens: 5 }, + }), }), }), text: "long secret article", @@ -177,9 +188,10 @@ test.each([ operation: "chat", outputType: "text", responseModel: "claude-resolved", - // Anthropic reports promptTokens without cache reads; the span total includes them. - input: 130, + // Anthropic reports promptTokens without cache reads and writes; the span total includes them. + input: 135, output: 10, + modality: {}, }, ])( "$activity emits one $operation span with provider, model, and usage", @@ -193,6 +205,7 @@ test.each([ responseModel, input, output, + modality, }) => { const { middleware, spans, metrics, errors } = capture(); @@ -212,6 +225,13 @@ test.each([ expect(span!.attributes["gen_ai.response.model"]).toBe(responseModel); expect(span!.attributes["gen_ai.usage.input_tokens"]).toBe(input); expect(span!.attributes["gen_ai.usage.output_tokens"]).toBe(output); + expect( + Object.fromEntries( + Object.entries(span!.attributes).filter(([key]) => + /^gen_ai\.usage\.(text|image|audio)\./.test(key), + ), + ), + ).toEqual(modality); // Inputs and outputs (prompts, text, audio, image bytes) are never captured. const values = JSON.stringify(Object.values(span!.attributes)); @@ -229,33 +249,6 @@ test.each([ }, ); -test("media usage records provider-reported modality token breakdowns", async () => { - const { middleware, spans } = capture(); - - await generateImage({ - adapter: adapter("image", "openai", { - generateImages: async () => ({ - id: "i1", - model: "gpt-image-1", - images: [{ url: "https://example.com/i.png" }], - usage: usage(12, 40, { - promptTokensDetails: { textTokens: 10, imageTokens: 2 }, - completionTokensDetails: { imageTokens: 40 }, - }), - }), - }), - prompt: "p", - middleware: [middleware], - }); - - expect(spans[0]!.attributes).toMatchObject({ - "gen_ai.usage.text.input_tokens": 10, - "gen_ai.usage.image.input_tokens": 2, - "gen_ai.usage.image.output_tokens": 40, - }); - expect(spans[0]!.attributes["gen_ai.usage.audio.input_tokens"]).toBeUndefined(); -}); - test("a generation without reported usage omits token attributes and metrics", async () => { const { middleware, spans, metrics } = capture(); diff --git a/packages/tanstack-ai/tests/middleware.test.ts b/packages/tanstack-ai/tests/middleware.test.ts index f87f359..54f852b 100644 --- a/packages/tanstack-ai/tests/middleware.test.ts +++ b/packages/tanstack-ai/tests/middleware.test.ts @@ -8,9 +8,9 @@ import { SamplingDecision, TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; -import { type ChatMiddlewareContext } from "@tanstack/ai"; +import { chat, type ChatMiddlewareContext, embed, type EmbeddingAdapter } from "@tanstack/ai"; import { sessionSpanContext } from "@telemetry-dev/otel"; -import { expect, test } from "vitest"; +import { expect, test, vi } from "vitest"; import { telemetryDev } from "../src/index.ts"; @@ -313,48 +313,34 @@ test.each([ } }); -test("bigint metadata IDs populate user and conversation attributes", async () => { - const { spanBatches, overrides } = makeCapture(); - const mw = middleware(overrides); - const ctx = makeCtx({ options: { metadata: { userId: 2n, sessionId: 9n } } }); - - await mw.onStart?.(ctx); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); - - const spans = spanBatches[0]!; - - for (const s of spans) { - expect(s.attributes["gen_ai.conversation.id"]).toBe("9"); - } - - const root = spans.find((s) => s.kind === SpanKind.INTERNAL)!; - expect(root.attributes["user.id"]).toBe("2"); - // Reserved keys never leak into td.metadata.*. - expect(root.attributes["td.metadata.userId"]).toBeUndefined(); - expect(root.attributes["td.metadata.sessionId"]).toBeUndefined(); -}); - -test("string metadata IDs override threadId and populate user attributes", async () => { - const { spanBatches, overrides } = makeCapture(); - const mw = middleware(overrides); - const ctx = makeCtx({ options: { metadata: { userId: "u2", sessionId: "sess_9" } } }); +test.each([ + { kind: "bigint", userId: 2n, sessionId: 9n, user: "2", conversation: "9" }, + { kind: "string", userId: "u2", sessionId: "sess_9", user: "u2", conversation: "sess_9" }, +])( + "$kind metadata IDs override threadId and populate user and conversation attributes", + async ({ userId, sessionId, user, conversation }) => { + const { spanBatches, overrides } = makeCapture(); + const mw = middleware(overrides); + const ctx = makeCtx({ options: { metadata: { userId, sessionId } } }); - await mw.onStart?.(ctx); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); + await mw.onStart?.(ctx); + await mw.onConfig?.(ctx, chatConfig()); + await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); + await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); - const spans = spanBatches[0]!; + const spans = spanBatches[0]!; - for (const span of spans) { - expect(span.attributes["gen_ai.conversation.id"]).toBe("sess_9"); - } + for (const s of spans) { + expect(s.attributes["gen_ai.conversation.id"]).toBe(conversation); + } - const root = spans.find((span) => span.kind === SpanKind.INTERNAL)!; - expect(root.attributes["user.id"]).toBe("u2"); -}); + const root = spans.find((s) => s.kind === SpanKind.INTERNAL)!; + expect(root.attributes["user.id"]).toBe(user); + // Reserved keys never leak into td.metadata.*. + expect(root.attributes["td.metadata.userId"]).toBeUndefined(); + expect(root.attributes["td.metadata.sessionId"]).toBeUndefined(); + }, +); test("multi-iteration run with a tool yields invoke_agent root and parents the tool to its iteration", async () => { const { spanBatches, metrics, overrides } = makeCapture(); @@ -559,18 +545,52 @@ test("onAbort closes everything as cancelled", async () => { expect(tool.attributes["error.type"]).toBe("cancelled"); }); -test("no apiKey is a complete no-op (transport never invoked)", async () => { - const { spanBatches, metrics, overrides } = makeCapture(); - const mw = telemetryDev({ environment: "test", serviceName: "svc" }, overrides); - const ctx = makeCtx(); +test("no apiKey makes chat and generation hooks a complete no-op (transport never invoked)", async () => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); - await mw.onStart?.(ctx); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "y" } as never); + try { + let calls = 0; - expect(spanBatches).toHaveLength(0); - expect(metrics).toHaveLength(0); + const mw = telemetryDev({ + environment: "test", + serviceName: "svc", + fetch: async () => { + calls += 1; + + return new Response(null, { status: 200 }); + }, + }); + + const ctx = makeCtx(); + + await mw.onStart?.(ctx); + await mw.onConfig?.(ctx, chatConfig()); + await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); + await mw.onUsage?.(ctx, { promptTokens: 1, completionTokens: 1, totalTokens: 2 } as never); + await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "y" } as never); + + const result = await embed({ + adapter: { + kind: "embedding", + name: "openai", + model: "text-embedding-3-small", + "~types": {}, + createEmbeddings: async () => ({ + id: "e1", + model: "text-embedding-3-small", + embeddings: [{ index: 0, vector: [0.1] }], + usage: { promptTokens: 1, completionTokens: 0, totalTokens: 1 }, + }), + } as unknown as EmbeddingAdapter, + input: "x", + middleware: [mw as never], + }); + + expect(result).toMatchObject({ id: "e1" }); + expect(calls).toBe(0); + } finally { + vi.unstubAllEnvs(); + } }); test("one middleware instance handles concurrent chats independently", async () => { @@ -715,22 +735,20 @@ test.each(["finish", "error", "abort"])( }, ); -test("metric export retries a transient ingest failure", async () => { +test("chat telemetry reaches the configured ingest through the configured fetch", async () => { const errors: unknown[] = []; - let metricCalls = 0; + const posts: Array<{ url: string; authorization: string | null }> = []; const mw = telemetryDev({ - apiKey: "td_live_metric_retry", + apiKey: "td_live_wiring", + baseUrl: "https://ingest.example.test/", environment: "test", - serviceName: "metric-retry", - fetch: async (url: string | Request | URL, _init?: RequestInit) => { - const href = url instanceof Request ? url.url : url.toString(); - - if (href.endsWith("/v1/metrics")) { - metricCalls += 1; - - return new Response(null, { status: metricCalls === 1 ? 503 : 200 }); - } + serviceName: "wiring", + fetch: async (url: string | Request | URL, init?: RequestInit) => { + posts.push({ + url: url instanceof Request ? url.url : url.toString(), + authorization: new Headers(init?.headers).get("authorization"), + }); return new Response(null, { status: 200 }); }, @@ -747,8 +765,11 @@ test("metric export retries a transient ingest failure", async () => { await mw.onUsage?.(ctx, { promptTokens: 10, completionTokens: 5, totalTokens: 15 } as never); await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); - expect(metricCalls).toBe(2); - expect(errors).toHaveLength(0); + expect(posts).toEqual([ + { url: "https://ingest.example.test/v1/traces", authorization: "Bearer td_live_wiring" }, + { url: "https://ingest.example.test/v1/metrics", authorization: "Bearer td_live_wiring" }, + ]); + expect(errors).toEqual([]); }); test("Ollama-style nested sampling options populate gen_ai.request attrs", async () => { @@ -794,23 +815,56 @@ test("an iteration with no usage records no token attributes or metrics (not zer expect(String(summary?.attributes?.["log.message"])).not.toContain("tokens"); }); -test("onConfig during init opens no iteration span", async () => { - const { spanBatches, overrides } = makeCapture(); - const mw = middleware(overrides); - const ctx = makeCtx(); +// The other chat tests script the hook order; this one pins it to the real engine, which fires +// onConfig for phase "init" before onStart and again before each model call. +test("a real chat() run opens one iteration span per model call", async () => { + const { spanBatches, metrics, overrides } = makeCapture(); + const timestamp = Date.now(); - await mw.onStart?.(ctx); - // The engine may fire onConfig at init before any model call; init must not open an iteration span. - Object.assign(ctx, { phase: "init" }); - await mw.onConfig?.(ctx, chatConfig()); - Object.assign(ctx, { phase: "beforeModel" }); - await mw.onConfig?.(ctx, chatConfig()); - await mw.onChunk?.(ctx, { type: "RUN_FINISHED", finishReason: "stop" } as never); - await mw.onFinish?.(ctx, { finishReason: "stop", duration: 5, content: "ok" } as never); + const adapter = { + kind: "text", + name: "openai", + model: "gpt-4o", + "~types": {}, + async *chatStream() { + yield { type: "RUN_STARTED", runId: "run_real", threadId: "thread_real", timestamp }; + yield { type: "TEXT_MESSAGE_START", messageId: "m1", role: "assistant", timestamp }; + yield { type: "TEXT_MESSAGE_CONTENT", messageId: "m1", delta: "hello", timestamp }; + yield { type: "TEXT_MESSAGE_END", messageId: "m1", timestamp }; + yield { + type: "RUN_FINISHED", + runId: "run_real", + threadId: "thread_real", + finishReason: "stop", + model: "gpt-4o-2024-11-20", + usage: { promptTokens: 3, completionTokens: 2, totalTokens: 5 }, + timestamp, + }; + }, + structuredOutput: async () => ({ data: {}, rawText: "{}" }), + }; + + const stream = chat({ + adapter: adapter as never, + messages: [{ role: "user", content: "hi" }], + middleware: [middleware(overrides)], + }); + + for await (const _chunk of stream); + expect(spanBatches).toHaveLength(1); const spans = spanBatches[0]!; - expect(spans).toHaveLength(2); - expect(spans.filter((s) => s.kind === SpanKind.CLIENT)).toHaveLength(1); + expect(spans.filter((s) => s.kind === SpanKind.INTERNAL)).toHaveLength(1); + const iterations = spans.filter((s) => s.kind === SpanKind.CLIENT); + expect(iterations).toHaveLength(1); + const iteration = iterations[0]!; + expect(JSON.parse(String(iteration.attributes["gen_ai.input.messages"]))).toEqual([ + { role: "user", content: "hi" }, + ]); + expect(iteration.attributes["gen_ai.output.messages"]).toBe("hello"); + expect(iteration.attributes["gen_ai.usage.input_tokens"]).toBe(3); + expect(iteration.attributes["gen_ai.response.model"]).toBe("gpt-4o-2024-11-20"); + expect(metrics.filter((m) => m.metric === "duration")).toHaveLength(1); }); test("structured output finalization records a separate iteration span", async () => { From bdc2fc02c625fa76f2ca06bdb47b62dd83d696ba Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Mon, 26 Oct 2026 18:10:07 +0000 Subject: [PATCH 2/5] test(sdk-python): remove Python tests that other tests already cover Each package keeps one test per contract, through the real provider client with a fake network where one exists. Tests that do the same checks on private helpers are removed. Some kept assertions could not fail. These assertions are repaired. For each repair, a deliberate change to the owner code made the test fail. Private parameters and fallbacks that only tests used are removed from the core metrics module and the OpenAI and Anthropic integrations. --- .../src/telemetry_dev_anthropic/__init__.py | 99 +- sdks/python-anthropic/tests/test_anthropic.py | 590 ++---- sdks/python-bedrock/tests/test_bedrock.py | 92 +- .../tests/test_google_genai.py | 83 +- sdks/python-litellm/tests/test_litellm.py | 266 +-- .../src/telemetry_dev_openai/__init__.py | 44 +- sdks/python-openai/tests/test_openai.py | 1824 ++++++----------- .../tests/test_openrouter.py | 121 +- sdks/python/src/telemetry_dev/_metrics.py | 5 +- sdks/python/tests/otlp_capture.py | 7 +- sdks/python/tests/test_config.py | 41 - sdks/python/tests/test_init.py | 37 +- sdks/python/tests/test_logs.py | 25 - sdks/python/tests/test_metrics.py | 42 +- sdks/python/tests/test_otel.py | 18 +- sdks/python/tests/test_process_session.py | 24 +- sdks/python/tests/test_propagation.py | 31 +- sdks/python/tests/test_wire.py | 24 +- 18 files changed, 1141 insertions(+), 2232 deletions(-) diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index d33efd0..ba6baba 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -21,8 +21,6 @@ __version__ = "0.1.6" ProviderResolver = Callable[[object | None], str] -RequestMapper = Callable[[Mapping[str, Any]], tuple[str, dict[str, Any]]] -ResponseMapper = Callable[[Any], dict[str, Any]] _WRAPPED_ATTR = "_telemetry_dev_anthropic_wrapped" _ORIGINAL_ATTR = "_telemetry_dev_anthropic_original" @@ -457,13 +455,8 @@ def end(**fields: Any) -> None: class _StreamState: - def __init__( - self, - capture_output: bool = True, - expect_message_stop: bool = False, - ) -> None: + def __init__(self, capture_output: bool = True) -> None: self.capture_output = capture_output - self.expect_message_stop = expect_message_stop self.message_stopped = False self.blocks: dict[int, dict[str, Any]] = {} self.tool_json: dict[int, str] = {} @@ -712,7 +705,7 @@ def _stream_partial( output_incomplete = bool( state.budget.truncated or output_truncated - or (state.expect_message_stop and not state.message_stopped) + or not state.message_stopped or state.unresolved_replacements or state.unresolved_encrypted_content ) @@ -1214,27 +1207,9 @@ def _stream_event_has_output(event: Any) -> bool: return input_value not in (None, "", [], {}) -def _stream_span_capture_policy(handle: telemetry_dev.SpanHandle) -> tuple[bool, bool]: - compatible = cast(Any, handle) - client = getattr(compatible, "_client", None) - state = getattr(compatible, "_state", None) - capture_output = getattr(compatible, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(state, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(client, "capture_output", False) - capture_masked = getattr(compatible, "capture_masked", None) - if not isinstance(capture_masked, bool): - capture_masked = getattr(client, "mask", _OMIT) is not None if client is not None else True - return capture_output, capture_masked - - def _stream_span_reporter( handle: telemetry_dev.SpanHandle, ) -> Callable[[BaseException], None]: - compatible = cast(Any, handle) - reporter = getattr(compatible, "report_error", None) - client = getattr(compatible, "_client", None) reported = False def report(cause: BaseException) -> None: @@ -1243,12 +1218,7 @@ def report(cause: BaseException) -> None: return reported = True try: - if callable(reporter): - reporter(cause) - return - client_report = getattr(client, "report", None) - if callable(client_report): - client_report("provider instrumentation failed", cause) + handle.report_error(cause) except Exception: return @@ -1256,13 +1226,10 @@ def report(cause: BaseException) -> None: def _end_mapped_response( - handle: telemetry_dev.SpanHandle, - end: Callable[..., None], - mapper: ResponseMapper, - response: Any, + handle: telemetry_dev.SpanHandle, end: Callable[..., None], response: Any ) -> None: try: - fields = mapper(response) + fields = _messages_response(response) except Exception as exc: _stream_span_reporter(handle)(exc) end(attributes={"telemetry.dev.capture.truncated": True}) @@ -1276,8 +1243,8 @@ def __init__(self, inner: Any, handle: telemetry_dev.SpanHandle, started_at: flo self._handle = handle self._end = _end_once(handle) self._started_at = started_at - capture_output, self._mask_output_when_incomplete = _stream_span_capture_policy(handle) - self._state = _StreamState(capture_output, expect_message_stop=True) + self._mask_output_when_incomplete = handle.capture_masked + self._state = _StreamState(handle.capture_output) self._report_error = _stream_span_reporter(handle) self._saw_first = False self._consume: Iterator[Any] | None = None @@ -1320,9 +1287,7 @@ def _iterate(self) -> Iterator[Any]: update = {} has_output = False if has_output: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) self._update_first(update) yield event finally: @@ -1365,8 +1330,8 @@ def __init__(self, inner: Any, handle: telemetry_dev.SpanHandle, started_at: flo self._handle = handle self._end = _end_once(handle) self._started_at = started_at - capture_output, self._mask_output_when_incomplete = _stream_span_capture_policy(handle) - self._state = _StreamState(capture_output, expect_message_stop=True) + self._mask_output_when_incomplete = handle.capture_masked + self._state = _StreamState(handle.capture_output) self._report_error = _stream_span_reporter(handle) self._saw_first = False self._consume: AsyncIterator[Any] | None = None @@ -1409,9 +1374,7 @@ async def _aiterate(self) -> AsyncIterator[Any]: update = {} has_output = False if has_output: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) self._update_first(update) yield event finally: @@ -1535,19 +1498,14 @@ def __getattr__(self, name: str) -> Any: def _start_span( - params: Mapping[str, Any], mapper: RequestMapper, provider: str + params: Mapping[str, Any], provider: str ) -> tuple[telemetry_dev.SpanHandle, Callable[..., None], float]: - name, fields = mapper(params) + name, fields = _messages_request(params) handle = telemetry_dev.start_span(name, provider=provider, **_clean_fields(fields)) return handle, _end_once(handle), time.perf_counter() -def _wrap_sync( - original: Callable[..., Any], - request_mapper: RequestMapper, - response_mapper: ResponseMapper, - provider: ProviderResolver, -) -> Callable[..., Any]: +def _wrap_sync(original: Callable[..., Any], provider: ProviderResolver) -> Callable[..., Any]: @wraps(original) def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None @@ -1555,7 +1513,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, request_mapper, provider(resource)) + handle, end, started_at = _start_span(call_kwargs, provider(resource)) try: result = original(*args, **call_kwargs) except BaseException as exc: @@ -1563,7 +1521,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: raise if streaming: return _InstrumentedStream(result, handle, started_at) - _end_mapped_response(handle, end, response_mapper, result) + _end_mapped_response(handle, end, result) return result setattr(wrapper, _WRAPPED_ATTR, True) @@ -1571,12 +1529,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return wrapper -def _wrap_async( - original: Callable[..., Any], - request_mapper: RequestMapper, - response_mapper: ResponseMapper, - provider: ProviderResolver, -) -> Callable[..., Any]: +def _wrap_async(original: Callable[..., Any], provider: ProviderResolver) -> Callable[..., Any]: @wraps(original) async def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None @@ -1584,7 +1537,7 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: if _raw_response_requested(call_kwargs): return await original(*args, **call_kwargs) streaming = call_kwargs.get("stream") is True - handle, end, started_at = _start_span(call_kwargs, request_mapper, provider(resource)) + handle, end, started_at = _start_span(call_kwargs, provider(resource)) try: result = await original(*args, **call_kwargs) except BaseException as exc: @@ -1592,7 +1545,7 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: raise if streaming: return _InstrumentedAsyncStream(result, handle, started_at) - _end_mapped_response(handle, end, response_mapper, result) + _end_mapped_response(handle, end, result) return result setattr(wrapper, _WRAPPED_ATTR, True) @@ -1601,13 +1554,13 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: def _wrap_stream_manager_sync( - original: Callable[..., Any], request_mapper: RequestMapper, provider: ProviderResolver + original: Callable[..., Any], provider: ProviderResolver ) -> Callable[..., Any]: @wraps(original) def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) - name, fields = request_mapper(call_kwargs) + name, fields = _messages_request(call_kwargs) manager = original(*args, **call_kwargs) return _InstrumentedMessageStreamManager(manager, name, fields, provider(resource)) @@ -1617,13 +1570,13 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: def _wrap_stream_manager_async( - original: Callable[..., Any], request_mapper: RequestMapper, provider: ProviderResolver + original: Callable[..., Any], provider: ProviderResolver ) -> Callable[..., Any]: @wraps(original) def wrapper(*args: Any, **kwargs: Any) -> Any: resource = args[0] if args else None call_kwargs = _normalize_request_params(kwargs) - name, fields = request_mapper(call_kwargs) + name, fields = _messages_request(call_kwargs) manager = original(*args, **call_kwargs) return _InstrumentedAsyncMessageStreamManager(manager, name, fields, provider(resource)) @@ -1651,9 +1604,9 @@ def _wrap_method( ) -> Callable[..., Any]: if name == "stream": factory = _wrap_stream_manager_async if async_resource else _wrap_stream_manager_sync - return factory(original, _messages_request, provider) - response_factory = _wrap_async if async_resource else _wrap_sync - return response_factory(original, _messages_request, _messages_response, provider) + else: + factory = _wrap_async if async_resource else _wrap_sync + return factory(original, provider) def _patch_instance_method( diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index ab8b724..3655220 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -2,11 +2,9 @@ import importlib import importlib.util -import inspect import json import random import sys -import time import types from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence from types import SimpleNamespace @@ -62,42 +60,35 @@ async def test_response_mapping_failure_returns_provider_result_and_marks_captur ) -> None: errors: list[BaseException] = [] memory = make(on_error=errors.append) - mapping_error = RuntimeError("mapping failed") - provider_result = object() - def request_mapper(_params: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: - return "chat claude-sonnet-4-6", { - "type": "generation", - "model": "claude-sonnet-4-6", - } - - def response_mapper(_response: Any) -> dict[str, Any]: - raise mapping_error + class MappingFailure: + @property + def content(self) -> Any: + raise RuntimeError("mapping failed") - def provider(_resource: object | None) -> str: - return "anthropic" + provider_result = MappingFailure() if async_mode: - async def async_original(**_kwargs: Any) -> object: + async def async_create(**_kwargs: Any) -> MappingFailure: return provider_result - wrapper = vars(telemetry_dev_anthropic)["_wrap_async"]( - async_original, request_mapper, response_mapper, provider - ) - result = await wrapper(model="claude-sonnet-4-6") + client: Any = object.__new__(AsyncAnthropic) + messages: Any = object.__new__(AsyncMessages) + messages.create = async_create + client.messages = messages + result = await wrap_anthropic(client).messages.create(model="claude-sonnet-4-6") else: - def sync_original(**_kwargs: Any) -> object: + def create(**_kwargs: Any) -> MappingFailure: return provider_result - wrapper = vars(telemetry_dev_anthropic)["_wrap_sync"]( - sync_original, request_mapper, response_mapper, provider - ) - result = wrapper(model="claude-sonnet-4-6") + client = object.__new__(Anthropic) + client.messages = SimpleNamespace(create=create) + result = wrap_anthropic(client).messages.create(model="claude-sonnet-4-6") assert result is provider_result - assert errors == [mapping_error] + assert [str(error) for error in errors] == ["mapping failed"] assert attrs(only_span(memory))["telemetry.dev.capture.truncated"] is True @@ -695,138 +686,88 @@ def handler(request: httpx.Request) -> httpx.Response: assert len(timestamps) == 2 -def test_stream_mapping_error_reports_and_still_yields_provider_event(make: Any) -> None: - errors: list[BaseException] = [] - memory = make(on_error=errors.append) - - def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(stream_events()) - - class BrokenDelta(Mapping[str, Any]): - def __len__(self) -> int: - return 1 - - def __iter__(self) -> Iterator[str]: - raise ValueError("mapping failed") - - def __getitem__(self, key: str) -> Any: - return key - - provider_event = SimpleNamespace( - type="content_block_delta", - index=0, - delta=BrokenDelta(), - ) - client = wrapped_sync_client(handler) - stream = client.messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True - ) - stream._inner = iter([provider_event, provider_event]) - - assert next(stream) is provider_event - assert next(stream) is provider_event - with pytest.raises(StopIteration): - next(stream) - client.close() - - assert len(errors) == 1 - assert isinstance(errors[0], ValueError) - assert str(errors[0]) == "mapping failed" - span = only_span(memory) - assert span.status.status_code == StatusCode.UNSET - assert attrs(span)["telemetry.dev.capture.truncated"] is True - - -def test_stream_capture_does_not_invoke_expanding_model_dump() -> None: - model_dump_calls = 0 - - class ExpandingDelta: - def __init__(self) -> None: - self.type = "text_delta" - self.text = "safe" - - def model_dump(self, **_kwargs: Any) -> dict[str, Any]: - nonlocal model_dump_calls - model_dump_calls += 1 - return {"type": "text_delta", "text": "safe", "items": list(range(5_000))} - - captured = telemetry_dev_anthropic._bounded_native(ExpandingDelta()) # pyright: ignore[reportPrivateUsage] - - assert captured == {"type": "text_delta", "text": "safe"} - assert model_dump_calls == 0 - - @pytest.mark.parametrize("async_mode", [False, True]) @pytest.mark.parametrize("capture_output", [False, True]) -async def test_stream_uses_bound_handle_capture_policy_and_reporter( - monkeypatch: pytest.MonkeyPatch, async_mode: bool, capture_output: bool +async def test_stream_mapping_error_reports_and_still_yields_provider_event( + make: Any, async_mode: bool, capture_output: bool ) -> None: errors: list[BaseException] = [] - global_reports: list[BaseException] = [] - ended: list[dict[str, Any]] = [] + memory = make(capture_output=capture_output, on_error=errors.append) class BrokenMessage: @property def id(self) -> str: raise RuntimeError("broken message") - def end(**fields: Any) -> None: - ended.append(fields) - - def update(**_fields: Any) -> None: - return None - - def report_global(_message: str, error: BaseException) -> None: - global_reports.append(error) - - handle = SimpleNamespace( - capture_output=capture_output, - capture_masked=True, - report_error=errors.append, - end=end, - update=update, - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(report=report_global), - ) + broken = {"type": "message_start", "message": BrokenMessage()} + # message_stop makes the stream complete, so only the mapping error can flag the span. events = [ { "type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": "sensitive"}, }, - {"type": "message_start", "message": BrokenMessage()}, + broken, + broken, + {"type": "message_stop"}, ] if async_mode: + async def handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(stream_events()) + async def source() -> AsyncIterator[Any]: for event in events: yield event - stream = telemetry_dev_anthropic._InstrumentedAsyncStream( # pyright: ignore[reportPrivateUsage] - source(), cast(Any, handle), time.perf_counter() + client = wrapped_async_client(handler) + stream = await client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True ) + stream._inner = source() delivered = [event async for event in stream] + await client.close() else: - stream = telemetry_dev_anthropic._InstrumentedStream( # pyright: ignore[reportPrivateUsage] - iter(events), cast(Any, handle), time.perf_counter() + + def sync_handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(stream_events()) + + client = wrapped_sync_client(sync_handler) + stream = client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True ) + stream._inner = iter(events) delivered = list(stream) + client.close() - state = cast(Any, stream)._state assert delivered == events - if capture_output: - assert state.blocks == {0: {"type": "text", "text": "sensitive"}} - else: - assert state.blocks == {} - assert "output" not in ended[0] - assert len(errors) == 1 - assert isinstance(errors[0], RuntimeError) - assert global_reports == [] - assert ended[0]["attributes"] == {"telemetry.dev.capture.truncated": True} + assert stream._state.blocks == ( + {0: {"type": "text", "text": "sensitive"}} if capture_output else {} + ) + assert [str(error) for error in errors] == ["broken message"] + span = only_span(memory) + assert span.status.status_code == StatusCode.UNSET + assert attrs(span)["telemetry.dev.capture.truncated"] is True + + +def test_stream_capture_does_not_invoke_expanding_model_dump() -> None: + model_dump_calls = 0 + + class ExpandingDelta: + def __init__(self) -> None: + self.type = "text_delta" + self.text = "safe" + + def model_dump(self, **_kwargs: Any) -> dict[str, Any]: + nonlocal model_dump_calls + model_dump_calls += 1 + return {"type": "text_delta", "text": "safe", "items": list(range(5_000))} + + captured = telemetry_dev_anthropic._bounded_native(ExpandingDelta()) # pyright: ignore[reportPrivateUsage] + + assert captured == {"type": "text_delta", "text": "safe"} + assert model_dump_calls == 0 def test_streaming_response_helper_preserves_api_response(memory: SimpleNamespace) -> None: @@ -886,19 +827,33 @@ async def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.usage.output_tokens"] == 2 -def test_sync_stream_bounds_retained_events_without_dropping_chunks( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_stream_bounds_retained_events_without_dropping_chunks( + memory: SimpleNamespace, async_mode: bool ) -> None: source_events = oversized_stream_events() - def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(source_events) + if async_mode: - client = wrapped_sync_client(handler) - stream = client.messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True - ) - delivered = list(stream) + async def handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(source_events) + + client = wrapped_async_client(handler) + stream = await client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True + ) + delivered = [event async for event in stream] + await client.close() + else: + + def sync_handler(request: httpx.Request) -> httpx.Response: + return named_sse_response(source_events) + + client = wrapped_sync_client(sync_handler) + stream = client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True + ) + delivered = list(stream) state = stream._state assert len(delivered) == len(source_events) @@ -908,24 +863,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert attrs(only_span(memory))["telemetry.dev.capture.truncated"] is True -def test_stream_capture_never_exceeds_the_48_kib_provider_ceiling() -> None: - state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] - telemetry_dev_anthropic._record_stream_event( # pyright: ignore[reportPrivateUsage] - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": "x" * (49 * 1024)}, - }, - state, - ) - - partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] - - assert state.budget.max_bytes == 48 * 1024 - assert partial["output"] is None - assert partial["attributes"] == {"telemetry.dev.capture.truncated": True} - - def test_stream_revalidates_expanded_tool_json_against_the_item_limit() -> None: state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] record = telemetry_dev_anthropic._record_stream_event # pyright: ignore[reportPrivateUsage] @@ -948,6 +885,7 @@ def test_stream_revalidates_expanded_tool_json_against_the_item_limit() -> None: }, state, ) + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] @@ -978,7 +916,8 @@ def test_stream_rejects_normalized_tool_block_over_budget_but_keeps_metadata( "id": "toolu_oversized", "name": "oversized_tool", "input": {}, - "oversized": "x" * (64 * 1024), + # Over the 48 KiB stream bound but under the 64 KiB core cap. + "oversized": "x" * (49 * 1024), }, }, { @@ -1001,33 +940,12 @@ def handler(request: httpx.Request) -> httpx.Response: assert stream._state.budget.truncated is True a = attrs(only_span(memory)) assert "gen_ai.output.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True assert a["gen_ai.usage.input_tokens"] == 5 assert a["gen_ai.usage.output_tokens"] == 3 assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["tool_use"] -async def test_async_stream_bounds_retained_events_without_dropping_chunks( - memory: SimpleNamespace, -) -> None: - source_events = oversized_stream_events() - - async def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(source_events) - - client = wrapped_async_client(handler) - stream = await client.messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True - ) - delivered = [event async for event in stream] - state = stream._state - await client.close() - - assert len(delivered) == len(source_events) - assert state.budget.truncated is True - assert state.budget.bytes_used <= state.budget.max_bytes - assert len(state.blocks[0]["text"]) < 64 * 1024 - - def test_create_streaming_preserves_citation_deltas(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return named_sse_response(citation_stream_events()) @@ -1229,14 +1147,21 @@ async def handler(request: httpx.Request) -> httpx.Response: def test_messages_stream_enter_failure_records_error(memory: SimpleNamespace) -> None: + requests: list[httpx.Request] = [] + def handler(request: httpx.Request) -> httpx.Response: - return error_response(500, "server broke") + requests.append(request) + response = error_response(500, "server broke") + response.headers["retry-after-ms"] = "1" + return response client = wrapped_sync_client(handler) with pytest.raises(anthropic.APIStatusError): with client.messages.stream(model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES): pass + # The SDK retries inside one span. + assert len(requests) > 1 span = only_span(memory) assert span.status.status_code == StatusCode.ERROR assert any(event.name == "exception" for event in span.events) @@ -1429,41 +1354,6 @@ def stream_stub(**_kwargs: Any) -> None: assert attrs(only_span(memory))["gen_ai.provider.name"] == expected -def test_provider_mapping_for_bedrock_vertex_and_plain_client(memory: SimpleNamespace) -> None: - anthropic_any: Any = anthropic - bedrock = object.__new__(anthropic_any.AnthropicBedrock) - vertex = object.__new__(anthropic_any.AnthropicVertex) - - def bedrock_create(**_kwargs: Any) -> dict[str, Any]: - return message_payload(id="msg_bedrock") - - def vertex_create(**_kwargs: Any) -> dict[str, Any]: - return message_payload(id="msg_vertex") - - def stream_stub(**_kwargs: Any) -> None: - return None - - bedrock.messages = SimpleNamespace(create=bedrock_create, stream=stream_stub) - vertex.messages = SimpleNamespace(create=vertex_create, stream=stream_stub) - - def handler(request: httpx.Request) -> httpx.Response: - return json_response(message_payload()) - - wrap_anthropic(bedrock).messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) - wrap_anthropic(vertex).messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) - wrapped_sync_client(handler).messages.create( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) - bedrock_span, vertex_span, plain_span = memory.span_exporter.get_finished_spans() - assert attrs(bedrock_span)["gen_ai.provider.name"] == "aws.bedrock" - assert attrs(vertex_span)["gen_ai.provider.name"] == "gcp.vertex_ai" - assert attrs(plain_span)["gen_ai.provider.name"] == "anthropic" - - def test_beta_create_records_generation_span(memory: SimpleNamespace) -> None: requests: list[httpx.Request] = [] @@ -1488,52 +1378,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.response.finish_reasons"] == ("end_turn",) -async def test_async_beta_create_records_generation_span(memory: SimpleNamespace) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return json_response(message_payload(id="msg_beta_async")) - - client = wrapped_async_client(handler) - await client.beta.messages.create(model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES) - await client.close() - - assert attrs(only_span(memory))["gen_ai.response.id"] == "msg_beta_async" - - -def test_beta_stream_context_manager_records_one_span(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(stream_events()) - - client = wrapped_sync_client(handler) - with client.beta.messages.stream( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) as stream: - text = "".join(stream.text_stream) - - assert text == "Hello world" - a = attrs(only_span(memory)) - assert json.loads(str(a["gen_ai.output.messages"])) == [ - {"role": "assistant", "content": [{"type": "text", "text": "Hello world"}]} - ] - assert a["gen_ai.usage.output_tokens"] == 2 - - -async def test_async_beta_stream_context_manager_records_one_span( - memory: SimpleNamespace, -) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(stream_events()) - - client = wrapped_async_client(handler) - async with client.beta.messages.stream( - model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES - ) as stream: - parts = [text async for text in stream.text_stream] - await client.close() - - assert "".join(parts) == "Hello world" - assert attrs(only_span(memory))["gen_ai.usage.output_tokens"] == 2 - - @pytest.mark.parametrize("namespace", ["stable", "beta"]) def test_parse_records_one_generation_span(memory: SimpleNamespace, namespace: str) -> None: def handler(request: httpx.Request) -> httpx.Response: @@ -1835,7 +1679,10 @@ def identity_mask(value: Any, _context: Any) -> Any: memory = make(mask=identity_mask) if masked else make() events = beta_stream_events( ({"type": "text", "text": ""}, [{"type": "text_delta", "text": "first"}]), - ({"type": "text", "text": ""}, [{"type": "text_delta", "text": '"' * 500}]), + ( + {"type": "text", "text": ""}, + [{"type": "text_delta", "text": "ab"}, {"type": "text_delta", "text": '"' * 500}], + ), ) a = streamed_beta_span(events, memory) @@ -1845,8 +1692,7 @@ def identity_mask(value: Any, _context: Any) -> Any: assert "gen_ai.output.messages" not in a else: content = json.loads(str(a["gen_ai.output.messages"]))[0]["content"] - assert content[0] == {"type": "text", "text": "first"} - assert all(('"' * 500).startswith(block["text"]) for block in content[1:]) + assert content == [{"type": "text", "text": "first"}, {"type": "text", "text": "ab"}] @pytest.mark.parametrize(("stream_limit", "complete"), [(48 * 1024, True), (1_000, False)]) @@ -1944,25 +1790,6 @@ def test_escape_heavy_deltas_are_charged_at_their_serialized_size( assert a["telemetry.dev.capture.truncated"] is True -def test_block_creating_text_delta_is_charged_at_its_serialized_size() -> None: - implementation = cast(Any, telemetry_dev_anthropic) - chunk = "\x01" * 100 - state = implementation._StreamState() - - implementation._record_stream_event( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": chunk}, - }, - state, - ) - - assert state.blocks[0] == {"type": "text", "text": chunk} - serialized = len(json.dumps(chunk, ensure_ascii=False).encode()) - 2 - assert state.budget.bytes_used >= serialized - - def test_block_creating_text_delta_also_charges_the_block_it_creates() -> None: implementation = cast(Any, telemetry_dev_anthropic) state = implementation._StreamState() @@ -2073,62 +1900,6 @@ def test_lone_surrogate_keeps_the_exported_span_whole(memory: SimpleNamespace) - assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["end_turn"] -def test_lone_surrogate_does_not_discard_the_output_that_already_fit() -> None: - implementation = cast(Any, telemetry_dev_anthropic) - state = implementation._StreamState() - record = implementation._record_stream_event - - record( - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": "keep me"}, - }, - state, - ) - record( - { - "type": "content_block_start", - "index": 1, - "content_block": {"type": "text", "text": "a\ud800"}, - }, - state, - ) - - partial = implementation._stream_partial(state) - - assert partial["output"] is not None - assert partial["output"][0]["content"][0]["text"] == "keep me" - - -def test_tool_input_over_the_bound_is_caught_while_streaming() -> None: - implementation = cast(Any, telemetry_dev_anthropic) - state = implementation._StreamState() - record = implementation._record_stream_event - - record( - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "tool_use", "id": "t1", "name": "calc"}, - }, - state, - ) - # Quote-light JSON: nothing to escape, but every element gains ", " once serialized. - payload = '{"n":[' + ",".join(["1"] * 20_000) + "]}" - record( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "input_json_delta", "partial_json": payload}, - }, - state, - ) - record({"type": "content_block_stop", "index": 0}, state) - - assert state.budget.truncated is True - - def _tool_stream(payloads: list[str], start_input: dict[str, Any] | None = None) -> Any: implementation = cast(Any, telemetry_dev_anthropic) state = implementation._StreamState() @@ -2167,15 +1938,20 @@ def test_settled_tool_input_charge_covers_what_is_still_retained( assert retained <= state.budget.bytes_used -@pytest.mark.parametrize("elements", [8, 1_500, 5_000, 20_000]) -def test_settled_tool_input_agrees_with_the_final_bounded_check(elements: int) -> None: +@pytest.mark.parametrize( + ("elements", "truncated"), [(8, False), (1_500, True), (5_000, True), (20_000, True)] +) +def test_settled_tool_input_agrees_with_the_final_bounded_check( + elements: int, truncated: bool +) -> None: implementation = cast(Any, telemetry_dev_anthropic) state = _tool_stream(['{"n":[' + ",".join(["1"] * elements) + "]}"]) truncated_while_streaming = state.budget.truncated dropped = implementation._stream_partial(state)["output"] is None - assert dropped is truncated_while_streaming + assert truncated_while_streaming is truncated + assert dropped is truncated def test_many_small_tool_blocks_are_all_kept() -> None: @@ -2474,22 +2250,6 @@ def fallback_stream(iterations: list[dict[str, Any]]) -> list[dict[str, Any]]: ) -def test_beta_stream_records_fallback_model_that_served_the_response( - memory: SimpleNamespace, -) -> None: - a = streamed_beta_span(fallback_stream([DECLINED_ITERATION, SERVED_ITERATION]), memory) - - assert a["gen_ai.response.model"] == "claude-opus-4-8" - - -def test_beta_stream_keeps_requested_model_when_fallback_request_failed( - memory: SimpleNamespace, -) -> None: - a = streamed_beta_span(fallback_stream([DECLINED_ITERATION]), memory) - - assert a["gen_ai.response.model"] == "claude-opus-5" - - ADVISOR_ITERATION: dict[str, Any] = { **DECLINED_ITERATION, "type": "advisor_message", @@ -2544,13 +2304,7 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: output = json.loads(str(a["gen_ai.output.messages"])) assert output == [ - { - "role": "assistant", - "content": [ - {**FALLBACK_BLOCK, "from_": {"model": "claude-opus-5"}}, - {"type": "text", "text": "Hi"}, - ], - } + {"role": "assistant", "content": [FALLBACK_BLOCK, {"type": "text", "text": "Hi"}]} ] assert a["telemetry.dev.capture.truncated"] is True @@ -2558,6 +2312,8 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: @pytest.mark.parametrize( ("iterations", "expected"), [ + ([DECLINED_ITERATION, SERVED_ITERATION], "claude-opus-4-8"), + ([DECLINED_ITERATION], "claude-opus-5"), ([DECLINED_ITERATION, ADVISOR_ITERATION], "claude-opus-5"), ([DECLINED_ITERATION, SERVED_ITERATION, ADVISOR_ITERATION], "claude-opus-4-8"), ( @@ -2565,9 +2321,15 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: "claude-sonnet-5", ), ], - ids=["advisor_without_fallback", "advisor_after_fallback", "last_fallback_wins"], + ids=[ + "fallback_served", + "fallback_failed", + "advisor_without_fallback", + "advisor_after_fallback", + "last_fallback_wins", + ], ) -def test_beta_stream_ignores_advisor_iterations_for_the_served_model( +def test_beta_stream_response_model_follows_the_last_served_fallback( memory: SimpleNamespace, iterations: list[dict[str, Any]], expected: str ) -> None: a = streamed_beta_span(fallback_stream(iterations), memory) @@ -2730,9 +2492,10 @@ def record_output_chunk( assert len(timestamps) == 2 +@pytest.mark.parametrize("provider", ["anthropic", "gcp.vertex_ai"]) @pytest.mark.parametrize("mode", ["wrap", "instrument"]) -async def test_vertex_beta_stream_records_one_span_sync_and_async( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, mode: str +async def test_beta_stream_helper_records_one_span_sync_and_async( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, mode: str, provider: str ) -> None: def sync_handler(request: httpx.Request) -> httpx.Response: return named_sse_response(stream_events()) @@ -2742,28 +2505,32 @@ async def handler(request: httpx.Request) -> httpx.Response: if mode == "instrument": instrument_anthropic() - sync_vertex, async_vertex = provider_clients(monkeypatch, handler, sync_handler)[ - "gcp.vertex_ai" - ] + if provider == "anthropic": + sync_provider, async_provider = sync_client(sync_handler), async_client(handler) + else: + sync_provider, async_provider = provider_clients(monkeypatch, handler, sync_handler)[ + provider + ] if mode == "wrap": - sync_vertex = wrap_anthropic(sync_vertex) - async_vertex = wrap_anthropic(async_vertex) + sync_provider = wrap_anthropic(sync_provider) + async_provider = wrap_anthropic(async_provider) - with sync_vertex.beta.messages.stream( + with sync_provider.beta.messages.stream( model="claude-sonnet-4-6", max_tokens=64, messages=BETA_MESSAGES ) as stream: sync_text = "".join(stream.text_stream) - async with async_vertex.beta.messages.stream( + async with async_provider.beta.messages.stream( model="claude-sonnet-4-6", max_tokens=64, messages=BETA_MESSAGES ) as stream: async_text = "".join([text async for text in stream.text_stream]) - await async_vertex.close() + await async_provider.close() assert sync_text == async_text == "Hello world" spans = memory.span_exporter.get_finished_spans() assert len(spans) == 2 for span in spans: - assert attrs(span)["gen_ai.provider.name"] == "gcp.vertex_ai" + assert attrs(span)["gen_ai.provider.name"] == provider + assert attrs(span)["gen_ai.usage.output_tokens"] == 2 assert json.loads(str(attrs(span)["gen_ai.output.messages"])) == [ {"role": "assistant", "content": [{"type": "text", "text": "Hello world"}]} ] @@ -2781,14 +2548,6 @@ def test_missing_provider_beta_module_is_skipped(monkeypatch: pytest.MonkeyPatch assert all("bedrock" not in cls.__module__ for cls in classes) -def test_global_instrumentation_wraps_async_provider_classes_as_async() -> None: - instrument_anthropic() - for cls in telemetry_dev_anthropic._provider_beta_classes(): # pyright: ignore[reportPrivateUsage] - original = getattr(cls.create, "_telemetry_dev_anthropic_original", None) - assert original is not None - assert inspect.iscoroutinefunction(cls.create) == cls.__name__.startswith("Async") - - def test_beta_stream_compaction_delta_without_start_is_a_compaction_block( memory: SimpleNamespace, ) -> None: @@ -2918,6 +2677,7 @@ def test_oversized_compaction_replacement_drops_stale_block_and_recovers( assert state.budget.truncated is False assert state.budget.bytes_used == expected.bytes_used assert state.blocks[0]["content"] == "final" + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] assert "attributes" not in partial @@ -2968,6 +2728,7 @@ def test_rejected_inherited_encrypted_content_stays_unresolved_until_replaced( assert state.blocks[0] == {"type": "compaction", "content": "recovered"} assert state.unresolved_encrypted_content == {0} + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial( # pyright: ignore[reportPrivateUsage] state, mask_output_when_incomplete=True ) @@ -3241,32 +3002,6 @@ def test_rejected_unseen_compaction_replacement_recovers( assert state.blocks[0]["content"] == "recovered" -def test_rejected_final_compaction_marks_capture_incomplete( - make: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(telemetry_dev_anthropic, "_STREAM_CAPTURE_MAX_BYTES", 300) - make() - state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] - record = telemetry_dev_anthropic._record_stream_event # pyright: ignore[reportPrivateUsage] - - record( - {"type": "content_block_start", "index": 0, "content_block": {"type": "compaction"}}, - state, - ) - record( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "compaction_delta", "content": "x" * 400}, - }, - state, - ) - - partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] - assert partial["attributes"] == {"telemetry.dev.capture.truncated": True} - assert state.budget.truncated is False - - def test_masked_beta_stream_omits_incomplete_retained_output(make: Any) -> None: def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: return value @@ -3492,6 +3227,7 @@ def test_stream_drops_compaction_superseded_by_a_rejected_final_replacement( assert json.loads(str(a["gen_ai.output.messages"])) == [ {"role": "assistant", "content": [{"type": "text", "text": "Hi"}]} ] + assert a["telemetry.dev.capture.truncated"] is True def test_rejected_signature_keeps_thinking_and_drops_the_stale_signature( @@ -3691,41 +3427,6 @@ def test_rejected_compaction_releases_budget_for_later_blocks( ] -def test_compaction_replacement_is_measured_once( - make: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - make() - state = telemetry_dev_anthropic._StreamState() # pyright: ignore[reportPrivateUsage] - record = telemetry_dev_anthropic._record_stream_event # pyright: ignore[reportPrivateUsage] - record( - {"type": "content_block_start", "index": 0, "content_block": {"type": "compaction"}}, - state, - ) - accept_calls = 0 - original_accept = telemetry_dev.CaptureBudget.accept - - def count_accept(self: telemetry_dev.CaptureBudget, value: object) -> bool: - nonlocal accept_calls - accept_calls += 1 - return original_accept(self, value) - - monkeypatch.setattr(telemetry_dev.CaptureBudget, "accept", count_accept) - record( - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "compaction_delta", "content": "summary"}, - }, - state, - ) - - assert accept_calls == 1 - assert state.blocks[0] == { - "type": "compaction", - "content": "summary", - } - - def test_rejected_compactions_after_truncation_leave_no_state( make: Any, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -3820,6 +3521,7 @@ def test_oversized_compaction_replacement_keeps_inherited_encrypted_content_unre assert state.blocks[0] == {"type": "compaction", "content": "recovered"} assert state.unresolved_replacements == set() assert state.unresolved_encrypted_content == {0} + record({"type": "message_stop"}, state) partial = telemetry_dev_anthropic._stream_partial(state) # pyright: ignore[reportPrivateUsage] assert partial["attributes"] == {"telemetry.dev.capture.truncated": True} masked = telemetry_dev_anthropic._stream_partial( # pyright: ignore[reportPrivateUsage] diff --git a/sdks/python-bedrock/tests/test_bedrock.py b/sdks/python-bedrock/tests/test_bedrock.py index 7f3fff1..3b6acac 100644 --- a/sdks/python-bedrock/tests/test_bedrock.py +++ b/sdks/python-bedrock/tests/test_bedrock.py @@ -489,6 +489,7 @@ def _streamed_converse( [ (1_000, False, "東" * 400), (1_000, True, "[redacted]"), + (100, False, "...[truncated]"), (0, False, None), ], ) @@ -512,28 +513,17 @@ def mask(value: Any, _context: Any) -> Any: assert state.budget.truncated is False assert state.blocks[0]["text"] == text assert "telemetry.dev.capture.truncated" not in span.attributes + exported = span.attributes.get("gen_ai.output.messages", "") if expected is None: - assert span.attributes.get("gen_ai.output.messages", "") == "" + assert exported == "" + elif expected == "...[truncated]": + assert isinstance(exported, str) + assert len(exported) <= max_attribute_length + assert exported.endswith(expected) else: assert _json_attr(span, "gen_ai.output.messages")[0]["parts"][0]["content"] == expected -def test_small_configured_cap_cuts_the_exported_stream_output_not_retention( - make: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - memory = make(max_attribute_length=100) - text = "x" * 5_000 - - state, span = _streamed_converse(memory, monkeypatch, _converse_text_stream(text)) - - assert state.blocks[0]["text"] == text - exported = span.attributes["gen_ai.output.messages"] - assert isinstance(exported, str) - assert len(exported) <= 100 - assert exported.endswith("...[truncated]") - assert "telemetry.dev.capture.truncated" not in span.attributes - - @pytest.mark.parametrize("masked", [False, True]) @pytest.mark.parametrize("max_attribute_length", [0, 100, 1024 * 1024]) def test_stream_beyond_the_fixed_bound_is_flagged_and_withheld_from_a_mask( @@ -662,7 +652,6 @@ def test_invoke_model_stream_bounds_content_but_keeps_terminal_metadata( assert state.budget.truncated is True assert state.budget.bytes_used <= state.budget.max_bytes span = memory.span_exporter.get_finished_spans()[0] - assert not hasattr(state, "text") retained_text = _json_attr(span, "gen_ai.output.messages")[0]["parts"][0]["content"] expected_prefix = "".join(f"{index:04d}" + "x" * 96 for index in range(len(state.chunks))) assert retained_text == expected_prefix @@ -691,6 +680,7 @@ def test_unconsumed_stream_finishes_partial( spans = memory.span_exporter.get_finished_spans() assert len(spans) == 1 assert spans[0].attributes["gen_ai.operation.name"] == "chat" + assert spans[0].attributes["telemetry.dev.capture.truncated"] is True @pytest.mark.parametrize("kind", ["absent", "compaction", "at-bound", "bounded"]) @@ -1586,36 +1576,42 @@ def test_invoke_model_stream_titan_counts_finish_and_unrewindable_body( assert list(titan_span.attributes["gen_ai.response.finish_reasons"]) == ["FINISHED"] +@pytest.mark.parametrize( + ("chunks", "content", "usage"), + [ + ( + [ + { + "text": "choice", + "finish_reason": "COMPLETE", + "meta": {"billed_units": {"input_tokens": 8, "output_tokens": 3}}, + } + ], + "choice", + (8, 3), + ), + ( + [ + {"generations": [{"text": "Cohere "}]}, + {"generations": [{"text": "done", "finish_reason": "COMPLETE"}]}, + ], + "Cohere done", + None, + ), + ], + ids=["text", "generations"], +) def test_invoke_model_stream_cohere_generations_finish( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch + memory: SimpleNamespace, + monkeypatch: pytest.MonkeyPatch, + chunks: list[dict[str, Any]], + content: str, + usage: tuple[int, int] | None, ) -> None: client = _client("bedrock-runtime") _stub_api_call( client, - [ - { - "body": FakeEventStream( - [ - { - "chunk": { - "bytes": _body( - { - "text": "choice", - "finish_reason": "COMPLETE", - "meta": { - "billed_units": { - "input_tokens": 8, - "output_tokens": 3, - } - }, - } - ) - } - } - ] - ) - }, - ], + [{"body": FakeEventStream([{"chunk": {"bytes": _body(chunk)}} for chunk in chunks])}], monkeypatch, ) wrap_bedrock(client) @@ -1628,11 +1624,13 @@ def test_invoke_model_stream_cohere_generations_finish( list(cohere_response["body"]) cohere_span = memory.span_exporter.get_finished_spans()[0] - assert _json_attr(cohere_span, "gen_ai.output.messages")[0]["parts"][0]["content"] == "choice" - assert _json_attr(cohere_span, "gen_ai.output.messages")[0]["finish_reason"] == "COMPLETE" - assert cohere_span.attributes["gen_ai.usage.input_tokens"] == 8 - assert cohere_span.attributes["gen_ai.usage.output_tokens"] == 3 + output = _json_attr(cohere_span, "gen_ai.output.messages")[0] + assert output["parts"][0]["content"] == content + assert output["finish_reason"] == "COMPLETE" assert list(cohere_span.attributes["gen_ai.response.finish_reasons"]) == ["COMPLETE"] + if usage is not None: + assert cohere_span.attributes["gen_ai.usage.input_tokens"] == usage[0] + assert cohere_span.attributes["gen_ai.usage.output_tokens"] == usage[1] def test_embeddings_apply_guardrail_and_error( diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index 5173ecb..f045e68 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -400,15 +400,22 @@ def sync_generate_images(**_kwargs: Any) -> dict[str, Any]: def test_media_capture_projects_cyclic_labels(memory: SimpleNamespace) -> None: + client, _transport = client_with_transport([]) + + def generate_images(**_kwargs: Any) -> dict[str, Any]: + return {"generatedImages": []} + + client.models.generate_images = generate_images labels: dict[str, Any] = {"ordinary": "value"} labels["cycle"] = labels - contents, truncated = google_genai_instrumentation._media_contents( - {"config": {"labels": labels}} - ) + wrap_google_genai(client).models.generate_images(model="imagen-4.0", config={"labels": labels}) - assert truncated is False - assert contents == {"labels": {"ordinary": "value", "cycle": None}} + a = attrs(only_span(memory)) + assert json.loads(str(a["gen_ai.input.messages"])) == { + "labels": {"ordinary": "value", "cycle": None} + } + assert "telemetry.dev.capture.truncated" not in a @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) @@ -699,6 +706,7 @@ async def run() -> Any: assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["STOP"] assert a["gen_ai.request.temperature"] == 0.2 assert a["gen_ai.request.top_p"] == 0.9 + assert a["gen_ai.request.top_k"] == 40 assert a["gen_ai.request.max_tokens"] == 64 assert list(cast(Any, a["gen_ai.request.stop_sequences"])) == ["END"] assert a["gen_ai.request.seed"] == 7 @@ -1560,8 +1568,11 @@ async def test_prompt_feedback_text_is_kept_only_when_output_is_captured_unmaske } await generate([response] if stream else response, is_async=is_async, stream=stream) - a = attrs(only_span(memory)) + span = only_span(memory) + a = attrs(span) assert a["google_genai.response.block_reason"] == "SAFETY" + assert "gen_ai.output.messages" not in a + assert span.status.status_code == StatusCode.UNSET prompt_ratings = json.loads(cast(str, a["google_genai.response.prompt_safety_ratings"])) if capture == "on": assert a["google_genai.response.block_reason_message"] == "private block message" @@ -1830,19 +1841,6 @@ def test_api_error_records_error_and_reraises(memory: SimpleNamespace, async_mod assert span.status.status_code == StatusCode.ERROR -def test_blocked_prompt_records_block_attrs_without_output(memory: SimpleNamespace) -> None: - payload = {"promptFeedback": {"blockReason": "SAFETY", "blockReasonMessage": "blocked"}} - client, _transport = client_with_transport([payload]) - wrapped = wrap_google_genai(client) - response = wrapped.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) - assert response.candidates is None - a = attrs(only_span(memory)) - assert a["google_genai.response.block_reason"] == "SAFETY" - assert a["google_genai.response.block_reason_message"] == "blocked" - assert "gen_ai.output.messages" not in a - assert only_span(memory).status.status_code == StatusCode.UNSET - - @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) def test_embed_content_maps_embedding_attrs(memory: SimpleNamespace, async_mode: bool) -> None: client, _transport = client_with_transport([embed_payload()]) @@ -1900,11 +1898,10 @@ def test_chats_emit_spans_with_history(memory: SimpleNamespace) -> None: assert json.loads(str(attrs(span)["gen_ai.input.messages"])) == expected_input -def test_vertex_provider(memory: SimpleNamespace) -> None: - _client, _transport = client_with_transport([happy_payload()]) - wrapped = wrap_google_genai(genai.Client(api_key="test", vertexai=True)) - wrapped._api_client.request = _transport.request - wrapped.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) +def test_global_instrumentation_reports_vertex_provider(memory: SimpleNamespace) -> None: + instrument_google_genai() + client, _transport = client_with_transport([happy_payload()], vertexai=True) + client.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) assert attrs(only_span(memory))["gen_ai.provider.name"] == "gcp.vertex_ai" @@ -2104,26 +2101,12 @@ def generate_videos(_self: Any, **_kwargs: Any) -> dict[str, Any]: @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize("poll_fails", [False, True], ids=["terminal", "exception"]) -def test_uninstrument_claims_callback_while_video_poll_is_blocked( +def test_uninstrument_ends_video_span_while_poll_is_blocked( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, async_mode: bool, poll_fails: bool, ) -> None: - callback_calls = 0 - original_end_once = google_genai_instrumentation._end_once - - def counted_end_once(handle: telemetry_dev.SpanHandle) -> Any: - end = original_end_once(handle) - - def counted_end(**fields: Any) -> None: - nonlocal callback_calls - callback_calls += 1 - end(**fields) - - return counted_end - - monkeypatch.setattr(google_genai_instrumentation, "_end_once", counted_end_once) failure = RuntimeError("blocked poll failed") if async_mode: @@ -2200,7 +2183,6 @@ def get_operation(_self: Any, operation: Any) -> dict[str, Any]: else: assert cast(Any, poll.result(timeout=5))["done"] is True - assert callback_calls == 1 span = only_span(memory) assert span.status.status_code is StatusCode.ERROR assert "google_genai.response.operation_done" not in attrs(span) @@ -2291,12 +2273,12 @@ def test_sync_stream_close_ends_partial_span(memory: SimpleNamespace) -> None: ] -def test_sync_stream_supports_next_send_close_protocol(memory: SimpleNamespace) -> None: +def test_sync_stream_supports_next_and_send_protocol(memory: SimpleNamespace) -> None: client, _transport = client_with_transport([stream_chunks()]) wrapped = wrap_google_genai(client) stream = wrapped.models.generate_content_stream(model="gemini-2.5-flash", contents=USER_CONTENT) assert next(stream).text == "Hello" - assert next(stream).text == " world" + assert stream.send(None).text == " world" next(stream) with pytest.raises(StopIteration): next(stream) @@ -2346,23 +2328,6 @@ async def run() -> None: ] -def test_blocked_prompt_records_prompt_safety_ratings(memory: SimpleNamespace) -> None: - payload = { - "promptFeedback": { - "blockReason": "SAFETY", - "safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "HIGH"}], - } - } - client, _transport = client_with_transport([payload]) - wrapped = wrap_google_genai(client) - wrapped.models.generate_content(model="gemini-2.5-flash", contents=USER_CONTENT) - a = attrs(only_span(memory)) - assert a["google_genai.response.block_reason"] == "SAFETY" - ratings = json.loads(str(a["google_genai.response.prompt_safety_ratings"])) - assert ratings[0]["category"] == "HARM_CATEGORY_HATE_SPEECH" - assert ratings[0]["probability"] == "HIGH" - - def test_mapper_failures_are_fail_open( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/sdks/python-litellm/tests/test_litellm.py b/sdks/python-litellm/tests/test_litellm.py index a591c26..f7b08a1 100644 --- a/sdks/python-litellm/tests/test_litellm.py +++ b/sdks/python-litellm/tests/test_litellm.py @@ -142,25 +142,16 @@ def test_completion_multiple_choices(memory: SimpleNamespace) -> None: assert tuple(cast(Any, a["gen_ai.response.finish_reasons"])) == ("stop", "stop") -def test_completion_metadata_param_maps_to_td_metadata(memory: SimpleNamespace) -> None: - instrument_litellm() - llm.completion( - model="gpt-4o-mini", - messages=CHAT_MESSAGES, - metadata={"tenant": "acme"}, - mock_response="ok", - ) - - assert attrs(only_span(memory))["td.metadata.tenant"] == "acme" - - -def test_completion_litellm_metadata_maps_to_td_metadata(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("argument", ["metadata", "litellm_metadata"]) +def test_completion_metadata_param_maps_to_td_metadata( + memory: SimpleNamespace, argument: str +) -> None: instrument_litellm() llm.completion( model="gpt-4o-mini", messages=CHAT_MESSAGES, - litellm_metadata={"tenant": "acme"}, mock_response="ok", + **{argument: {"tenant": "acme"}}, ) assert attrs(only_span(memory))["td.metadata.tenant"] == "acme" @@ -339,6 +330,7 @@ def test_router_fallback_drops_the_failed_call_history(memory: SimpleNamespace) }, ], fallbacks=[{"primary": ["backup"]}], + num_retries=0, ) router.completion( model="primary", @@ -409,22 +401,6 @@ def test_completion_error_records_error_span(memory: SimpleNamespace) -> None: assert a["gen_ai.provider.name"] == "openai" -def test_completion_timeout_error(memory: SimpleNamespace) -> None: - instrument_litellm() - - with pytest.raises(llm.Timeout): - llm.completion( - model="gpt-4o-mini", - messages=CHAT_MESSAGES, - mock_timeout=True, - timeout=0.01, - ) - - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "Timeout" - - def test_completion_response_mapping_base_exception_escapes( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -841,13 +817,10 @@ def fake_completion(*args: Any, **kwargs: Any) -> Any: instrument_litellm() stream = llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) delivered = list(stream) - budget = stream._budget assert delivered == chunks - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert len(stream._chunks) < len(chunks) a = attrs(only_span(memory)) + assert a["telemetry.dev.capture.truncated"] is True assert a["gen_ai.response.id"] == "bounded-stream" assert a["gen_ai.response.model"] == "gpt-4o-mini" assert tuple(cast(Any, a["gen_ai.response.finish_reasons"])) == ( @@ -1007,7 +980,10 @@ def test_configured_cap_applies_after_the_mask_not_to_stream_retention( redact: bool, expected: str | None, ) -> None: + seen: list[Any] = [] + def mask(value: Any, _context: Any) -> Any: + seen.append(value) if not redact: return value return [{"role": "assistant", "content": "[redacted]"}] @@ -1024,10 +1000,10 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: stream = llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) assert list(stream) == chunks - assert stream._budget.truncated is False - assert len(stream._chunks) == len(chunks) a = attrs(only_span(memory)) assert "telemetry.dev.capture.truncated" not in a + if redact: + assert [{"role": "assistant", "content": text}] in seen if expected is None: assert a.get("gen_ai.output.messages", "") == "" else: @@ -1071,18 +1047,17 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: stream = llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) assert list(stream) == chunks - budget = stream._budget - assert budget.truncated is True - assert budget.max_bytes == telemetry_dev_litellm._STREAM_CAPTURE_MAX_BYTES # pyright: ignore[reportPrivateUsage] - assert 0 < len(stream._chunks) < len(chunks) a = attrs(only_span(memory)) assert a["telemetry.dev.capture.truncated"] is True if masked: assert "gen_ai.output.messages" not in a assert not any("x" * 100 in value for value in seen) - elif max_attribute_length > budget.max_bytes: - retained = "x" * 100 * len(stream._chunks) - assert json_attr(a["gen_ai.output.messages"])[0]["content"] == retained + elif max_attribute_length == 1024 * 1024: + # The 48 KiB retention bound, not the 1 MiB cap, cuts the stream at a chunk boundary. + content = json_attr(a["gen_ai.output.messages"])[0]["content"] + assert set(content) == {"x"} + assert 0 < len(content) <= 48 * 1024 + assert len(content) % 100 == 0 else: assert len(str(a.get("gen_ai.output.messages", ""))) <= max_attribute_length @@ -1139,9 +1114,7 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: if extra_indices == 0: assert finish_reasons == ("length", "content_filter") return - max_items = telemetry_dev_litellm._STREAM_CAPTURE_MAX_ITEMS # pyright: ignore[reportPrivateUsage] - assert len(stream._finish_reasons) <= max_items - assert len(finish_reasons) <= max_items + assert len(finish_reasons) <= 1_000 assert a["telemetry.dev.capture.truncated"] is True assert "gen_ai.output.messages" not in a assert not any('"first"' in value for value in seen) @@ -1182,7 +1155,6 @@ def fake_completion(*args: Any, **kwargs: Any) -> Iterator[Any]: a = attrs(only_span(memory)) finish_reasons = tuple(cast(Any, a["gen_ai.response.finish_reasons"])) assert finish_reasons == ("r" * max_length, "stop") - assert all(len(reason) <= max_length for reason in stream._finish_reasons.values()) assert (a.get("telemetry.dev.capture.truncated") is True) is spent @@ -1377,80 +1349,6 @@ def record_output_chunk( assert timestamps -async def test_async_streaming_bounds_retained_chunks_without_dropping_output( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch -) -> None: - chunks = [ - SimpleNamespace( - id=None, - model=None, - choices=[ - SimpleNamespace( - index=0, - delta=SimpleNamespace(content="x" * 100), - finish_reason=None, - ) - ], - ) - for _ in range(1100) - ] - chunks.append( - SimpleNamespace( - id="bounded-async-stream", - model="gpt-4o-mini", - choices=[ - SimpleNamespace( - index=0, - delta=SimpleNamespace(content=None), - finish_reason="stop", - ) - ], - usage=SimpleNamespace( - prompt_tokens=13, - completion_tokens=8, - total_tokens=21, - ), - ) - ) - - class AsyncChunks: - def __init__(self) -> None: - self._iterator = iter(chunks) - - def __aiter__(self) -> AsyncChunks: - return self - - async def __anext__(self) -> Any: - try: - return next(self._iterator) - except StopIteration as error: - raise StopAsyncIteration from error - - async def aclose(self) -> None: - return None - - async def fake_acompletion(*args: Any, **kwargs: Any) -> Any: - return AsyncChunks() - - monkeypatch.setattr(litellm, "acompletion", fake_acompletion) - instrument_litellm() - stream = await llm.acompletion(model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True) - delivered = [chunk async for chunk in stream] - budget = stream._budget - - assert delivered == chunks - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert len(stream._chunks) < len(chunks) - a = attrs(only_span(memory)) - assert a["gen_ai.response.id"] == "bounded-async-stream" - assert a["gen_ai.response.model"] == "gpt-4o-mini" - assert tuple(cast(Any, a["gen_ai.response.finish_reasons"])) == ("stop",) - assert a["gen_ai.usage.input_tokens"] == 13 - assert a["gen_ai.usage.output_tokens"] == 8 - assert a["gen_ai.usage.total_tokens"] == 21 - - async def test_async_positional_stream_argument_is_instrumented(memory: SimpleNamespace) -> None: instrument_litellm() stream = await llm.acompletion( @@ -1914,6 +1812,7 @@ def __len__(self) -> int: list(llm.responses(input=UnreadableInput(), model="gpt-4.1-mini", stream=True)) + assert only_span(env).name == "chat gpt-4.1-mini" assert "gen_ai.input.messages" not in attrs(only_span(env)) @@ -2251,10 +2150,8 @@ def test_responses_stream_small_cap_keeps_retention_and_cuts_after_the_mask( monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() - stream = llm.responses(input="bounded", model="gpt-4.1-mini", stream=True) - list(stream) + list(llm.responses(input="bounded", model="gpt-4.1-mini", stream=True)) - assert stream._budget.truncated is False a = attrs(only_span(memory)) assert "telemetry.dev.capture.truncated" not in a exported = str(a["gen_ai.output.messages"]) @@ -2368,6 +2265,7 @@ def test_responses_stream_rejected_annotation_preserves_text_and_marks_truncated annotation_index=0, annotation={"type": "url_citation", "url": "https://example.com/" + "c" * 7_000}, ), + completed_without_output(), ] monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() @@ -2396,6 +2294,7 @@ def test_responses_stream_rejects_sparse_annotation_indexes( annotation_index=1_000_000_000, annotation={"type": "url_citation", "url": "https://example.com"}, ), + completed_without_output(), ] monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() @@ -2492,45 +2391,75 @@ def test_responses_stream_merges_seeded_textual_event_families( assert output[2]["code"] == "print(1)" -def test_bounded_responses_conversion_stops_without_model_dump(memory: SimpleNamespace) -> None: - class Bomb: - def __init__(self) -> None: - self.output = [{"type": "output_text", "text": "x" * 100_000}] - self.status = "completed" +class _Opaque: + __slots__ = () - def model_dump(self, **kwargs: Any) -> Any: - raise AssertionError("model_dump must not run") - fields = vars(telemetry_dev_litellm)["_responses_response"](Bomb()) - assert fields["output"] is None - assert fields["attributes"]["telemetry.dev.capture.truncated"] is True +class _ModelDumpBomb: + """A response whose model_dump must not run; its output exceeds the 64 KiB non-stream bound.""" + def __init__(self) -> None: + self.output = [{"type": "output_text", "text": "x" * 100_000}] + self.status = "completed" -def test_bounded_responses_conversion_repeats_shared_acyclic_values( - memory: SimpleNamespace, -) -> None: - shared = {"text": "same"} + def model_dump(self, **kwargs: Any) -> Any: + raise AssertionError("model_dump must not run") - captured, budget = vars(telemetry_dev_litellm)["_bounded_responses_native"]([shared, shared]) - assert captured == [{"text": "same"}, {"text": "same"}] - assert budget.truncated is False +_SHARED_ITEM = {"text": "same"} -@pytest.mark.parametrize("value", ["\x01" * 1000, {"\x01" * 1000: 1}], ids=["string", "key"]) -def test_bounded_responses_conversion_charges_strings_at_their_escaped_size( - memory: SimpleNamespace, value: object +@pytest.mark.parametrize( + ("response", "expected"), + [ + pytest.param( + SimpleNamespace(status="completed", output=[_SHARED_ITEM, _SHARED_ITEM]), + [{"text": "same"}, {"text": "same"}], + id="shared-acyclic-values-repeat", + ), + pytest.param( + SimpleNamespace(status="completed", output=[{"opaque": _Opaque()}]), + [{"opaque": None}], + id="opaque-value-is-null", + ), + # Control characters cost six bytes each once escaped, one as raw UTF-8, so 10,000 of + # them fit the 64 KiB bound and 11,000 do not. + pytest.param( + SimpleNamespace(status="completed", output=[{"payload": "\x01" * 10_000}]), + [{"payload": "\x01" * 10_000}], + id="escaped-string-within-bound", + ), + pytest.param( + SimpleNamespace(status="completed", output=[{"payload": "\x01" * 11_000}]), + None, + id="escaped-string-over-bound", + ), + pytest.param( + SimpleNamespace(status="completed", output=[{"\x01" * 11_000: 1}]), + None, + id="escaped-key-over-bound", + ), + pytest.param(_ModelDumpBomb(), None, id="oversized-without-model-dump"), + ], +) +def test_responses_output_conversion_is_bounded_and_json_shaped( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, response: Any, expected: Any ) -> None: - # Control characters cost six bytes each once escaped, one as raw UTF-8. - convert = vars(telemetry_dev_litellm)["_bounded_responses_native"] + def fake_responses(*args: Any, **kwargs: Any) -> Any: + return response - rejected, small = convert({"payload": value}, budget=telemetry_dev.CaptureBudget(2048)) - kept, large = convert({"payload": value}, budget=telemetry_dev.CaptureBudget(8192)) + monkeypatch.setattr(litellm, "responses", fake_responses) + instrument_litellm() + + llm.responses(input="hi", model="gpt-4.1-mini") - assert rejected is None - assert small.truncated is True - assert kept == {"payload": value} - assert large.bytes_used >= len(json.dumps(kept)) + a = attrs(only_span(memory)) + if expected is None: + assert a["telemetry.dev.capture.truncated"] is True + assert "gen_ai.output.messages" not in a + else: + assert json_attr(a["gen_ai.output.messages"]) == expected + assert "telemetry.dev.capture.truncated" not in a def test_responses_stream_charges_appended_deltas_at_their_escaped_size( @@ -2554,19 +2483,6 @@ def test_responses_stream_charges_appended_deltas_at_their_escaped_size( assert json_attr(a["gen_ai.output.messages"]) == [{"type": "output_text", "text": "kept"}] -def test_bounded_responses_conversion_normalizes_opaque_values(memory: SimpleNamespace) -> None: - class Opaque: - __slots__ = () - - captured, budget = vars(telemetry_dev_litellm)["_bounded_responses_native"]( - {"opaque": Opaque()} - ) - - assert captured == {"opaque": None} - assert json.dumps(captured) == '{"opaque": null}' - assert budget.truncated is False - - def test_responses_stream_terminal_item_preserves_truncation_overflow( memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -2593,6 +2509,7 @@ def test_responses_stream_terminal_item_preserves_truncation_overflow( content_index=0, delta="fits", ), + completed_without_output(), ] ) monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) @@ -2630,6 +2547,7 @@ def events() -> list[SimpleNamespace]: ) for index in range(8) ], + completed_without_output(), ] sync_inner = ResponsesStream(events()) @@ -2804,16 +2722,19 @@ def test_responses_stream_charges_one_character_fragments_against_item_budget( ) for _ in range(2_000) ] + events.append(completed_without_output()) monkeypatch.setattr(litellm, "responses", fake_responses_stream(events)) instrument_litellm() stream = llm.responses(input="bounded fragments", model="gpt-4.1-mini", stream=True) assert list(stream) == events - fragments = stream._partial_content[("index", 0)][("index", 0)]["_fragments"] - assert len(fragments) <= stream._budget.max_items - assert stream._budget.items_used <= stream._budget.max_items - assert attrs(only_span(memory))["telemetry.dev.capture.truncated"] is True + a = attrs(only_span(memory)) + text = json_attr(a["gen_ai.output.messages"])[0]["text"] + # 2,000 one-character deltas fit the byte bound; only the 1,000-item bound cuts them. + assert set(text) == {"x"} + assert 0 < len(text) <= 1_000 + assert a["telemetry.dev.capture.truncated"] is True async def test_responses_stream_surrogate_deltas_fail_closed_and_finish( @@ -3395,10 +3316,3 @@ def test_provider_attribution_non_openai(memory: SimpleNamespace) -> None: assert azure_flag_attrs["gen_ai.request.model"] == "my-azure-deploy" assert deployment_attrs["gen_ai.provider.name"] == "azure.ai.openai" assert deployment_attrs["gen_ai.request.model"] == "deployment-from-kwarg" - - -def test_cost_maps_to_cost_usd_when_available(memory: SimpleNamespace) -> None: - instrument_litellm() - llm.completion(model="gpt-4o-mini", messages=CHAT_MESSAGES, mock_response="priced") - - assert number_attr(attrs(only_span(memory))["gen_ai.usage.cost"]) > 0 diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index 5b74984..9a13a82 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -524,11 +524,8 @@ def _media_response(response: Any) -> dict[str, Any]: } -def _text_media_response(response: Any, *, capture_output: bool | None = None) -> dict[str, Any]: +def _text_media_response(response: Any, *, capture_output: bool = False) -> dict[str, Any]: fields = _media_response(response) - if capture_output is None: - client = telemetry_dev.get_client() - capture_output = client is not None and client.capture_output if capture_output: text = _string(_field(response, "text")) if text is not None: @@ -1458,26 +1455,12 @@ async def _aclose_hook(*args: Any, **kwargs: Any) -> Any: def _chat_span_capture_policy(handle: telemetry_dev.SpanHandle) -> tuple[bool, bool]: - compatible = cast(Any, handle) - client = getattr(compatible, "_client", None) - state = getattr(compatible, "_state", None) - capture_output = getattr(compatible, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(state, "capture_output", None) - if not isinstance(capture_output, bool): - capture_output = getattr(client, "capture_output", False) - capture_masked = getattr(compatible, "capture_masked", None) - if not isinstance(capture_masked, bool): - capture_masked = getattr(client, "mask", _OMIT) is not None if client is not None else True - return capture_output, capture_masked + return handle.capture_output, handle.capture_masked def _chat_span_reporter( handle: telemetry_dev.SpanHandle, ) -> Callable[[BaseException], None]: - compatible = cast(Any, handle) - reporter = getattr(compatible, "report_error", None) - client = getattr(compatible, "_client", None) reported = False def report(cause: BaseException) -> None: @@ -1486,12 +1469,7 @@ def report(cause: BaseException) -> None: return reported = True try: - if callable(reporter): - reporter(cause) - return - client_report = getattr(client, "report", None) - if callable(client_report): - client_report("provider instrumentation failed", cause) + handle.report_error(cause) except Exception: return @@ -1668,9 +1646,7 @@ def _record(self, chunk: Any, received_at: float) -> None: self._unterminated_choices, ) if update["has_output"]: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( @@ -1838,9 +1814,7 @@ def _record(self, chunk: Any, received_at: float) -> None: self._unterminated_choices, ) if update["has_output"]: - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( @@ -1955,9 +1929,7 @@ def __getattr__(self, name: str) -> Any: def _record(self, event: Any, received_at: float) -> None: if _response_event_has_output(event): - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( @@ -2108,9 +2080,7 @@ def __getattr__(self, name: str) -> Any: def _record(self, event: Any, received_at: float) -> None: if _response_event_has_output(event): - record_output_chunk = getattr(self._handle, "record_output_chunk", None) - if callable(record_output_chunk): - record_output_chunk(received_at * 1000) + self._handle.record_output_chunk(received_at * 1000) if not self._saw_first: self._saw_first = True self._handle.update( diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index 86a6ee6..d2b4b40 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -12,8 +12,13 @@ import pytest import telemetry_dev from openai import AsyncOpenAI, OpenAI +from openai.resources.audio.speech import AsyncSpeech, Speech +from openai.resources.audio.transcriptions import AsyncTranscriptions, Transcriptions +from openai.resources.audio.translations import AsyncTranslations, Translations +from openai.resources.batches import AsyncBatches, Batches from openai.resources.chat.completions.completions import AsyncCompletions, Completions from openai.resources.embeddings import AsyncEmbeddings, Embeddings +from openai.resources.images import AsyncImages, Images from openai.resources.responses.responses import AsyncResponses, Responses from openai.types.chat.chat_completion_chunk import ChoiceDelta from opentelemetry.sdk.trace import ReadableSpan @@ -150,6 +155,30 @@ async def __aiter__(self) -> AsyncIterator[bytes]: raise httpx.ReadError(self._message) +class CloseFailingSyncByteStream(httpx.SyncByteStream): + def __init__(self, body: bytes, error: Exception) -> None: + self._body = body + self._error = error + + def __iter__(self) -> Iterator[bytes]: + yield self._body + + def close(self) -> None: + raise self._error + + +class CloseFailingAsyncByteStream(httpx.AsyncByteStream): + def __init__(self, body: bytes, error: Exception) -> None: + self._body = body + self._error = error + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._body + + async def aclose(self) -> None: + raise self._error + + class UnencodableText(str): def encode(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: raise AssertionError("capture must not encode the complete transcript") @@ -177,6 +206,7 @@ def sync_client(handler: SyncHandler) -> OpenAI: return OpenAI( api_key="test", base_url="https://api.test/v1", + max_retries=0, http_client=httpx.Client(transport=httpx.MockTransport(handler)), ) @@ -185,6 +215,7 @@ def async_client(handler: AsyncHandler) -> AsyncOpenAI: return AsyncOpenAI( api_key="test", base_url="https://api.test/v1", + max_retries=0, http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), ) @@ -465,20 +496,24 @@ def test_chat_capture_recursively_omits_binary_media(memory: SimpleNamespace) -> assert "spoken reply" in captured -def test_disabled_responses_capture_does_not_convert_hostile_values(make: Any) -> None: - class Hostile: - def model_dump(self, **_kwargs: Any) -> Any: - raise AssertionError("capture-disabled payload was traversed") +def test_disabled_responses_capture_does_not_read_values(make: Any) -> None: + reads: list[str] = [] + + class Counting(dict[str, Any]): + def items(self) -> Any: + reads.append("items") + return super().items() make(capture_input=False, capture_output=False) responses_request = vars(telemetry_dev_openai)["_responses_request"] responses_response = vars(telemetry_dev_openai)["_responses_response"] - _, request = responses_request({"model": "gpt-4.1", "input": Hostile()}) - response = responses_response({"status": "completed", "output": Hostile()}) + _, request = responses_request({"model": "gpt-4.1", "input": Counting(text="x")}) + response = responses_response({"status": "completed", "output": Counting(text="y")}) assert request["input"] is None assert response["output"] is None + assert reads == [] def test_streamed_image_edit_maps_aggregate_output_tokens_to_image(memory: SimpleNamespace) -> None: @@ -535,59 +570,85 @@ def test_image_stream_is_flagged_only_without_a_terminal_event( assert a["telemetry.dev.capture.truncated"] is True -def test_media_stream_preserves_sync_context_manager_cleanup(memory: SimpleNamespace) -> None: - client = wrap_openai( - sync_client( - lambda _request: sse_response( - [ - { - "type": "image_edit.completed", - "usage": {"input_tokens": 4, "output_tokens": 9, "total_tokens": 13}, - } - ] - ) - ) - ) - - with cast(Any, client.images.edit)( - image=b"image", - prompt="Add telemetry", - model="gpt-image-1", - stream=True, - ) as stream: - assert next(stream).type == "image_edit.completed" - - assert len(memory.span_exporter.get_finished_spans()) == 1 - - -async def test_media_stream_preserves_async_context_manager_cleanup( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("exit_kind", ["clean", "body_error", "delegated_error"]) +async def test_media_stream_context_exit_ends_the_span_once( + memory: SimpleNamespace, async_mode: bool, exit_kind: str ) -> None: - async def handler(_request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "type": "image_edit.completed", - "usage": {"input_tokens": 4, "output_tokens": 9, "total_tokens": 13}, - } - ] - ) + # A non-terminal event leaves the span open, so only the context exit can end it. + event = {"type": "image_edit.partial_image", "b64_json": "QQ==", "partial_image_index": 0} + error = { + "clean": None, + "body_error": ValueError("body failed"), + "delegated_error": RuntimeError("exit failed"), + }[exit_kind] - client = wrap_openai(async_client(handler)) - async with await cast(Any, client.images.edit)( - image=b"image", - prompt="Add telemetry", - model="gpt-image-1", - stream=True, - ) as stream: - assert (await stream.__anext__()).type == "image_edit.completed" - await client.close() + def response() -> httpx.Response: + if not isinstance(error, RuntimeError): + return sse_response([event]) + body = f"data: {json.dumps(event)}\n\n".encode() + stream = ( + CloseFailingAsyncByteStream(body, error) + if async_mode + else CloseFailingSyncByteStream(body, error) + ) + return httpx.Response(200, stream=stream, headers={"Content-Type": "text/event-stream"}) + + async def read_one_event() -> Any: + edit: dict[str, Any] = { + "image": b"image", + "prompt": "Add telemetry", + "model": "gpt-image-1", + "stream": True, + } + if async_mode: - assert len(memory.span_exporter.get_finished_spans()) == 1 + async def handler(_request: httpx.Request) -> httpx.Response: + return response() + + async with await cast(Any, wrap_openai(async_client(handler)).images.edit)( + **edit + ) as stream: + assert (await stream.__anext__()).type == "image_edit.partial_image" + if isinstance(error, ValueError): + raise error + return stream + with cast(Any, wrap_openai(sync_client(lambda _request: response())).images.edit)( + **edit + ) as stream: + assert next(stream).type == "image_edit.partial_image" + if isinstance(error, ValueError): + raise error + return stream + + if error is None: + stream = await read_one_event() + span = only_span(memory) + assert span.status.status_code == StatusCode.UNSET + assert attrs(span)["telemetry.dev.capture.truncated"] is True + assert stream.response.is_closed + else: + with pytest.raises(type(error)) as exc_info: + await read_one_event() + assert exc_info.value is error + span = only_span(memory) + assert span.status.status_code == StatusCode.ERROR + assert [dict(e.attributes or {})["exception.message"] for e in span.events] == [str(error)] -def test_streaming_transcription_records_terminal_text_and_usage(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_streaming_transcription_records_terminal_text_and_usage( + memory: SimpleNamespace, async_mode: bool +) -> None: events = [ + { + "type": "transcript.text.segment", + "id": "segment-1", + "start": 0, + "end": 1, + "speaker": "A", + "text": "partial", + }, {"type": "transcript.text.delta", "delta": "partial"}, { "type": "transcript.text.done", @@ -595,13 +656,26 @@ def test_streaming_transcription_records_terminal_text_and_usage(memory: SimpleN "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, }, ] - client = wrap_openai(sync_client(lambda _request: sse_response(events))) - stream = cast(Any, client.audio.transcriptions.create)( - model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True - ) + if async_mode: - assert [event.type for event in stream] == [event["type"] for event in events] + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True + ) + received = [event.type async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True + ) + received = [event.type for event in stream] + + assert received == [event["type"] for event in events] span = only_span(memory) assert span.name == "transcription gpt-4o-transcribe" a = attrs(span) @@ -685,40 +759,6 @@ def test_streaming_transcription_terminal_text_replaces_truncated_deltas( assert "telemetry.dev.capture.truncated" not in a -async def test_async_streaming_transcription_records_terminal_text_and_usage( - memory: SimpleNamespace, -) -> None: - events = [ - { - "type": "transcript.text.segment", - "id": "segment-1", - "start": 0, - "end": 1, - "speaker": "A", - "text": "partial", - }, - { - "type": "transcript.text.done", - "text": "complete transcript", - "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, - }, - ] - - async def handler(_request: httpx.Request) -> httpx.Response: - return sse_response(events) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.audio.transcriptions.create)( - model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=True - ) - assert [event.type async for event in stream] == [event["type"] for event in events] - await client.close() - - a = attrs(only_span(memory)) - assert a["gen_ai.output.messages"] == "complete transcript" - assert a["gen_ai.usage.text.output_tokens"] == 3 - - @pytest.mark.parametrize("async_mode", [False, True]) async def test_interrupted_transcription_stream_records_bounded_partial_text( memory: SimpleNamespace, async_mode: bool @@ -763,7 +803,10 @@ def mask(value: Any, _context: Any) -> Any: memory = make(max_attribute_length=1_000, mask=mask) transcript = "x" * 1_500 - client = wrap_openai(sync_client(lambda _request: json_response({"text": transcript}))) + usage = {"type": "tokens", "input_tokens": 4, "output_tokens": 7, "total_tokens": 11} + client = wrap_openai( + sync_client(lambda _request: json_response({"text": transcript, "usage": usage})) + ) cast(Any, client.audio.transcriptions.create)( model="gpt-4o-transcribe", file=("audio.wav", b"audio") @@ -774,6 +817,7 @@ def mask(value: Any, _context: Any) -> Any: assert len(str(a["gen_ai.output.messages"])) == 1_000 assert str(a["gen_ai.output.messages"]).endswith("...[truncated]") assert "telemetry.dev.capture.truncated" not in a + assert a["gen_ai.usage.text.output_tokens"] == 7 @pytest.mark.parametrize( @@ -993,88 +1037,47 @@ def test_terminal_transcription_capture_is_incrementally_bounded( assert fields["attributes"]["telemetry.dev.capture.truncated"] is True -@pytest.mark.parametrize("capture_output", [False, True]) -def test_text_media_response_uses_the_policy_that_created_its_span( - monkeypatch: pytest.MonkeyPatch, capture_output: bool -) -> None: - ended: list[dict[str, Any]] = [] - reported: list[BaseException] = [] - - def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: - return value - - def end(**fields: Any) -> None: - ended.append(fields) - - handle = SimpleNamespace( - _client=SimpleNamespace(capture_output=capture_output, mask=mask), - _state=SimpleNamespace(capture_output=capture_output), - report_error=reported.append, - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(capture_output=not capture_output, mask=None), - ) - implementation = cast(Any, telemetry_dev_openai) - - implementation._end_mapped_response( - handle, - end, - implementation._text_media_response, - SimpleNamespace(text="sensitive transcript"), - ) - - assert reported == [] - assert ended[-1].get("output") == ("sensitive transcript" if capture_output else None) - assert "attributes" not in ended[-1] - - +@pytest.mark.parametrize("streaming", [False, True]) @pytest.mark.parametrize("async_mode", [False, True]) -async def test_transcription_stream_uses_the_policy_that_created_its_span( - monkeypatch: pytest.MonkeyPatch, async_mode: bool +async def test_transcription_capture_uses_the_policy_that_created_its_span( + make: Any, monkeypatch: pytest.MonkeyPatch, streaming: bool, async_mode: bool ) -> None: - ended: list[dict[str, Any]] = [] - reported: list[BaseException] = [] - event = SimpleNamespace(type="transcript.text.done", text="sensitive transcript") - - def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: - return value + memory = make(capture_output=True) - def end(**fields: Any) -> None: - ended.append(fields) - - handle = SimpleNamespace( - _client=SimpleNamespace(capture_output=True, mask=mask), - _state=SimpleNamespace(capture_output=True), - end=end, - report_error=reported.append, - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(capture_output=False, mask=None), - ) - implementation = cast(Any, telemetry_dev_openai) + def response() -> httpx.Response: + # The span has started by now; a later global policy must not apply to it. + monkeypatch.setattr( + telemetry_dev, + "get_client", + lambda: SimpleNamespace(capture_output=False, capture_input=False, mask=None), + ) + if streaming: + return sse_response([{"type": "transcript.text.done", "text": "sensitive transcript"}]) + return json_response({"text": "sensitive transcript"}) if async_mode: - async def source() -> AsyncIterator[Any]: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return response() - stream = implementation._InstrumentedAsyncMediaStream( - source(), handle, implementation._text_media_response + client = wrap_openai(async_client(handler)) + result = await cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=streaming ) - assert await stream.__anext__() is event + if streaming: + _ = [event async for event in result] + await client.close() else: - stream = implementation._InstrumentedMediaStream( - iter([event]), handle, implementation._text_media_response + client = wrap_openai(sync_client(lambda _request: response())) + result = cast(Any, client.audio.transcriptions.create)( + model="gpt-4o-transcribe", file=("audio.wav", b"audio"), stream=streaming ) - assert next(stream) is event + if streaming: + list(result) - assert reported == [] - assert ended[-1]["output"] == "sensitive transcript" - assert "attributes" not in ended[-1] + a = attrs(only_span(memory)) + assert a["gen_ai.output.messages"] == "sensitive transcript" + assert "telemetry.dev.capture.truncated" not in a @pytest.mark.parametrize("async_mode", [False, True]) @@ -1130,35 +1133,29 @@ async def __anext__(self) -> Any: async def test_failed_media_stream_events_record_errors( memory: SimpleNamespace, async_mode: bool ) -> None: - event = SimpleNamespace( - type="image_generation.failed", - error=SimpleNamespace(code="provider_error", message="provider failed"), - ) - handle = telemetry_dev.start_span("image failed", type="generation") + # The SDK raises APIError for an event with an `error` object, so a failed event + # reaches the wrapper only with top-level code and message. + events = [ + {"type": "image_generation.failed", "code": "provider_error", "message": "provider failed"} + ] if async_mode: - class Inner: - def __init__(self) -> None: - self._remaining = True - - def __aiter__(self) -> Inner: - return self - - async def __anext__(self) -> Any: - if not self._remaining: - raise StopAsyncIteration - self._remaining = False - return event + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) - stream_type = vars(telemetry_dev_openai)["_InstrumentedAsyncMediaStream"] - stream = stream_type(Inner(), handle, vars(telemetry_dev_openai)["_media_response"]) - assert await stream.__anext__() is event + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.images.generate)( + model="gpt-image-1", prompt="otter", stream=True + ) + received = [event.type async for event in stream] + await client.close() else: - stream_type = vars(telemetry_dev_openai)["_InstrumentedMediaStream"] - stream = stream_type(iter([event]), handle, vars(telemetry_dev_openai)["_media_response"]) - assert next(stream) is event + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.images.generate)(model="gpt-image-1", prompt="otter", stream=True) + received = [event.type for event in stream] + assert received == ["image_generation.failed"] span = only_span(memory) assert span.status.status_code == StatusCode.ERROR assert attrs(span)["error.type"] == "RuntimeError" @@ -1176,79 +1173,19 @@ class Event: def text(self) -> str: raise AssertionError("capture-disabled output was read") - class Handle: - capture_output = False - capture_masked = False - max_attribute_length = 5 - - def end(self, **_fields: Any) -> None: - pass - - make(capture_output=False) - stream_type = vars(telemetry_dev_openai)["_InstrumentedMediaStream"] - stream = stream_type( - iter([Event()]), cast(Any, Handle()), vars(telemetry_dev_openai)["_text_media_response"] - ) - - assert next(stream).type == "transcript.text.done" - - -@pytest.mark.parametrize("delegated", [False, True]) -def test_media_stream_context_exit_records_body_and_delegated_errors(delegated: bool) -> None: errors: list[BaseException] = [] - - class Handle: - def end(self, **fields: Any) -> None: - errors.append(fields["error"]) - - class Inner: - def __exit__(self, _type: Any, _exc: Any, _tb: Any) -> bool: - if delegated: - raise RuntimeError("exit failed") - return False - + memory = make(capture_output=False, on_error=errors.append) + handle = telemetry_dev.start_span("transcription gpt-4o-transcribe", type="generation") stream_type = vars(telemetry_dev_openai)["_InstrumentedMediaStream"] stream = stream_type( - Inner(), cast(Any, Handle()), vars(telemetry_dev_openai)["_media_response"] + iter([Event()]), handle, vars(telemetry_dev_openai)["_text_media_response"] ) - body_error = ValueError("body failed") - if delegated: - with pytest.raises(RuntimeError, match="exit failed"): - stream.__exit__(None, None, None) - assert str(errors[0]) == "exit failed" - else: - assert stream.__exit__(ValueError, body_error, None) is False - assert errors == [body_error] - - -@pytest.mark.parametrize("delegated", [False, True]) -async def test_async_media_stream_context_exit_records_body_and_delegated_errors( - delegated: bool, -) -> None: - errors: list[BaseException] = [] - - class Handle: - def end(self, **fields: Any) -> None: - errors.append(fields["error"]) - - class Inner: - async def __aexit__(self, _type: Any, _exc: Any, _tb: Any) -> bool: - if delegated: - raise RuntimeError("exit failed") - return False - stream_type = vars(telemetry_dev_openai)["_InstrumentedAsyncMediaStream"] - stream = stream_type( - Inner(), cast(Any, Handle()), vars(telemetry_dev_openai)["_media_response"] - ) - body_error = ValueError("body failed") - if delegated: - with pytest.raises(RuntimeError, match="exit failed"): - await stream.__aexit__(None, None, None) - assert str(errors[0]) == "exit failed" - else: - assert await stream.__aexit__(ValueError, body_error, None) is False - assert errors == [body_error] + assert next(stream).type == "transcript.text.done" + assert errors == [] + a = attrs(only_span(memory)) + assert "telemetry.dev.capture.truncated" not in a + assert a["gen_ai.usage.text.output_tokens"] == 2 def test_chat_completion_maps_native_messages_usage_finish_provider_and_sampling( @@ -1442,7 +1379,7 @@ def handler(request: httpx.Request) -> httpx.Response: { "index": 1, "message": {"role": "assistant", "content": "second"}, - "finish_reason": "stop", + "finish_reason": "length", }, ] ) @@ -1499,6 +1436,7 @@ def handler(request: httpx.Request) -> httpx.Response: {"role": "assistant", "content": "first"}, {"role": "assistant", "content": "second"}, ] + assert attrs(choices_span)["gen_ai.response.finish_reasons"] == ("stop", "length") def test_chat_streaming_injects_usage_when_opted_in_and_filters_synthetic_chunk( @@ -1705,54 +1643,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: wrapped.close() assert len(received) == len(events) assert len(calls) == 6 - assert memory.metric_reader.get_metrics_data() is not None - - -def test_chat_streaming_reconstructs_legacy_function_call() -> None: - implementation = cast(Any, telemetry_dev_openai) - states: dict[int, Any] = {} - output_budget = implementation._chat_capture_budget(reserve_output_list=True) - finish_reason_budget = implementation._chat_capture_budget() - events = [ - { - "choices": [ - { - "index": 0, - "delta": {"function_call": {"name": "lookup", "arguments": '{"city":'}}, - } - ] - }, - { - "choices": [ - { - "index": 0, - "delta": {"function_call": {"arguments": '"Paris"}'}}, - "finish_reason": "stop", - } - ] - }, - ] - - for event in events: - implementation._record_chat_chunk( - event, - states, - {}, - {}, - set(), - output_budget, - finish_reason_budget, - True, - ) - - assert implementation._chat_output(states) == [ - { - "role": "assistant", - "content": None, - "function_call": {"name": "lookup", "arguments": '{"city":"Paris"}'}, - } - ] - assert output_budget.truncated is False @pytest.mark.parametrize( @@ -1930,82 +1820,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: assert only_span(memory).status.status_code == StatusCode.UNSET -@pytest.mark.parametrize( - "stream_kind", ["sync-chat", "async-chat", "sync-responses", "async-responses"] -) -async def test_streaming_preserves_tracing_without_output_chunk_support( - memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, stream_kind: str -) -> None: - monkeypatch.delattr(telemetry_dev.SpanHandle, "record_output_chunk") - - if stream_kind == "sync-chat": - - def sync_chat_handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) - - client = wrap_openai(sync_client(sync_chat_handler)) - received = list( - cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - ) - client.close() - elif stream_kind == "async-chat": - - async def async_chat_handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) - - client = wrap_openai(async_client(async_chat_handler)) - stream = await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - received = [chunk async for chunk in stream] - await client.close() - else: - delta = { - "type": "response.output_text.delta", - "sequence_number": 0, - "item_id": "item_old_core", - "output_index": 0, - "content_index": 0, - "delta": "Still traced.", - } - response = { - "type": "response.completed", - "sequence_number": 1, - "response": response_payload(response_id="resp_old_core", text="Still traced."), - } - events = [("response.output_text.delta", delta), ("response.completed", response)] - if stream_kind == "sync-responses": - - def sync_responses_handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(events) - - client = wrap_openai(sync_client(sync_responses_handler)) - received = list( - cast(Any, client.responses.create)(model="gpt-4o-mini", input="stream", stream=True) - ) - client.close() - else: - - async def async_responses_handler(request: httpx.Request) -> httpx.Response: - return named_sse_response(events) - - client = wrap_openai(async_client(async_responses_handler)) - stream = await cast(Any, client.responses.create)( - model="gpt-4o-mini", input="stream", stream=True - ) - received = [event async for event in stream] - await client.close() - assert len(received) == 2 - assert received[0].delta == "Still traced." - - assert received - span = only_span(memory) - assert span.status.status_code == StatusCode.UNSET - assert attrs(span)["gen_ai.response.id"] in {"chatcmpl_stream", "resp_old_core"} - - def test_chat_streaming_default_leaves_request_unchanged(memory: SimpleNamespace) -> None: requests: list[dict[str, Any]] = [] @@ -2028,8 +1842,9 @@ def handler(request: httpx.Request) -> httpx.Response: assert "gen_ai.usage.total_tokens" not in a -def test_chat_streaming_bounds_retained_state_without_dropping_metadata( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_streaming_bounds_retained_state_without_dropping_metadata( + memory: SimpleNamespace, async_mode: bool ) -> None: source_events = oversized_chat_stream_events() source_content = "".join( @@ -2038,25 +1853,34 @@ def test_chat_streaming_bounds_retained_state_without_dropping_metadata( if event["choices"] ) - def handler(request: httpx.Request) -> httpx.Response: - return sse_response(source_events) + if async_mode: - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - delivered = list(stream) - budget = stream._budget - state = stream._states[0] + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(source_events) + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + delivered = [chunk async for chunk in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(source_events))) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + delivered = list(stream) assert len(delivered) == len(source_events) - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert source_content.startswith(state.content) - assert len(state.content) < len(source_content) - assert state.role is None - assert not state.content.endswith("not-retained") a = attrs(only_span(memory)) + output = json.loads(str(a["gen_ai.output.messages"])) + assert len(output) == 1 + assert output[0]["role"] == "assistant" + assert set(output[0]) == {"role", "content"} + content = output[0]["content"] + assert source_content.startswith(content) + assert len(content) < len(source_content) + assert not content.endswith("not-retained") assert len(str(a["gen_ai.output.messages"]).encode()) <= 48 * 1024 assert a["gen_ai.response.finish_reasons"] == ("stop",) assert a["gen_ai.usage.total_tokens"] == 7 @@ -2116,21 +1940,32 @@ async def handler(_request: httpx.Request) -> httpx.Response: @pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("capture_output", [False, True]) async def test_multi_choice_finish_reasons_survive_a_zero_content_limit( - make: Any, async_mode: bool + make: Any, async_mode: bool, capture_output: bool ) -> None: - memory = make(max_attribute_length=0, capture_input=False, capture_output=False) + memory = make(max_attribute_length=0, capture_input=False, capture_output=capture_output) + base = { + "id": "chatcmpl_two", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o", + } events: list[dict[str, Any]] = [ { - "id": "chatcmpl_two", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o", + **base, + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "first"}}, + {"index": 1, "delta": {"role": "assistant", "content": "second"}}, + ], + }, + { + **base, "choices": [ {"index": 0, "delta": {}, "finish_reason": "stop"}, {"index": 1, "delta": {}, "finish_reason": "length"}, ], - } + }, ] if async_mode: @@ -2151,7 +1986,7 @@ async def handler(_request: httpx.Request) -> httpx.Response: ) delivered = list(stream) - assert len(delivered) == 1 + assert len(delivered) == 2 a = attrs(only_span(memory)) assert a["gen_ai.response.finish_reasons"] == ("stop", "length") assert "telemetry.dev.capture.truncated" not in a @@ -2315,7 +2150,6 @@ async def async_handler(_request: httpx.Request) -> httpx.Response: client.close() assert len(delivered) == 1 - assert len(stream._states) == 333 a = attrs(only_span(memory)) output = json.loads(str(a["gen_ai.output.messages"])) finish_reasons = cast(Sequence[str], a["gen_ai.response.finish_reasons"]) @@ -2325,26 +2159,43 @@ async def async_handler(_request: httpx.Request) -> httpx.Response: assert a["telemetry.dev.capture.truncated"] is True -def test_chat_streaming_reclaims_finish_reason_items_on_replacement(make: Any) -> None: - memory = make(capture_output=False) - events: list[dict[str, Any]] = [ - { - "id": "chatcmpl_finish_reasons", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - {"index": index, "delta": {}, "finish_reason": "stop"} for index in range(200) +def finish_reason_chunk(choices: list[dict[str, Any]]) -> dict[str, Any]: + return { + "id": "chatcmpl_finish_reasons", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": choices, + } + + +@pytest.mark.parametrize( + ("events", "expected"), + [ + pytest.param( + [ + finish_reason_chunk( + [{"index": index, "delta": {}, "finish_reason": "stop"} for index in range(200)] + ), + finish_reason_chunk([{"index": 0, "delta": {}, "finish_reason": "length"}]), ], - }, - { - "id": "chatcmpl_finish_reasons", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini", - "choices": [{"index": 0, "delta": {}, "finish_reason": "length"}], - }, - ] + ("length",) + ("stop",) * 199, + id="replacement-reclaims-items", + ), + pytest.param( + [ + finish_reason_chunk([{"index": 0, "delta": {}, "finish_reason": "x" * 70_000}]), + finish_reason_chunk([{"index": 0, "delta": {}, "finish_reason": "stop"}]), + ], + ("stop",), + id="fitting-reason-clears-rejected-oversized-reason", + ), + ], +) +def test_chat_streaming_reclaims_finish_reason_items_on_replacement( + make: Any, events: list[dict[str, Any]], expected: tuple[str, ...] +) -> None: + memory = make(capture_output=False) client = wrap_openai(sync_client(lambda _request: sse_response(events))) list( @@ -2354,9 +2205,7 @@ def test_chat_streaming_reclaims_finish_reason_items_on_replacement(make: Any) - ) a = attrs(only_span(memory)) - finish_reasons = cast(Sequence[str], a["gen_ai.response.finish_reasons"]) - assert len(finish_reasons) == 200 - assert finish_reasons[0] == "length" + assert a["gen_ai.response.finish_reasons"] == expected assert "gen_ai.output.messages" not in a assert "telemetry.dev.capture.truncated" not in a @@ -2388,49 +2237,6 @@ def test_chat_streaming_retains_exact_multibyte_finish_reason_budget(make: Any) assert "telemetry.dev.capture.truncated" not in a -def test_chat_streaming_clears_recovered_finish_reason_truncation(memory: SimpleNamespace) -> None: - implementation = cast(Any, telemetry_dev_openai) - states: dict[int, Any] = {} - finish_reason_states: dict[int, str] = {} - finish_reason_reservations: dict[int, tuple[int, int]] = {} - rejected_finish_reasons: set[int] = set() - output_budget = implementation._chat_capture_budget(reserve_output_list=True) - finish_reason_budget = implementation._chat_capture_budget() - - implementation._record_chat_chunk( - {"choices": [{"index": 0, "delta": {}, "finish_reason": "x" * 70_000}]}, - states, - finish_reason_states, - finish_reason_reservations, - rejected_finish_reasons, - output_budget, - finish_reason_budget, - False, - ) - implementation._record_chat_chunk( - {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, - states, - finish_reason_states, - finish_reason_reservations, - rejected_finish_reasons, - output_budget, - finish_reason_budget, - False, - ) - - assert finish_reason_states == {0: "stop"} - assert rejected_finish_reasons == set() - partial = implementation._chat_partial( - states, - finish_reason_states, - None, - False, - output_budget.truncated or finish_reason_budget.truncated or bool(rejected_finish_reasons), - ) - assert finish_reason_budget.truncated is False - assert partial["attributes"] is None - - def test_chat_streaming_incrementally_accounts_for_many_tiny_deltas( memory: SimpleNamespace, ) -> None: @@ -2915,12 +2721,17 @@ def test_chat_streaming_retains_exact_content_for_all_fragment_accumulators() -> @pytest.mark.parametrize("async_mode", [False, True]) async def test_chat_streaming_reconstructs_legacy_function_calls_and_custom_tools( - make: Any, async_mode: bool + memory: SimpleNamespace, async_mode: bool ) -> None: - memory = make() - implementation = cast(Any, telemetry_dev_openai) - events = [ + base = { + "id": "chatcmpl_legacy", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + } + events: list[dict[str, Any]] = [ { + **base, "choices": [ { "index": 0, @@ -2936,9 +2747,10 @@ async def test_chat_streaming_reconstructs_legacy_function_calls_and_custom_tool ], }, } - ] + ], }, { + **base, "choices": [ { "index": 0, @@ -2948,31 +2760,35 @@ async def test_chat_streaming_reconstructs_legacy_function_calls_and_custom_tool }, "finish_reason": "stop", } - ] + ], }, ] - handle = telemetry_dev.start_span( - "chat gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) if async_mode: - async def source() -> AsyncIterator[Any]: - for event in events: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) - stream = implementation._InstrumentedAsyncStream( - source(), handle, False, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) - delivered = [event async for event in stream] + delivered = [chunk async for chunk in stream] + await client.close() else: - stream = implementation._InstrumentedStream( - iter(events), handle, False, time.perf_counter() + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) delivered = list(stream) - assert delivered == events - assert json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) == [ + assert [chunk.choices[0].delta.function_call.arguments for chunk in delivered] == [ + '{"city":', + '"Paris"}', + ] + a = attrs(only_span(memory)) + assert "telemetry.dev.capture.truncated" not in a + assert json.loads(str(a["gen_ai.output.messages"])) == [ { "role": "assistant", "content": None, @@ -3284,82 +3100,88 @@ async def test_chat_stream_clean_eof_without_finish_reason_is_incomplete_when_ou make: Any, async_mode: bool ) -> None: memory = make(capture_output=False) - implementation = cast(Any, telemetry_dev_openai) - events = [{"choices": [{"index": 0, "delta": {"content": "partial"}}]}] - handle = telemetry_dev.start_span( - "chat gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) + events = [ + { + "id": "chatcmpl_partial", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {"content": "partial"}}], + } + ] if async_mode: - async def source() -> AsyncIterator[Any]: - for event in events: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) - stream = implementation._InstrumentedAsyncStream( - source(), handle, False, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) - delivered = [event async for event in stream] + delivered = [chunk async for chunk in stream] + await client.close() else: - stream = implementation._InstrumentedStream( - iter(events), handle, False, time.perf_counter() + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) delivered = list(stream) - assert delivered == events + assert len(delivered) == 1 a = attrs(only_span(memory)) assert "gen_ai.output.messages" not in a assert "gen_ai.response.finish_reasons" not in a assert a["telemetry.dev.capture.truncated"] is True -def test_chat_stream_uses_the_client_policy_that_created_its_span( - make: Any, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_stream_uses_the_client_policy_that_created_its_span( + make: Any, monkeypatch: pytest.MonkeyPatch, async_mode: bool ) -> None: def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: return value - original = make( - capture_output=True, - mask=mask, - max_attribute_length=256, - ) - handle = telemetry_dev.start_span( - "chat gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - monkeypatch.setattr( - telemetry_dev, - "get_client", - lambda: SimpleNamespace(capture_output=False, mask=None, max_attribute_length=1), - ) - old_handle = SimpleNamespace( - _client=cast(Any, handle)._client, - _state=cast(Any, handle)._state, - end=handle.end, - update=handle.update, - ) - implementation = cast(Any, telemetry_dev_openai) - stream = implementation._InstrumentedStream( - iter( - [ - { - "choices": [ - { - "index": 0, - "delta": {"content": "x" * 100}, - "finish_reason": "stop", - } - ] - } - ] - ), - old_handle, - False, - time.perf_counter(), - ) - list(stream) + memory = make(capture_output=True, mask=mask, max_attribute_length=256) + event = { + "id": "chatcmpl_policy", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {"content": "x" * 100}, "finish_reason": "stop"}], + } - a = attrs(only_span(original)) + def response() -> httpx.Response: + # The span has started by now; a later global policy must not apply to it. + monkeypatch.setattr( + telemetry_dev, + "get_client", + lambda: SimpleNamespace( + capture_output=False, capture_input=False, mask=None, max_attribute_length=1 + ), + ) + return sse_response([event]) + + if async_mode: + + async def handler(_request: httpx.Request) -> httpx.Response: + return response() + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + _ = [chunk async for chunk in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: response())) + list( + cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + ) + + a = attrs(only_span(memory)) assert json.loads(str(a["gen_ai.output.messages"])) == [ {"role": "assistant", "content": "x" * 100} ] @@ -3477,10 +3299,13 @@ def handler(request: httpx.Request) -> httpx.Response: assert requests[1]["stream"] is True -def test_responses_stream_existing_response_is_instrumented(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_existing_response_is_instrumented( + memory: SimpleNamespace, async_mode: bool +) -> None: requests: list[httpx.Request] = [] - def handler(request: httpx.Request) -> httpx.Response: + def response(request: httpx.Request) -> httpx.Response: requests.append(request) events: list[tuple[str, Mapping[str, Any]]] = [ ( @@ -3502,9 +3327,19 @@ def handler(request: httpx.Request) -> httpx.Response: ] return named_sse_response(events) - client = wrap_openai(sync_client(handler)) - with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: - events = list(stream) + if async_mode: + + async def handler(request: httpx.Request) -> httpx.Response: + return response(request) + + client = wrap_openai(async_client(handler)) + async with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: + events = [event async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(response)) + with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: + events = list(stream) assert [event.type for event in events] == ["response.created", "response.completed"] assert requests[0].method == "GET" @@ -3521,14 +3356,15 @@ def handler(request: httpx.Request) -> httpx.Response: ) -def test_responses_stream_bounds_output_without_dropping_terminal_metadata( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_bounds_output_without_dropping_terminal_metadata( + memory: SimpleNamespace, async_mode: bool ) -> None: retained = response_payload(response_id="resp_bounded", text="prefix") retained["status"] = "in_progress" completed = response_payload(response_id="resp_bounded", text="x" * 70_000) - def handler(request: httpx.Request) -> httpx.Response: + def response() -> httpx.Response: events: list[tuple[str, Mapping[str, Any]]] = [ ( "response.created", @@ -3549,10 +3385,25 @@ def handler(request: httpx.Request) -> httpx.Response: ] return named_sse_response(events) - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="bounded", stream=True) - assert [event.type for event in stream] == ["response.created", "response.completed"] + if async_mode: + + async def handler(_request: httpx.Request) -> httpx.Response: + return response() + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="bounded", stream=True + ) + received = [event.type async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: response())) + stream = cast(Any, client.responses.create)( + model="gpt-4o-mini", input="bounded", stream=True + ) + received = [event.type for event in stream] + assert received == ["response.created", "response.completed"] a = attrs(only_span(memory)) assert a["gen_ai.response.id"] == "resp_bounded" assert a["gen_ai.response.model"] == "gpt-4o-2024-08-06" @@ -3592,51 +3443,68 @@ def empty_created_then_oversized_terminal() -> list[tuple[str, dict[str, Any]]]: ] -def assert_empty_created_output_not_reported(memory: SimpleNamespace) -> None: - a = attrs(only_span(memory)) - assert "gen_ai.output.messages" not in a - assert a["telemetry.dev.capture.truncated"] is True +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_does_not_report_empty_created_output_for_truncated_terminal( + memory: SimpleNamespace, async_mode: bool +) -> None: + events = empty_created_then_oversized_terminal() + if async_mode: -def test_responses_stream_does_not_report_empty_created_output_for_truncated_terminal( - memory: SimpleNamespace, -) -> None: - def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response(empty_created_then_oversized_terminal()) + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) - list( - cast(Any, wrap_openai(sync_client(handler)).responses.create)( + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( model="gpt-4o-mini", input="x", stream=True ) - ) - assert_empty_created_output_not_reported(memory) + delivered = [event async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + delivered = list( + cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + ) + + assert len(delivered) == len(events) + a = attrs(only_span(memory)) + assert "gen_ai.output.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True -def test_responses_stream_fitting_terminal_clears_prior_truncation(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_fitting_terminal_clears_prior_truncation( + memory: SimpleNamespace, async_mode: bool +) -> None: retained = response_payload(text="partial") retained["status"] = "in_progress" oversized = response_payload(text="x" * 70_000) oversized["status"] = "in_progress" completed = response_payload(text="final") + events: list[tuple[str, Mapping[str, Any]]] = [ + ("response.created", {"type": "response.created", "response": retained}), + ("response.in_progress", {"type": "response.in_progress", "response": oversized}), + ("response.completed", {"type": "response.completed", "response": completed}), + ] - def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response( - [ - ("response.created", {"type": "response.created", "response": retained}), - ( - "response.in_progress", - {"type": "response.in_progress", "response": oversized}, - ), - ("response.completed", {"type": "response.completed", "response": completed}), - ] - ) + if async_mode: - list( - cast(Any, wrap_openai(sync_client(handler)).responses.create)( + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) + + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( model="gpt-4o-mini", input="x", stream=True ) - ) + delivered = [event async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + delivered = list( + cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + ) + assert len(delivered) == len(events) a = attrs(only_span(memory)) assert json.loads(str(a["gen_ai.output.messages"]))[0]["content"][0]["text"] == "final" assert "telemetry.dev.capture.truncated" not in a @@ -3705,32 +3573,28 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: retained = response_payload(response_id="resp_masked_terminal", text="public") retained["status"] = "in_progress" terminal = response_payload(response_id="resp_masked_terminal", text=f"SECRET{'x' * 70_000}") - events = [ - {"type": "response.in_progress", "response": retained}, - {"type": "response.completed", "response": terminal}, + events: list[tuple[str, Mapping[str, Any]]] = [ + ("response.in_progress", {"type": "response.in_progress", "response": retained}), + ("response.completed", {"type": "response.completed", "response": terminal}), ] - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) if async_mode: - async def source() -> AsyncIterator[Any]: - for event in events: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) - stream = implementation._InstrumentedAsyncResponsesStream( - source(), handle, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="x", stream=True ) - delivered = [event async for event in stream] + delivered = [event.type async for event in stream] + await client.close() else: - stream = implementation._InstrumentedResponsesStream( - iter(events), handle, time.perf_counter() - ) - delivered = list(stream) + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + delivered = [event.type for event in stream] - assert delivered == events + assert delivered == ["response.in_progress", "response.completed"] a = attrs(only_span(memory)) assert "gen_ai.output.messages" not in a assert a["telemetry.dev.capture.truncated"] is True @@ -3835,77 +3699,44 @@ async def source() -> AsyncIterator[Any]: @pytest.mark.parametrize("async_mode", [False, True]) -@pytest.mark.parametrize("stream_limited", [False, True]) +@pytest.mark.parametrize( + ("max_attribute_length", "text_length", "kept"), + [(128, 1_000, True), (100 * 1024, 60 * 1024, False)], +) async def test_responses_stream_retention_follows_the_stream_ceiling_not_the_cap( - make: Any, monkeypatch: pytest.MonkeyPatch, async_mode: bool, stream_limited: bool + make: Any, async_mode: bool, max_attribute_length: int, text_length: int, kept: bool ) -> None: - if stream_limited: - monkeypatch.setattr(telemetry_dev_openai, "_CHAT_STREAM_CAPTURE_MAX_BYTES", 128) - memory = make() - else: - memory = make(max_attribute_length=128) - terminal = response_payload(response_id="resp_small_limit", text="x" * 1_000) - event = {"type": "response.completed", "response": terminal} - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) + memory = make(max_attribute_length=max_attribute_length) + terminal = response_payload(response_id="resp_stream_ceiling", text="x" * text_length) + events: list[tuple[str, Mapping[str, Any]]] = [ + ("response.completed", {"type": "response.completed", "response": terminal}) + ] if async_mode: - async def source() -> AsyncIterator[Any]: - yield event + async def handler(_request: httpx.Request) -> httpx.Response: + return named_sse_response(events) - stream = implementation._InstrumentedAsyncResponsesStream( - source(), handle, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="x", stream=True ) - delivered = [item async for item in stream] + delivered = [event.type async for event in stream] + await client.close() else: - stream = implementation._InstrumentedResponsesStream( - iter([event]), handle, time.perf_counter() - ) - delivered = list(stream) + client = wrap_openai(sync_client(lambda _request: named_sse_response(events))) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) + delivered = [event.type for event in stream] - assert delivered == [event] + assert delivered == ["response.completed"] a = attrs(only_span(memory)) - if stream_limited: - assert "gen_ai.output.messages" not in a - assert a["telemetry.dev.capture.truncated"] is True - else: + if kept: assert len(str(a["gen_ai.output.messages"])) == 128 assert str(a["gen_ai.output.messages"]).endswith("...[truncated]") assert "telemetry.dev.capture.truncated" not in a - - -@pytest.mark.parametrize("async_mode", [False, True]) -async def test_responses_stream_honors_hard_capture_limit(make: Any, async_mode: bool) -> None: - memory = make(max_attribute_length=100 * 1024) - terminal = response_payload(response_id="resp_hard_limit", text="x" * (60 * 1024)) - event = {"type": "response.completed", "response": terminal} - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) - - if async_mode: - - async def source() -> AsyncIterator[Any]: - yield event - - stream = implementation._InstrumentedAsyncResponsesStream( - source(), handle, time.perf_counter() - ) - delivered = [item async for item in stream] else: - stream = implementation._InstrumentedResponsesStream( - iter([event]), handle, time.perf_counter() - ) - delivered = list(stream) - - assert delivered == [event] - a = attrs(only_span(memory)) - assert "gen_ai.output.messages" not in a - assert a["telemetry.dev.capture.truncated"] is True + assert "gen_ai.output.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True @pytest.mark.parametrize("async_mode", [False, True]) @@ -3920,57 +3751,52 @@ def identity_mask(value: Any, _context: Any) -> Any: retained_response = response_payload(response_id="resp_interrupted", text="partial") retained_response["status"] = "in_progress" retained = {"type": "response.in_progress", "response": retained_response} - error_event = {"type": "error", "code": "stream_error", "message": "interrupted"} terminal = { "type": "response.completed", "response": {**retained_response, "status": "completed"}, } - stream_error = RuntimeError("responses stream interrupted") - handle = telemetry_dev.start_span( - "responses gpt-4o-mini", type="generation", model="gpt-4o-mini", provider="openai" - ) - implementation = cast(Any, telemetry_dev_openai) + + def response() -> httpx.Response: + if mode == "eof": + return sse_response([retained]) + if mode == "early_close": + return sse_response([retained, terminal]) + if mode == "error_event": + error_event = {"type": "error", "code": "stream_error", "message": "interrupted"} + return sse_response([retained, error_event]) + if async_mode: + return failing_async_sse_response(retained, "interrupted") + return failing_sync_sse_response(retained, "interrupted") if async_mode: - async def async_source() -> AsyncIterator[Any]: - yield retained - if mode == "error_event": - yield error_event - elif mode == "throw": - raise stream_error - elif mode == "early_close": - yield terminal + async def handler(_request: httpx.Request) -> httpx.Response: + return response() - stream = implementation._InstrumentedAsyncResponsesStream( - async_source(), handle, time.perf_counter() + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="x", stream=True ) if mode == "early_close": - assert await stream.__anext__() is retained + assert (await stream.__anext__()).type == "response.in_progress" await stream.close() elif mode == "throw": - with pytest.raises(RuntimeError, match="responses stream interrupted"): - [item async for item in stream] + with pytest.raises(Exception) as exc_info: + _ = [event async for event in stream] + assert_stream_transport_error(exc_info.value, "interrupted") else: - [item async for item in stream] + _ = [event async for event in stream] + await client.close() else: - - def source() -> Iterator[Any]: - yield retained - if mode == "error_event": - yield error_event - elif mode == "throw": - raise stream_error - elif mode == "early_close": - yield terminal - - stream = implementation._InstrumentedResponsesStream(source(), handle, time.perf_counter()) + client = wrap_openai(sync_client(lambda _request: response())) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) if mode == "early_close": - assert next(stream) is retained + assert next(stream).type == "response.in_progress" stream.close() elif mode == "throw": - with pytest.raises(RuntimeError, match="responses stream interrupted"): + with pytest.raises(Exception) as exc_info: list(stream) + assert_stream_transport_error(exc_info.value, "interrupted") else: list(stream) @@ -4104,42 +3930,6 @@ async def handler(request: httpx.Request) -> httpx.Response: assert attrs(stream_span)["gen_ai.usage.total_tokens"] == 7 -async def test_async_chat_streaming_bounds_retained_state_without_dropping_metadata( - memory: SimpleNamespace, -) -> None: - source_events = oversized_chat_stream_events() - source_content = "".join( - cast(str, event["choices"][0]["delta"].get("content", "")) - for event in source_events - if event["choices"] - ) - - async def handler(request: httpx.Request) -> httpx.Response: - return sse_response(source_events) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - delivered = [chunk async for chunk in stream] - budget = stream._budget - state = stream._states[0] - await client.close() - - assert len(delivered) == len(source_events) - assert budget.truncated is True - assert budget.bytes_used <= budget.max_bytes - assert source_content.startswith(state.content) - assert len(state.content) < len(source_content) - assert state.role is None - assert not state.content.endswith("not-retained") - a = attrs(only_span(memory)) - assert len(str(a["gen_ai.output.messages"]).encode()) <= 48 * 1024 - assert a["gen_ai.response.finish_reasons"] == ("stop",) - assert a["gen_ai.usage.total_tokens"] == 7 - assert a["telemetry.dev.capture.truncated"] is True - - async def test_async_responses_create_and_embeddings_map_usage_like_sync( memory: SimpleNamespace, ) -> None: @@ -4186,35 +3976,27 @@ async def handler(request: httpx.Request) -> httpx.Response: assert embedding_attrs["gen_ai.usage.total_tokens"] == 6 +@pytest.mark.parametrize("async_mode", [False, True]) @pytest.mark.parametrize("method", ["retrieve", "cancel"]) -def test_batch_positional_id_is_recorded_when_sync_request_errors( - memory: SimpleNamespace, method: str +async def test_batch_positional_id_is_recorded_when_request_errors( + memory: SimpleNamespace, async_mode: bool, method: str ) -> None: - def handler(_request: httpx.Request) -> httpx.Response: + def response() -> httpx.Response: return httpx.Response(500, json={"error": {"message": "batch failed"}}) - client = wrap_openai(sync_client(handler)) - - with pytest.raises(openai.APIStatusError): - getattr(client.batches, method)("batch_positional") - - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["openai.batch.id"] == "batch_positional" - - -@pytest.mark.parametrize("method", ["retrieve", "cancel"]) -async def test_batch_positional_id_is_recorded_when_async_request_errors( - memory: SimpleNamespace, method: str -) -> None: - async def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(500, json={"error": {"message": "batch failed"}}) + if async_mode: - client = wrap_openai(async_client(handler)) + async def handler(_request: httpx.Request) -> httpx.Response: + return response() - with pytest.raises(openai.APIStatusError): - await getattr(client.batches, method)("batch_positional") - await client.close() + client = wrap_openai(async_client(handler)) + with pytest.raises(openai.APIStatusError): + await getattr(client.batches, method)("batch_positional") + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: response())) + with pytest.raises(openai.APIStatusError): + getattr(client.batches, method)("batch_positional") span = only_span(memory) assert span.status.status_code == StatusCode.ERROR @@ -4233,46 +4015,11 @@ async def handler(_request: httpx.Request) -> httpx.Response: assert memory.span_exporter.get_finished_spans() == () -async def test_async_chat_stream_iteration_error_records_one_error_span( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_stream_terminal_chunk_then_early_close_is_incomplete( + memory: SimpleNamespace, async_mode: bool ) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return failing_async_sse_response(terminal_chat_stream_event(), "async stream broke") - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - first = await stream.__anext__() - assert first.choices[0].delta.content == "Hello" - - with pytest.raises(Exception) as exc_info: - await stream.__anext__() - await client.close() - - assert_stream_transport_error(exc_info.value, "async stream broke") - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - a = attrs(span) - assert a["error.type"] == type(exc_info.value).__name__ - assert json.loads(str(a["gen_ai.output.messages"])) == [ - {"role": "assistant", "content": "Hello"} - ] - assert a["gen_ai.response.finish_reasons"] == ("stop",) - assert a["telemetry.dev.capture.truncated"] is True - events = list(span.events) - assert len(events) == 1 - event_attrs = dict(events[0].attributes or {}) - assert events[0].name == "exception" - assert event_attrs["exception.type"] == type(exc_info.value).__name__ - assert event_attrs["exception.message"] == str(exc_info.value) - - -@pytest.mark.parametrize("async_mode", [False, True]) -async def test_chat_stream_terminal_chunk_then_early_close_is_incomplete( - memory: SimpleNamespace, async_mode: bool -) -> None: - events = [terminal_chat_stream_event(), *chat_stream_events()[1:]] + events = [terminal_chat_stream_event(), *chat_stream_events()[1:]] def handler(request: httpx.Request) -> httpx.Response: return sse_response(events) @@ -4325,21 +4072,37 @@ def test_chat_stream_plain_for_break_ends_span_once(memory: SimpleNamespace) -> client.close() -def test_chat_stream_iteration_error_records_one_error_span(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return failing_sync_sse_response(terminal_chat_stream_event(), "sync stream broke") +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_stream_iteration_error_records_one_error_span( + memory: SimpleNamespace, async_mode: bool +) -> None: + event = terminal_chat_stream_event() - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - first = next(stream) - assert first.choices[0].delta.content == "Hello" + if async_mode: - with pytest.raises(Exception) as exc_info: - next(stream) + async def handler(_request: httpx.Request) -> httpx.Response: + return failing_async_sse_response(event, "stream broke") - assert_stream_transport_error(exc_info.value, "sync stream broke") + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + assert (await stream.__anext__()).choices[0].delta.content == "Hello" + with pytest.raises(Exception) as exc_info: + await stream.__anext__() + await client.close() + else: + client = wrap_openai( + sync_client(lambda _request: failing_sync_sse_response(event, "stream broke")) + ) + stream = cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) + assert next(stream).choices[0].delta.content == "Hello" + with pytest.raises(Exception) as exc_info: + next(stream) + + assert_stream_transport_error(exc_info.value, "stream broke") span = only_span(memory) assert span.status.status_code == StatusCode.ERROR a = attrs(span) @@ -4398,23 +4161,31 @@ def handler(request: httpx.Request) -> httpx.Response: assert span.status.status_code == StatusCode.ERROR -def test_responses_stream_bare_error_event_sets_error_status(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "type": "error", - "code": "server_error", - "message": "model blew up", - "sequence_number": 0, - } - ] - ) +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_responses_stream_bare_error_event_sets_error_status( + memory: SimpleNamespace, async_mode: bool +) -> None: + events = [ + {"type": "error", "code": "server_error", "message": "model blew up", "sequence_number": 0} + ] - client = wrap_openai(sync_client(handler)) - stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="fail", stream=True) + if async_mode: + + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(events) - assert [event.type for event in stream] == ["error"] + client = wrap_openai(async_client(handler)) + stream = await cast(Any, client.responses.create)( + model="gpt-4o-mini", input="fail", stream=True + ) + received = [event.type async for event in stream] + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(events))) + stream = cast(Any, client.responses.create)(model="gpt-4o-mini", input="fail", stream=True) + received = [event.type for event in stream] + + assert received == ["error"] assert only_span(memory).status.status_code == StatusCode.ERROR @@ -4468,50 +4239,46 @@ def handler(request: httpx.Request) -> httpx.Response: def test_double_instrument_is_idempotent_and_uninstrument_restores_once( memory: SimpleNamespace, ) -> None: - originals = ( - Completions.create, - Completions.parse, - AsyncCompletions.create, - AsyncCompletions.parse, - Responses.create, - Responses.retrieve, - Responses.parse, - AsyncResponses.create, - AsyncResponses.parse, - AsyncResponses.retrieve, - Embeddings.create, - AsyncEmbeddings.create, + # The resource classes the README documents as instrumented. + classes: tuple[type[Any], ...] = ( + Completions, + AsyncCompletions, + Responses, + AsyncResponses, + Embeddings, + AsyncEmbeddings, + Images, + AsyncImages, + Speech, + AsyncSpeech, + Transcriptions, + AsyncTranscriptions, + Translations, + AsyncTranslations, + Batches, + AsyncBatches, ) + def snapshot() -> dict[type[Any], dict[str, Any]]: + return {cls: dict(vars(cls)) for cls in classes} + def handler(request: httpx.Request) -> httpx.Response: return json_response(chat_completion()) + before = snapshot() instrument_openai() try: - instrumented_create = Completions.create - assert instrumented_create is not originals[0] + instrumented = snapshot() + assert [cls for cls in classes if instrumented[cls] == before[cls]] == [] instrument_openai() - assert Completions.create is instrumented_create - instrumented = sync_client(handler) - cast(Any, instrumented.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) + assert snapshot() == instrumented + client = sync_client(handler) + cast(Any, client.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) assert len(memory.span_exporter.get_finished_spans()) == 1 finally: uninstrument_openai() - assert ( - Completions.create, - Completions.parse, - AsyncCompletions.create, - AsyncCompletions.parse, - Responses.create, - Responses.retrieve, - Responses.parse, - AsyncResponses.create, - AsyncResponses.parse, - AsyncResponses.retrieve, - Embeddings.create, - AsyncEmbeddings.create, - ) == originals + assert snapshot() == before uninstrumented = sync_client(handler) cast(Any, uninstrumented.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) assert len(memory.span_exporter.get_finished_spans()) == 1 @@ -4524,11 +4291,12 @@ def later_owner(self: Any, *args: Any, **kwargs: Any) -> Any: return original(self, *args, **kwargs) instrument_openai() - Completions.create = later_owner - uninstrument_openai() - - assert Completions.create is later_owner - Completions.create = original + try: + Completions.create = later_owner + uninstrument_openai() + assert Completions.create is later_owner + finally: + Completions.create = original def test_wrap_openai_is_idempotent(memory: SimpleNamespace) -> None: @@ -4627,76 +4395,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert all(span.name == "chat gpt-4o-mini" for span in spans) -def test_chat_completion_finish_reasons_preserve_single_and_multi_choice_order( - memory: SimpleNamespace, -) -> None: - def handler(request: httpx.Request) -> httpx.Response: - body = request_json(request) - if body.get("n") == 2: - return json_response( - chat_completion( - choices=[ - { - "index": 0, - "message": {"role": "assistant", "content": "first"}, - "finish_reason": "stop", - }, - { - "index": 1, - "message": {"role": "assistant", "content": "second"}, - "finish_reason": "length", - }, - ] - ) - ) - return json_response(chat_completion()) - - client = wrap_openai(sync_client(handler)) - cast(Any, client.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES) - cast(Any, client.chat.completions.create)(model="gpt-4o-mini", messages=CHAT_MESSAGES, n=2) - - single_span, multi_span = memory.span_exporter.get_finished_spans() - assert attrs(single_span)["gen_ai.response.finish_reasons"] == ("stop",) - assert attrs(multi_span)["gen_ai.response.finish_reasons"] == ("stop", "length") - - -def test_chat_streaming_finish_reasons_include_all_choices(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "id": "chatcmpl_stream_choices", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini-2024-07-18", - "choices": [ - {"index": 0, "delta": {"role": "assistant", "content": "first"}}, - {"index": 1, "delta": {"role": "assistant", "content": "second"}}, - ], - }, - { - "id": "chatcmpl_stream_choices", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini-2024-07-18", - "choices": [ - {"index": 0, "delta": {}, "finish_reason": "stop"}, - {"index": 1, "delta": {}, "finish_reason": "length"}, - ], - }, - ] - ) - - client = wrap_openai(sync_client(handler)) - list( - cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) - ) - - assert attrs(only_span(memory))["gen_ai.response.finish_reasons"] == ("stop", "length") - - def test_chat_streaming_reconstructs_split_tool_call_arguments(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return sse_response( @@ -4837,212 +4535,27 @@ async def handler(request: httpx.Request) -> httpx.Response: assert attrs(failed_span)["error.type"] == "RuntimeError" -async def test_async_responses_stream_bounds_output_without_dropping_terminal_metadata( - memory: SimpleNamespace, -) -> None: - retained = response_payload(response_id="resp_bounded_async", text="prefix") - retained["status"] = "in_progress" - completed = response_payload(response_id="resp_bounded_async", text="x" * 70_000) - - async def handler(request: httpx.Request) -> httpx.Response: - events: list[tuple[str, Mapping[str, Any]]] = [ - ( - "response.created", - { - "type": "response.created", - "sequence_number": 0, - "response": retained, - }, - ), - ( - "response.completed", - { - "type": "response.completed", - "sequence_number": 1, - "response": completed, - }, - ), - ] - return named_sse_response(events) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)( - model="gpt-4o-mini", input="bounded", stream=True - ) - assert [event.type async for event in stream] == [ - "response.created", - "response.completed", - ] - await client.close() - - a = attrs(only_span(memory)) - assert a["gen_ai.response.id"] == "resp_bounded_async" - assert a["gen_ai.response.model"] == "gpt-4o-2024-08-06" - assert a["gen_ai.usage.total_tokens"] == 17 - assert a["gen_ai.response.finish_reasons"] == ("stop",) - assert json.loads(str(a["gen_ai.output.messages"])) == [ - { - "id": "msg_123", - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "prefix", "annotations": []}], - } - ] - assert a["telemetry.dev.capture.truncated"] is True - - -async def test_async_responses_stream_does_not_report_empty_created_output_for_truncated_terminal( - memory: SimpleNamespace, -) -> None: - async def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response(empty_created_then_oversized_terminal()) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) - [event async for event in stream] - await client.close() - assert_empty_created_output_not_reported(memory) - - -async def test_async_responses_stream_fitting_terminal_clears_prior_truncation( - memory: SimpleNamespace, -) -> None: - retained = response_payload(text="partial") - retained["status"] = "in_progress" - oversized = response_payload(text="x" * 70_000) - oversized["status"] = "in_progress" - completed = response_payload(text="final") - - async def handler(_request: httpx.Request) -> httpx.Response: - return named_sse_response( - [ - ("response.created", {"type": "response.created", "response": retained}), - ( - "response.in_progress", - {"type": "response.in_progress", "response": oversized}, - ), - ("response.completed", {"type": "response.completed", "response": completed}), - ] - ) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)(model="gpt-4o-mini", input="x", stream=True) - assert len([event async for event in stream]) == 3 - await client.close() - - a = attrs(only_span(memory)) - assert json.loads(str(a["gen_ai.output.messages"]))[0]["content"][0]["text"] == "final" - assert "telemetry.dev.capture.truncated" not in a - - -async def test_async_responses_stream_existing_response_is_instrumented( - memory: SimpleNamespace, -) -> None: - requests: list[httpx.Request] = [] - - async def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - events: list[tuple[str, Mapping[str, Any]]] = [ - ( - "response.created", - { - "type": "response.created", - "sequence_number": 0, - "response": {"id": "resp_existing", "status": "in_progress", "output": []}, - }, - ), - ( - "response.completed", - { - "type": "response.completed", - "sequence_number": 1, - "response": response_payload(response_id="resp_existing", text="Streamed."), - }, - ), - ] - return named_sse_response(events) - - client = wrap_openai(async_client(handler)) - async with cast(Any, client.responses.stream)(response_id="resp_existing") as stream: - events = [event async for event in stream] - await client.close() - - assert [event.type for event in events] == ["response.created", "response.completed"] - assert requests[0].method == "GET" - assert requests[0].url.path == "/v1/responses/resp_existing" - span = only_span(memory) - a = attrs(span) - assert a["gen_ai.response.id"] == "resp_existing" - assert ( - json.loads(str(a["gen_ai.output.messages"])) - == response_payload( - response_id="resp_existing", - text="Streamed.", - )["output"] - ) - - -async def test_async_responses_stream_bare_error_event_sets_error_status( - memory: SimpleNamespace, -) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return sse_response( - [ - { - "type": "error", - "code": "server_error", - "message": "model blew up", - "sequence_number": 0, - } - ] - ) - - client = wrap_openai(async_client(handler)) - stream = await cast(Any, client.responses.create)( - model="gpt-4o-mini", input="fail", stream=True - ) - events = [event async for event in stream] - await client.close() - - assert [event.type for event in events] == ["error"] - assert only_span(memory).status.status_code == StatusCode.ERROR - - -def test_chat_stream_context_manager_exit_ends_partial_span_once( - memory: SimpleNamespace, +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_chat_stream_context_manager_exit_ends_partial_span_once( + memory: SimpleNamespace, async_mode: bool ) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) - - client = wrap_openai(sync_client(handler)) - with cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) as stream: - first = next(stream) - - spans = memory.span_exporter.get_finished_spans() - assert first.choices[0].delta.content == "Hello" - assert len(spans) == 1 - assert spans[0].status.status_code == StatusCode.UNSET - assert json.loads(str(attrs(spans[0])["gen_ai.output.messages"])) == [ - {"role": "assistant", "content": "Hello"} - ] - assert attrs(spans[0])["telemetry.dev.capture.truncated"] is True - + if async_mode: -async def test_async_chat_stream_context_manager_exit_ends_partial_span_once( - memory: SimpleNamespace, -) -> None: - async def handler(request: httpx.Request) -> httpx.Response: - return sse_response(chat_stream_events()) + async def handler(_request: httpx.Request) -> httpx.Response: + return sse_response(chat_stream_events()) - client = wrap_openai(async_client(handler)) - async with await cast(Any, client.chat.completions.create)( - model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True - ) as stream: - first = await stream.__anext__() - await client.close() + client = wrap_openai(async_client(handler)) + async with await cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) as stream: + first = await stream.__anext__() + await client.close() + else: + client = wrap_openai(sync_client(lambda _request: sse_response(chat_stream_events()))) + with cast(Any, client.chat.completions.create)( + model="gpt-4o-mini", messages=CHAT_MESSAGES, stream=True + ) as stream: + first = next(stream) spans = memory.span_exporter.get_finished_spans() assert first.choices[0].delta.content == "Hello" @@ -5099,39 +4612,31 @@ def handler(request: httpx.Request) -> httpx.Response: assert attrs(spans[0])["telemetry.dev.capture.truncated"] is True -def test_text_media_response_captures_text_and_falls_back_text_tokens( - memory: SimpleNamespace, -) -> None: - mapper = vars(telemetry_dev_openai)["_text_media_response"] - fields = mapper( - SimpleNamespace(text="transcript", usage=SimpleNamespace(output_tokens=7, total_tokens=7)) - ) - assert fields["output"] == "transcript" - assert fields["usage"]["text_output_tokens"] == 7 - - -def test_text_media_response_passes_the_complete_transcript_to_the_core(make: Any) -> None: - make(max_attribute_length=5) - mapper = vars(telemetry_dev_openai)["_text_media_response"] - - fields = mapper(SimpleNamespace(text="ééé")) - - assert fields["output"] == "ééé" - assert "attributes" not in fields +def test_bounded_responses_conversion_does_not_model_dump(make: Any) -> None: + class Item: + def __init__(self) -> None: + self.type = "output_text" + self.text = "x" * 100_000 + def model_dump(self, **_kwargs: Any) -> Any: + raise AssertionError("model_dump must not run") -def test_bounded_responses_conversion_does_not_model_dump(memory: SimpleNamespace) -> None: class Bomb: def __init__(self) -> None: - self.output = [{"type": "output_text", "text": "x" * 100_000}] + self.output = [Item()] self.status = "completed" - def model_dump(self, **kwargs: Any) -> Any: + def model_dump(self, **_kwargs: Any) -> Any: raise AssertionError("model_dump must not run") + errors: list[BaseException] = [] + make(on_error=errors.append) + fields = vars(telemetry_dev_openai)["_responses_response"](Bomb()) + assert fields["output"] is None assert fields["attributes"]["telemetry.dev.capture.truncated"] is True + assert errors == [] def test_bounded_responses_capture_repeats_shared_acyclic_values( @@ -5182,9 +4687,10 @@ def items(self) -> Any: @pytest.mark.parametrize("kind", ["oversized", "deep", "cyclic", "hostile"]) -def test_responses_input_capture_failure_never_blocks_provider( - memory: SimpleNamespace, kind: str -) -> None: +def test_responses_input_capture_failure_never_blocks_provider(make: Any, kind: str) -> None: + errors: list[BaseException] = [] + memory = make(on_error=errors.append) + class HostileMapping(dict[str, Any]): def items(self) -> Any: raise RuntimeError("hostile mapping") @@ -5228,7 +4734,11 @@ def provider_resolver(_resource: object | None) -> str: wrapper(model="gpt-4.1", input=value) assert called - if kind == "oversized": - span = memory.span_exporter.get_finished_spans()[-1] - assert "gen_ai.input.messages" not in attrs(span) - assert attrs(span)["telemetry.dev.capture.truncated"] is True + a = attrs(only_span(memory)) + if kind == "cyclic": + assert a["gen_ai.input.messages"] == '{"self":null}' + assert "telemetry.dev.capture.truncated" not in a + else: + assert "gen_ai.input.messages" not in a + assert a["telemetry.dev.capture.truncated"] is True + assert [str(error) for error in errors] == (["hostile mapping"] if kind == "hostile" else []) diff --git a/sdks/python-openrouter/tests/test_openrouter.py b/sdks/python-openrouter/tests/test_openrouter.py index 4206b9d..3f6ea13 100644 --- a/sdks/python-openrouter/tests/test_openrouter.py +++ b/sdks/python-openrouter/tests/test_openrouter.py @@ -23,7 +23,6 @@ from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.trace import StatusCode -import telemetry_dev_openrouter from telemetry_dev_openrouter import ( instrument_openrouter, uninstrument_openrouter, @@ -199,6 +198,14 @@ def responses_payload( } +def completed_without_output(sequence_number: int) -> dict[str, Any]: + """A terminal event whose null output fails SDK validation, so it arrives raw and ends the + stream without replacing the reconstructed output.""" + response = responses_payload() + response["output"] = None + return {"type": "response.completed", "sequence_number": sequence_number, "response": response} + + def embeddings_payload() -> dict[str, Any]: return { "id": "embed_123", @@ -535,7 +542,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: assert len(calls) == 3 assert len(memory.span_exporter.get_finished_spans()) == 1 - assert memory.metric_reader.get_metrics_data() is not None @pytest.mark.parametrize("async_mode", [False, True]) @@ -590,7 +596,6 @@ async def async_handler(request: httpx.Request) -> httpx.Response: assert len(calls) == 3 assert len(memory.span_exporter.get_finished_spans()) == 1 - assert memory.metric_reader.get_metrics_data() is not None def test_chat_stream_capture_is_bounded_without_losing_terminal_metadata( @@ -623,7 +628,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["gen_ai.usage.cost"] == 0.04 -def _streamed_chat(memory: SimpleNamespace, events: list[dict[str, Any]]) -> tuple[Any, Any]: +def _streamed_chat(memory: SimpleNamespace, events: list[dict[str, Any]]) -> dict[str, object]: def handler(request: httpx.Request) -> httpx.Response: return sse_response(events) @@ -631,7 +636,7 @@ def handler(request: httpx.Request) -> httpx.Response: model="openai/gpt-4o-mini", messages=CHAT_MESSAGES, stream=True ) assert len(list(stream)) == len(events) - return stream, attrs(only_span(memory)) + return attrs(only_span(memory)) @pytest.mark.parametrize( @@ -645,7 +650,10 @@ def handler(request: httpx.Request) -> httpx.Response: def test_configured_cap_applies_after_the_mask_not_to_stream_retention( make: Any, max_attribute_length: int, redact: bool, expected: str | None ) -> None: + seen: list[Any] = [] + def mask(value: Any, _context: Any) -> Any: + seen.append(value) if not redact: return value return [{"role": "assistant", "content": "[redacted]"}] @@ -657,11 +665,11 @@ def mask(value: Any, _context: Any) -> Any: chat_chunk({}, finish_reason="stop"), ] - stream, a = _streamed_chat(memory, events) + a = _streamed_chat(memory, events) - assert stream._budget.truncated is False - assert stream._states[0].content == text assert "telemetry.dev.capture.truncated" not in a + if redact: + assert [{"role": "assistant", "content": text}] in seen if expected is None: assert a.get("gen_ai.output.messages", "") == "" else: @@ -688,19 +696,18 @@ def mask(value: Any, _context: Any) -> Any: chat_chunk({}, finish_reason="stop"), ] - stream, a = _streamed_chat(memory, events) + a = _streamed_chat(memory, events) - budget = stream._budget - retained = stream._states[0].content - assert budget.truncated is True - assert budget.max_bytes == telemetry_dev_openrouter._STREAM_CAPTURE_MAX_BYTES # pyright: ignore[reportPrivateUsage] - assert 0 < len(retained) < 1_100 * 100 assert a["telemetry.dev.capture.truncated"] is True if masked: assert "gen_ai.output.messages" not in a assert not any("x" * 100 in value for value in seen) - elif max_attribute_length > len(retained): - assert json.loads(str(a["gen_ai.output.messages"]))[0]["content"] == retained + elif max_attribute_length == 1024 * 1024: + # The 64 KiB retention bound, not the 1 MiB cap, cuts the stream at a chunk boundary. + content = json.loads(str(a["gen_ai.output.messages"]))[0]["content"] + assert set(content) == {"x"} + assert 0 < len(content) <= 64 * 1024 + assert len(content) % 100 == 0 else: assert len(str(a.get("gen_ai.output.messages", ""))) <= max_attribute_length @@ -797,7 +804,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert len(chunks) == 1 a = attrs(only_span(memory)) output = json.loads(str(a["gen_ai.output.messages"])) - assert len(output) == telemetry_dev_openrouter._STREAM_CAPTURE_MAX_ITEMS # pyright: ignore[reportPrivateUsage] + assert len(output) == 1024 assert output[0] == {"role": "assistant", "content": "Hi"} assert a["telemetry.dev.capture.truncated"] is True @@ -819,10 +826,9 @@ def test_chat_stream_finish_reasons_are_capped_at_a_fixed_length( }, ] - stream, a = _streamed_chat(memory, events) + a = _streamed_chat(memory, events) - assert list(a["gen_ai.response.finish_reasons"]) == ["r" * max_length, "stop"] - assert all(len(state.finish_reason or "") <= max_length for state in stream._states.values()) + assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["r" * max_length, "stop"] assert (a.get("telemetry.dev.capture.truncated") is True) is spent @@ -1056,30 +1062,18 @@ def handler(request: httpx.Request) -> httpx.Response: ) -def test_responses_stream_failed_event_sets_error(memory: SimpleNamespace) -> None: - def handler(request: httpx.Request) -> httpx.Response: - event = { - "type": "response.failed", - "sequence_number": 0, - "response": responses_payload( - response_id="resp_failed", - status="failed", - error={"code": "server_error", "message": "model blew up"}, - ), - } - return sse_response([event]) - - client = wrap_open_router(sync_client(handler)) - stream = cast(Any, client.responses.send)(model="openai/gpt-4o-mini", input="fail", stream=True) - events = list(stream) - - assert [event.type for event in events] == ["response.failed"] - span = only_span(memory) - assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "RuntimeError" - - -def test_responses_stream_failed_event_retains_numeric_code(memory: SimpleNamespace) -> None: +@pytest.mark.parametrize( + "code", + [ + # Fails SDK validation, so it arrives as a raw unknown event. + pytest.param(502, id="numeric-code"), + # Parses as the typed failed event. + pytest.param("server_error", id="string-code"), + ], +) +def test_responses_stream_failed_event_retains_error_code( + memory: SimpleNamespace, code: int | str +) -> None: def handler(request: httpx.Request) -> httpx.Response: event = { "type": "response.failed", @@ -1087,7 +1081,7 @@ def handler(request: httpx.Request) -> httpx.Response: "response": responses_payload( response_id="resp_failed", status="failed", - error={"code": 502, "message": "model blew up"}, + error={"code": code, "message": "model blew up"}, ), } return sse_response([event]) @@ -1098,12 +1092,13 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) assert span.status.status_code == StatusCode.ERROR + assert attrs(span)["error.type"] == "RuntimeError" events_recorded = list(span.events) assert len(events_recorded) == 1 assert events_recorded[0].name == "exception" event_attrs = dict(events_recorded[0].attributes or {}) assert event_attrs["exception.type"] == "RuntimeError" - assert event_attrs["exception.message"] == "response.failed: 502: model blew up" + assert event_attrs["exception.message"] == f"response.failed: {code}: model blew up" def test_responses_stream_error_event_retains_numeric_code(memory: SimpleNamespace) -> None: @@ -1623,6 +1618,7 @@ def test_responses_stream_capture_rejects_sparse_annotation_indexes( "url": "https://example.test/unsafe", }, }, + completed_without_output(3), ] def handler(request: httpx.Request) -> httpx.Response: @@ -1631,9 +1627,7 @@ def handler(request: httpx.Request) -> httpx.Response: stream = cast(Any, wrap_open_router(sync_client(handler)).responses.send)( model="openai/gpt-4o-mini", input="annotate", stream=True ) - for _ in events: - next(stream) - stream.close() + list(stream) a = attrs(only_span(memory)) assert json.loads(str(a["gen_ai.output.messages"])) == [ @@ -1648,9 +1642,11 @@ def handler(request: httpx.Request) -> httpx.Response: assert a["telemetry.dev.capture.truncated"] is True -def test_responses_stream_boolean_indexes_do_not_collide_with_index_one() -> None: - state_type = vars(telemetry_dev_openrouter)["_ResponsesStreamState"] - state = state_type() +def test_responses_stream_boolean_indexes_do_not_collide_with_index_one( + memory: SimpleNamespace, +) -> None: + # The SDK coerces a JSON true index to 1 on events that validate, so the boolean-index + # events below omit item_id: they fail validation and arrive raw. events: list[dict[str, Any]] = [ { "type": "response.output_item.added", @@ -1675,11 +1671,9 @@ def test_responses_stream_boolean_indexes_do_not_collide_with_index_one() -> Non { "type": "response.output_text.delta", "sequence_number": 3, - "item_id": "message_zero", "output_index": True, "content_index": True, "delta": "Content zero", - "logprobs": [], }, { "type": "response.reasoning_summary_part.added", @@ -1692,17 +1686,21 @@ def test_responses_stream_boolean_indexes_do_not_collide_with_index_one() -> Non { "type": "response.reasoning_summary_text.delta", "sequence_number": 5, - "item_id": "reasoning_two", "output_index": 2, "summary_index": True, "delta": "Summary zero", }, ] - for event in events: - state.record(event) + def handler(request: httpx.Request) -> httpx.Response: + return sse_response(events) - assert state.partial["output"] == [ + stream = cast(Any, wrap_open_router(sync_client(handler)).responses.send)( + model="openai/gpt-4o-mini", input="indexes", stream=True + ) + list(stream) + + assert json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) == [ { "id": "message_zero", "type": "message", @@ -2000,6 +1998,7 @@ def test_final_responses_text_replaces_delta_truncated_output_with_fresh_budget( "text": "Final answer", "logprobs": [], }, + completed_without_output(len(deltas) + 1), ] def handler(request: httpx.Request) -> httpx.Response: @@ -2020,10 +2019,7 @@ def handler(request: httpx.Request) -> httpx.Response: "content": [{"type": "output_text", "text": "Final answer", "annotations": []}], } ] - # The done event resolves the cut deltas; the flag remains because no terminal - # response arrived. - assert stream._state._budget.truncated is False - assert a["telemetry.dev.capture.truncated"] is True + assert "telemetry.dev.capture.truncated" not in a def test_final_responses_text_preserves_provider_event_truncation( @@ -2047,6 +2043,7 @@ def test_final_responses_text_preserves_provider_event_truncation( "text": "Final answer", "logprobs": [], }, + completed_without_output(3), ] def handler(request: httpx.Request) -> httpx.Response: diff --git a/sdks/python/src/telemetry_dev/_metrics.py b/sdks/python/src/telemetry_dev/_metrics.py index ceb89a5..6aebff4 100644 --- a/sdks/python/src/telemetry_dev/_metrics.py +++ b/sdks/python/src/telemetry_dev/_metrics.py @@ -72,8 +72,7 @@ class _ChunkAggregate: class OutputChunkAggregation: - def __init__(self, max_attribute_sets: int = _MAX_CHUNK_ATTRIBUTE_SETS) -> None: - self._max_attribute_sets = max_attribute_sets + def __init__(self) -> None: self._values: dict[tuple[tuple[str, Any], ...], _ChunkAggregate] = {} self._lock = threading.Lock() self._collection_start = time.time_ns() @@ -85,7 +84,7 @@ def add( ) -> None: key = tuple(sorted(attributes.items())) with self._lock: - if key not in self._values and len(self._values) >= self._max_attribute_sets - 1: + if key not in self._values and len(self._values) >= _MAX_CHUNK_ATTRIBUTE_SETS - 1: key = (("otel.metric.overflow", True),) count, total, minimum, maximum, buckets = aggregate existing = self._values.get(key) diff --git a/sdks/python/tests/otlp_capture.py b/sdks/python/tests/otlp_capture.py index 5b08844..7a88773 100644 --- a/sdks/python/tests/otlp_capture.py +++ b/sdks/python/tests/otlp_capture.py @@ -41,7 +41,8 @@ def do_POST(self) -> None: body = self.rfile.read(length) captured.append( CapturedRequest( - path=self.path, + # self.path collapses a leading "//" (gh-87389); keep the raw target. + path=self.requestline.split(" ")[1], headers={key.lower(): value for key, value in self.headers.items()}, body=body, ) @@ -55,7 +56,9 @@ def log_message(self, format: str, *args: object) -> None: pass server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) thread.start() try: host, port = server.server_address[0], server.server_address[1] diff --git a/sdks/python/tests/test_config.py b/sdks/python/tests/test_config.py index b7f8fb6..44cd783 100644 --- a/sdks/python/tests/test_config.py +++ b/sdks/python/tests/test_config.py @@ -1,7 +1,5 @@ from __future__ import annotations -import pytest - from telemetry_dev._config import resolve_config @@ -11,42 +9,3 @@ def test_defaults_without_env() -> None: assert config.base_url == "https://ingest.telemetry.dev" assert config.environment == "production" assert config.service_name == "unknown_service" - - -def test_env_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_env") - monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", "https://example.test") - monkeypatch.setenv("TELEMETRY_DEV_ENVIRONMENT", "staging") - monkeypatch.setenv("OTEL_SERVICE_NAME", "svc") - config = resolve_config() - assert config.api_key == "td_live_env" - assert config.base_url == "https://example.test" - assert config.environment == "staging" - assert config.service_name == "svc" - - -def test_arguments_take_precedence_over_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_env") - monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", "https://env.test") - monkeypatch.setenv("TELEMETRY_DEV_ENVIRONMENT", "staging") - monkeypatch.setenv("OTEL_SERVICE_NAME", "env-svc") - config = resolve_config( - api_key="td_live_arg", - base_url="https://arg.test", - environment="dev", - service_name="arg-svc", - ) - assert config.api_key == "td_live_arg" - assert config.base_url == "https://arg.test" - assert config.environment == "dev" - assert config.service_name == "arg-svc" - - -def test_base_url_trailing_slashes_stripped() -> None: - assert resolve_config(base_url="https://x.test/").base_url == "https://x.test" - assert resolve_config(base_url="https://x.test///").base_url == "https://x.test" - - -def test_base_url_env_trailing_slash_stripped(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", "https://env.test//") - assert resolve_config().base_url == "https://env.test" diff --git a/sdks/python/tests/test_init.py b/sdks/python/tests/test_init.py index 85c5d13..2b21214 100644 --- a/sdks/python/tests/test_init.py +++ b/sdks/python/tests/test_init.py @@ -12,23 +12,13 @@ from tests.conftest import MakeClient -def test_no_api_key_is_silent_noop() -> None: - client = telemetry_dev.init(log_level="silent", disable_atexit=True) - assert client.enabled is False - handle = telemetry_dev.start_span("x", type="generation", input="hi") - assert handle.traceparent() is None - handle.update(output="y") - handle.end() - telemetry_dev.log("nope") - telemetry_dev.update_current_span(output="y") - telemetry_dev.flush() - - def test_enabled_false_kill_switch_beats_exporter_seams(make: MakeClient) -> None: - env = make(enabled=False) + env = make(enabled=False, session_mode="process") assert env.client.enabled is False telemetry_dev.start_span("x").end() + telemetry_dev.log("x") assert env.span_exporter.get_finished_spans() == () + assert env.log_exporter.get_finished_logs() == () @pytest.mark.parametrize("max_attribute_length", [-1, 1.5, float("nan"), float("inf"), True]) @@ -92,19 +82,6 @@ def test_raw_attributes_follow_the_attribute_backstop_not_the_content_cap( assert attributes[key] == expected -def test_exporter_seam_enables_without_api_key(make: MakeClient) -> None: - env = make() - assert env.client.enabled is True - telemetry_dev.start_span("x").end() - assert len(env.span_exporter.get_finished_spans()) == 1 - - -def test_api_key_from_env_enables(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_test") - client = telemetry_dev.init(log_level="silent", disable_atexit=True) - assert client.enabled is True - - def test_double_init_replaces_previous_client(make: MakeClient) -> None: first = make() second = make() @@ -140,14 +117,6 @@ def test_isolated_by_default(make: MakeClient) -> None: assert env.span_exporter.get_finished_spans() == () -def test_register_global_sets_global_provider( - make: MakeClient, monkeypatch: pytest.MonkeyPatch -) -> None: - _reset_global_tracer_provider(monkeypatch) - env = make(register_global=True) - assert trace.get_tracer_provider() is env.client._tracer_provider # pyright: ignore[reportPrivateUsage] - - def test_register_global_default_filter_is_scope_name_only( make: MakeClient, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/sdks/python/tests/test_logs.py b/sdks/python/tests/test_logs.py index ad0abae..acf3942 100644 --- a/sdks/python/tests/test_logs.py +++ b/sdks/python/tests/test_logs.py @@ -43,20 +43,6 @@ def test_default_level_is_info(memory: SimpleNamespace) -> None: assert record.severity_number.value == 9 -def test_body_is_message(memory: SimpleNamespace) -> None: - telemetry_dev.log("hello world") - assert only_log(memory).log_record.body == "hello world" - - -def test_trace_correlation_inside_span(memory: SimpleNamespace) -> None: - with telemetry_dev.start_span("parent") as handle: - telemetry_dev.log("inside") - record = only_log(memory).log_record - span_context = handle.span.get_span_context() - assert record.trace_id == span_context.trace_id - assert record.span_id == span_context.span_id - - def test_no_trace_correlation_outside_span(memory: SimpleNamespace) -> None: telemetry_dev.log("outside") record = only_log(memory).log_record @@ -64,11 +50,6 @@ def test_no_trace_correlation_outside_span(memory: SimpleNamespace) -> None: assert not record.span_id -def test_event_name(memory: SimpleNamespace) -> None: - telemetry_dev.log("msg", event_name="cache.miss") - assert only_log(memory).log_record.event_name == "cache.miss" - - def test_custom_attributes(memory: SimpleNamespace) -> None: telemetry_dev.log("msg", attributes={"request.id": "r-1", "retry": 2}) log_attrs = dict(only_log(memory).log_record.attributes or {}) @@ -110,12 +91,6 @@ def test_propagated_attributes_use_client_attribute_cap(make: MakeClient) -> Non assert value == "z" * (30 - len(TRUNCATION_MARKER)) + TRUNCATION_MARKER -def test_log_records_use_sdk_scope(memory: SimpleNamespace) -> None: - telemetry_dev.log("msg") - scope = only_log(memory).instrumentation_scope - assert scope is not None and scope.name == "telemetry_dev" - - def test_log_is_noop_without_init() -> None: telemetry_dev.log("nowhere") diff --git a/sdks/python/tests/test_metrics.py b/sdks/python/tests/test_metrics.py index b5d1f20..9eb6df8 100644 --- a/sdks/python/tests/test_metrics.py +++ b/sdks/python/tests/test_metrics.py @@ -3,7 +3,6 @@ import gc import math import threading -import time from types import SimpleNamespace from weakref import ref @@ -191,6 +190,8 @@ def keep(span: ReadableSpan) -> bool: telemetry_dev.start_span("filtered", type="generation").record_output_chunk( 1000 ).record_output_chunk(9000).end(error=RuntimeError("filtered failure")) + telemetry_dev.start_span("zero", type="generation").end() + telemetry_dev.start_span("one", type="generation").record_output_chunk(1000).end() handle = telemetry_dev.start_span( "stream", type="generation", model="gpt-4o", provider="openai" ) @@ -288,22 +289,6 @@ def close() -> None: client.shutdown() -def test_output_chunk_zero_or_one_and_filter_rejection_emit_nothing(make: MakeClient) -> None: - def keep(span: ReadableSpan) -> bool: - return span.name != "rejected" - - errors: list[BaseException] = [] - env = make(span_filter=keep, on_error=errors.append) - telemetry_dev.start_span("zero", type="generation").end() - one = telemetry_dev.start_span("one", type="generation") - one.record_output_chunk(1000).end() - rejected = telemetry_dev.start_span("rejected", type="generation") - rejected.record_output_chunk(1000).record_output_chunk(1020).end() - - assert env.client._output_chunks is None - assert errors == [] - - def test_rejected_chunk_state_does_not_retain_abandoned_handles(make: MakeClient) -> None: def reject(span: ReadableSpan) -> bool: return False @@ -318,15 +303,15 @@ def reject(span: ReadableSpan) -> bool: def test_output_chunk_attribute_cardinality_is_bounded() -> None: - aggregation = OutputChunkAggregation(max_attribute_sets=2) + aggregation = OutputChunkAggregation() value = (1, 0.01, 0.01, 0.01, (1, *([0] * len(DURATION_BUCKETS)))) - for model in ("a", "b", "c", "d"): - aggregation.add({"gen_ai.request.model": model}, value) + for index in range(2001): + aggregation.add({"gen_ai.request.model": f"model-{index}"}, value) points = aggregation.drain_points() - assert len(points) == 2 + assert len(points) == 2000 overflow = next(point for point in points if point.attributes == {"otel.metric.overflow": True}) - assert overflow.count == 3 + assert overflow.count == 2 def test_ended_child_metrics_reference_child_not_active_parent(memory: SimpleNamespace) -> None: @@ -393,24 +378,27 @@ def test_plain_span_records_nothing(memory: SimpleNamespace) -> None: def test_quiet_interval_emits_zero_metric_posts( monkeypatch: pytest.MonkeyPatch, internal_metrics: bool ) -> None: - # Empty-datapoint guard: collections with no data points are never POSTed, even when - # OpenTelemetry records its own metric-reader metrics on the provider. + # Quiet collections are never POSTed, even when OpenTelemetry records its own metric-reader + # metrics on the provider: the otel.sdk.* drop View keeps them out, and the reader skips + # the exporter when a collection is empty. The second flush would carry the internal + # metrics that the first one recorded. # Same exporter, reader, and provider wiring the client uses, against a real local server. if internal_metrics: monkeypatch.setenv("OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED", "true") with otlp_capture_server() as server: exporter = GuardedOTLPMetricExporter(endpoint=f"{server.url}/v1/metrics") - reader = PeriodicExportingMetricReader(exporter, export_interval_millis=50) + reader = PeriodicExportingMetricReader(exporter, export_interval_millis=2**31 - 1) provider = sdk_meter_provider(reader, Resource.create()) meter = provider.get_meter("quiet") histogram = meter.create_histogram("gen_ai.client.operation.duration", unit="s") - time.sleep(0.3) # several quiet collection intervals + provider.force_flush() + provider.force_flush() assert server.requests_for("/v1/metrics") == [] histogram.record(0.5, {"gen_ai.operation.name": "chat"}) provider.force_flush() - assert len(server.requests_for("/v1/metrics")) >= 1 + assert len(server.requests_for("/v1/metrics")) == 1 provider.shutdown() diff --git a/sdks/python/tests/test_otel.py b/sdks/python/tests/test_otel.py index 718d4e8..4bb128f 100644 --- a/sdks/python/tests/test_otel.py +++ b/sdks/python/tests/test_otel.py @@ -137,19 +137,10 @@ def test_create_span_exporter_posts_traces_with_bearer_gzip() -> None: assert request.headers["content-encoding"] == "gzip" -def test_no_op_without_key(caplog: pytest.LogCaptureFixture) -> None: +def test_no_posts_keyless_even_with_base_url_env( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: caplog.set_level(logging.DEBUG, logger="telemetry_dev") - processor = TelemetrySpanProcessor() - provider = provider_with(processor) - try: - end_span(provider, "noop") - assert processor.force_flush() is True - finally: - provider.shutdown() - assert "no-op" in caplog.text - - -def test_no_posts_keyless_even_with_base_url_env(monkeypatch: pytest.MonkeyPatch) -> None: with otlp_capture_server() as server: monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", server.url) processor = TelemetrySpanProcessor() @@ -160,6 +151,7 @@ def test_no_posts_keyless_even_with_base_url_env(monkeypatch: pytest.MonkeyPatch finally: provider.shutdown() assert server.requests == [] + assert "no-op" in caplog.text def test_env_var_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None: @@ -185,7 +177,7 @@ def test_kwargs_beat_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", f"{server.url}/env") processor = TelemetrySpanProcessor( api_key="td_live_kwarg", - base_url=server.url, + base_url=f"{server.url}//", export_mode="immediate", metrics=False, ) diff --git a/sdks/python/tests/test_process_session.py b/sdks/python/tests/test_process_session.py index aa1523c..6e9c21b 100644 --- a/sdks/python/tests/test_process_session.py +++ b/sdks/python/tests/test_process_session.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -from types import SimpleNamespace from typing import Any, cast from opentelemetry.trace import SpanContext, TraceFlags, TraceState @@ -36,7 +35,9 @@ def test_explicit_parent_session_wins_for_children_and_logs(make: MakeClient) -> ): telemetry_dev.start_span("explicit-child").end() telemetry_dev.log("explicit-log") - for span in env.span_exporter.get_finished_spans(): + spans = env.span_exporter.get_finished_spans() + assert [span.name for span in spans] == ["explicit-child", "explicit-root"] + for span in spans: assert attrs(span)["gen_ai.conversation.id"] == "explicit" (record,) = env.log_exporter.get_finished_logs() assert attrs(record.log_record)["gen_ai.conversation.id"] == "explicit" @@ -56,7 +57,9 @@ async def main() -> None: await asyncio.gather(emit("one"), emit("two")) asyncio.run(main()) - for span in env.span_exporter.get_finished_spans(): + spans = env.span_exporter.get_finished_spans() + assert sorted(span.name for span in spans) == ["one-child", "one-root", "two-child", "two-root"] + for span in spans: assert attrs(span)["gen_ai.conversation.id"] == span.name.split("-")[0] @@ -77,10 +80,16 @@ def test_process_mode_honors_remote_parent(make: MakeClient) -> None: def test_reinit_generates_a_new_process_session(make: MakeClient) -> None: first = make(api_key="td_live_k", session_mode="process") - first_id = first.client._process_session_id # pyright: ignore[reportPrivateUsage] + telemetry_dev.start_span("first").end() + (first_span,) = first.span_exporter.get_finished_spans() telemetry_dev.shutdown() second = make(api_key="td_live_k", session_mode="process") - assert second.client._process_session_id != first_id # pyright: ignore[reportPrivateUsage] + telemetry_dev.start_span("second").end() + (second_span,) = second.span_exporter.get_finished_spans() + first_id = attrs(first_span)["gen_ai.conversation.id"] + second_id = attrs(second_span)["gen_ai.conversation.id"] + assert isinstance(first_id, str) and isinstance(second_id, str) + assert first_id != second_id def test_invalid_mode_fails_open_via_on_error() -> None: @@ -94,8 +103,3 @@ def test_invalid_mode_fails_open_via_on_error() -> None: ) assert client.enabled is False assert isinstance(errors[0], ValueError) - - -def test_disabled_process_mode_is_a_noop(make: MakeClient) -> None: - env: SimpleNamespace = make(enabled=False, session_mode="process") - assert env.client._process_session_id is None # pyright: ignore[reportPrivateUsage] diff --git a/sdks/python/tests/test_propagation.py b/sdks/python/tests/test_propagation.py index cd21112..b68c521 100644 --- a/sdks/python/tests/test_propagation.py +++ b/sdks/python/tests/test_propagation.py @@ -23,7 +23,7 @@ from opentelemetry.util.types import Attributes import telemetry_dev -from telemetry_dev._context import session_span_context, with_session_parent +from telemetry_dev._context import session_span_context from telemetry_dev._serialize import MaskContext from tests.conftest import MakeClient, span_context @@ -131,15 +131,6 @@ def test_get_traceparent_round_trip(memory: SimpleNamespace) -> None: assert telemetry_dev.get_traceparent() is None -def test_logs_get_propagated_attributes(memory: SimpleNamespace) -> None: - with telemetry_dev.propagate_attributes(user_id="u-log", metadata={"plan": "pro"}): - telemetry_dev.log("hello") - (record,) = memory.log_exporter.get_finished_logs() - log_attrs = dict(record.log_record.attributes or {}) - assert log_attrs["user.id"] == "u-log" - assert log_attrs["td.metadata.plan"] == "pro" - - def test_invalid_traceparent_falls_back_to_active_context(memory: SimpleNamespace) -> None: with telemetry_dev.start_span("root") as root: telemetry_dev.start_span("child", parent="not-a-traceparent").end() @@ -381,9 +372,7 @@ def mask(_value: object, _context: MaskContext) -> str: assert "gen_ai.output.messages" not in a -def test_session_sampling_keeps_baggage_changed_after_attaching_the_session_parent( - make: MakeClient, -) -> None: +def test_session_root_sampling_sees_caller_baggage(make: MakeClient) -> None: class TenantSampler(Sampler): def should_sample( self, @@ -404,10 +393,12 @@ def get_description(self) -> str: return "TenantSampler" env = make(api_key="td_live_k", sampler=ParentBased(TenantSampler())) - before = baggage.set_baggage("tenant", "denied", Context()) - parent = with_session_parent(before, "s1", "td_live_k") - allowed = baggage.set_baggage("tenant", "allowed", parent) - telemetry_dev.start_span("allowed", parent=allowed).end() - telemetry_dev.start_span("denied", parent=parent).end() - telemetry_dev.start_span("removed", parent=baggage.remove_baggage("tenant", allowed)).end() - assert [span.name for span in env.span_exporter.get_finished_spans()] == ["allowed"] + allowed = baggage.set_baggage("tenant", "allowed", Context()) + denied = baggage.set_baggage("tenant", "denied", Context()) + with telemetry_dev.propagate_attributes(session_id="s1"): + telemetry_dev.start_span("allowed", parent=allowed).end() + telemetry_dev.start_span("denied", parent=denied).end() + telemetry_dev.start_span("removed", parent=baggage.remove_baggage("tenant", allowed)).end() + (span,) = env.span_exporter.get_finished_spans() + assert span.name == "allowed" + assert span_context(span).trace_id == session_span_context("td_live_k", "s1").trace_id diff --git a/sdks/python/tests/test_wire.py b/sdks/python/tests/test_wire.py index f16bd2e..1fc048e 100644 --- a/sdks/python/tests/test_wire.py +++ b/sdks/python/tests/test_wire.py @@ -110,13 +110,33 @@ def test_log_export_wire_format(server: CaptureServer) -> None: def test_no_posts_when_disabled() -> None: with otlp_capture_server() as capture: - telemetry_dev.init( + client = telemetry_dev.init( base_url=capture.url, log_level="silent", disable_atexit=True, ) - telemetry_dev.start_span("nope", type="generation", input="x").end() + assert client.enabled is False + handle = telemetry_dev.start_span("nope", type="generation", input="x") + assert handle.traceparent() is None + handle.update(output="y") + handle.end() + telemetry_dev.update_current_span(output="y") telemetry_dev.log("nope") telemetry_dev.flush() telemetry_dev.shutdown() assert capture.requests == [] + + +def test_env_api_key_and_base_url_reach_the_wire(monkeypatch: pytest.MonkeyPatch) -> None: + with otlp_capture_server() as capture: + monkeypatch.setenv("TELEMETRY_DEV_API_KEY", "td_live_env") + monkeypatch.setenv("TELEMETRY_DEV_BASE_URL", f"{capture.url}//") + client = telemetry_dev.init( + export_mode="immediate", log_level="silent", disable_atexit=True + ) + assert client.enabled is True + telemetry_dev.start_span("env").end() + telemetry_dev.flush() + (request,) = capture.requests_for("/v1/traces") + assert request.headers["authorization"] == "Bearer td_live_env" + telemetry_dev.shutdown() From cb37f7671973ca08be464e0ea65296a3f5cb5796 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Mon, 26 Oct 2026 18:10:09 +0000 Subject: [PATCH 3/5] fix(sdk-anthropic): record fallback blocks under their wire keys The SDK model keeps the "from" field of a fallback block under the Python name "from_", and it also keeps the wire key "from" as an extra field. The captured output of the unary and the stream paths had the two keys. The unary path now dumps the model by alias. The stream path changes field names to their aliases from the class field table, so that no property runs during the bounded read. --- .../src/telemetry_dev_anthropic/__init__.py | 19 ++++++++++++++++++- sdks/python-anthropic/tests/test_anthropic.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index ba6baba..3319fd5 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -139,7 +139,7 @@ def _raw_response_requested(params: Mapping[str, Any]) -> bool: def _native(value: Any) -> Any: model_dump = getattr(value, "model_dump", None) if callable(model_dump): - return model_dump(mode="json", exclude_none=True) + return model_dump(mode="json", by_alias=True, exclude_none=True) if isinstance(value, Mapping): mapping = cast(Mapping[Any, Any], value) return {str(key): _native(item) for key, item in mapping.items() if item is not None} @@ -575,15 +575,30 @@ def _bounded_stream_native( remaining_items, depth, seen, + _field_aliases(value), ) +def _field_aliases(value: Any) -> dict[str, str]: + # Pydantic keeps an aliased field under its Python name (`from_` for the wire key + # `from`). Read the class's field table directly, so that no property runs. + for klass in _TYPE_MRO.__get__(type(cast(object, value))): + namespace = _TYPE_NAMESPACE.__get__(klass) + fields = namespace.get("__pydantic_fields__", namespace.get("model_fields")) + if type(fields) is dict: + items = cast(dict[str, Any], fields).items() + aliases = {name: _stored(field, "alias") for name, field in items} + return {name: alias for name, alias in aliases.items() if type(alias) is str} + return {} + + def _bounded_stream_mappings( mappings: tuple[Mapping[Any, Any], ...], source_id: int, remaining_items: list[int], depth: int, seen: set[int], + aliases: Mapping[str, str] | None = None, ) -> dict[str, Any]: if sum(len(mapping) for mapping in mappings) * 2 > remaining_items[0] or source_id in seen: raise _StreamCaptureLimit @@ -596,6 +611,8 @@ def _bounded_stream_mappings( if remaining_items[0] < 2: raise _StreamCaptureLimit remaining_items[0] -= 1 + if aliases: + key = aliases.get(key, key) if key in seen_keys: remaining_items[0] -= 1 continue diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index 3655220..bdb0152 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -2309,6 +2309,20 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: assert a["telemetry.dev.capture.truncated"] is True +def test_beta_create_records_a_fallback_block_under_its_wire_keys(memory: SimpleNamespace) -> None: + content = [FALLBACK_BLOCK, {"type": "text", "text": "Hi"}] + + def handler(request: httpx.Request) -> httpx.Response: + return json_response(message_payload(content=content)) + + wrapped_sync_client(handler).beta.messages.create( + model="claude-opus-5", max_tokens=64, messages=MESSAGES + ) + + output = json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) + assert output[0]["content"] == content + + @pytest.mark.parametrize( ("iterations", "expected"), [ From 3b45e2173dbbecb0d06cbf9a401aaad18aec397f Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Tue, 27 Oct 2026 09:24:00 +0000 Subject: [PATCH 4/5] fix(sdk-anthropic): bound stream field alias inspection --- .../src/telemetry_dev_anthropic/__init__.py | 16 +++---- sdks/python-anthropic/tests/test_anthropic.py | 43 ++++++++++++++++++- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py index 3319fd5..c09fc39 100644 --- a/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py +++ b/sdks/python-anthropic/src/telemetry_dev_anthropic/__init__.py @@ -575,20 +575,18 @@ def _bounded_stream_native( remaining_items, depth, seen, - _field_aliases(value), + _model_fields(value), ) -def _field_aliases(value: Any) -> dict[str, str]: +def _model_fields(value: Any) -> Mapping[str, Any]: # Pydantic keeps an aliased field under its Python name (`from_` for the wire key # `from`). Read the class's field table directly, so that no property runs. for klass in _TYPE_MRO.__get__(type(cast(object, value))): namespace = _TYPE_NAMESPACE.__get__(klass) fields = namespace.get("__pydantic_fields__", namespace.get("model_fields")) if type(fields) is dict: - items = cast(dict[str, Any], fields).items() - aliases = {name: _stored(field, "alias") for name, field in items} - return {name: alias for name, alias in aliases.items() if type(alias) is str} + return cast(dict[str, Any], fields) return {} @@ -598,7 +596,7 @@ def _bounded_stream_mappings( remaining_items: list[int], depth: int, seen: set[int], - aliases: Mapping[str, str] | None = None, + fields: Mapping[str, Any] | None = None, ) -> dict[str, Any]: if sum(len(mapping) for mapping in mappings) * 2 > remaining_items[0] or source_id in seen: raise _StreamCaptureLimit @@ -611,8 +609,10 @@ def _bounded_stream_mappings( if remaining_items[0] < 2: raise _StreamCaptureLimit remaining_items[0] -= 1 - if aliases: - key = aliases.get(key, key) + if fields: + alias = _stored(fields.get(key), "alias") + if type(alias) is str: + key = alias if key in seen_keys: remaining_items[0] -= 1 continue diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index bdb0152..26708f6 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -23,7 +23,7 @@ from anthropic.types.beta import BetaMessageParam from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.trace import StatusCode -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field import telemetry_dev_anthropic from telemetry_dev_anthropic import instrument_anthropic, uninstrument_anthropic, wrap_anthropic @@ -2911,6 +2911,47 @@ class Model(BaseModel): assert bounded == {"type": "text", "citations_extra": "kept"} +def test_stream_capture_alias_inspection_is_bounded( + memory: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + implementation = cast(Any, telemetry_dev_anthropic) + stored = implementation._stored + alias_reads = 0 + + def counted_stored(value: Any, name: str) -> Any: + nonlocal alias_reads + if name == "alias": + alias_reads += 1 + return stored(value, name) + + class SparseModel: + __pydantic_fields__ = { + f"unused_{index}": Field(alias=f"unused-{index}") for index in range(5_000) + } | {"from_": Field(alias="from")} + + def __init__(self) -> None: + self.type = "fallback" + self.from_ = {"model": "claude-opus-5"} + + monkeypatch.setattr(implementation, "_stored", counted_stored) + events = [ + {"type": "content_block_start", "index": 0, "content_block": SparseModel()}, + {"type": "message_stop"}, + ] + + def create(**_kwargs: Any) -> Iterator[Any]: + return iter(events) + + client: Any = object.__new__(Anthropic) + client.messages = SimpleNamespace(create=create) + delivered = list(wrap_anthropic(client).messages.create(model="claude-opus-5", stream=True)) + + assert delivered == events + output = json.loads(str(attrs(only_span(memory))["gen_ai.output.messages"])) + assert output[0]["content"] == [{"type": "fallback", "from": {"model": "claude-opus-5"}}] + assert alias_reads <= 1_000 + + def _spoofed_and_aliased_storage(calls: list[str]) -> list[Any]: class Payload: def __get__(self, instance: Any, owner: Any) -> dict[str, Any]: From 84b2be2adf07cd92c04570a9cf6a0347db15dbdc Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Tue, 27 Oct 2026 09:24:03 +0000 Subject: [PATCH 5/5] test(sdk-openai): protect disabled transcription capture --- sdks/python-openai/tests/test_openai.py | 26 +++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index d2b4b40..238c7e6 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -1039,17 +1039,34 @@ def test_terminal_transcription_capture_is_incrementally_bounded( @pytest.mark.parametrize("streaming", [False, True]) @pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("capture_output", [False, True]) async def test_transcription_capture_uses_the_policy_that_created_its_span( - make: Any, monkeypatch: pytest.MonkeyPatch, streaming: bool, async_mode: bool + make: Any, + monkeypatch: pytest.MonkeyPatch, + streaming: bool, + async_mode: bool, + capture_output: bool, ) -> None: - memory = make(capture_output=True) + memory = make(capture_output=capture_output) + text_reads = 0 + original_getattribute = BaseModel.__getattribute__ + + def counted_getattribute(self: BaseModel, name: str) -> Any: + nonlocal text_reads + if name == "text": + text_reads += 1 + return original_getattribute(self, name) + + monkeypatch.setattr(BaseModel, "__getattribute__", counted_getattribute) def response() -> httpx.Response: # The span has started by now; a later global policy must not apply to it. monkeypatch.setattr( telemetry_dev, "get_client", - lambda: SimpleNamespace(capture_output=False, capture_input=False, mask=None), + lambda: SimpleNamespace( + capture_output=not capture_output, capture_input=False, mask=None + ), ) if streaming: return sse_response([{"type": "transcript.text.done", "text": "sensitive transcript"}]) @@ -1076,7 +1093,8 @@ async def handler(_request: httpx.Request) -> httpx.Response: list(result) a = attrs(only_span(memory)) - assert a["gen_ai.output.messages"] == "sensitive transcript" + assert a.get("gen_ai.output.messages") == ("sensitive transcript" if capture_output else None) + assert (text_reads > 0) is capture_output assert "telemetry.dev.capture.truncated" not in a