From 50789edad1bbf16ebd59630e10ef326f85a63aa3 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:27:26 +0000 Subject: [PATCH 1/2] feat: Instrument OpenAI agents api (`openai.beta.agents`) --- e2e/config/pr-comment-scenarios.json | 15 + .../openai-agents-api-v7-latest.cassette.json | 92 ++ .../openai-agents-api-v7.cassette.json | 92 ++ ...openai-agents-api-v7-latest.span-tree.json | 53 ++ .../openai-agents-api-v7-latest.span-tree.txt | 36 + .../openai-agents-api-v7.span-tree.json | 53 ++ .../openai-agents-api-v7.span-tree.txt | 36 + .../assertions.ts | 97 +++ .../cassette-filter.mjs | 3 + .../constants.mjs | 9 + .../package.json | 19 + .../pnpm-lock.yaml | 46 + .../scenario.mjs | 112 +++ .../scenario.test.ts | 61 ++ js/src/exports.ts | 6 + .../openai-agents-api-instrumentation.test.ts | 400 +++++++++ .../openai-agents-api-instrumentation.ts | 816 ++++++++++++++++++ .../plugins/openai-batch-instrumentation.ts | 25 +- .../plugins/openai-channels.ts | 31 + .../openai-manual-instrumentation-utils.ts | 22 + .../instrumentation/plugins/openai-plugin.ts | 12 + js/src/logger.ts | 55 ++ js/src/openai-agents-api-types.ts | 50 ++ js/src/openai-agents-api.ts | 307 +++++++ .../wrappers/ai-sdk/harness-agent-context.ts | 29 +- 25 files changed, 2432 insertions(+), 45 deletions(-) create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7-latest.cassette.json create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7.cassette.json create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.json create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.txt create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.json create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.txt create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/assertions.ts create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/cassette-filter.mjs create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/constants.mjs create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/package.json create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/pnpm-lock.yaml create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/scenario.mjs create mode 100644 e2e/scenarios/openai-agents-api-instrumentation/scenario.test.ts create mode 100644 js/src/instrumentation/plugins/openai-agents-api-instrumentation.test.ts create mode 100644 js/src/instrumentation/plugins/openai-agents-api-instrumentation.ts create mode 100644 js/src/instrumentation/plugins/openai-manual-instrumentation-utils.ts create mode 100644 js/src/openai-agents-api-types.ts create mode 100644 js/src/openai-agents-api.ts diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 73030708b..a13b89416 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -166,6 +166,21 @@ } ] }, + { + "scenarioDirName": "openai-agents-api-instrumentation", + "label": "OpenAI Agents API Instrumentation", + "metadataScenario": "openai-agents-api-instrumentation", + "variants": [ + { + "variantKey": "openai-agents-api-v7", + "label": "v7 pinned" + }, + { + "variantKey": "openai-agents-api-v7-latest", + "label": "v7 latest" + } + ] + }, { "scenarioDirName": "anthropic-instrumentation", "label": "Anthropic Instrumentation", diff --git a/e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7-latest.cassette.json b/e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7-latest.cassette.json new file mode 100644 index 000000000..906ee036a --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7-latest.cassette.json @@ -0,0 +1,92 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "49d07f1d39cd5c3d", + "matchKey": "POST api.openai.com/v1/agents/sessions", + "recordedAt": "2026-09-11T11:18:39.262Z", + "request": { + "body": { + "kind": "json", + "value": { + "agent": { + "instructions": "You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna", + "model": "gpt-5.6-luna", + "reasoning": { + "effort": "none" + }, + "tools": [ + { + "type": "web_search" + } + ] + }, + "environment": { + "type": "none" + }, + "input": "Use web search once for Vienna weather, then answer exactly: Sunny in Vienna", + "metadata": { + "braintrust_correlation_id": "agents-e2e-correlation" + }, + "stream": true + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/agents/sessions" + }, + "response": { + "body": { + "chunks": [ + "event: agent.session.created\ndata: {\"type\":\"agent.session.created\",\"event_id\":\"evt_e6596ea1eca04f4e8cdafe16280ecbb5b6fbcfd5807247799f\",\"session\":{\"metadata\":{\"braintrust_correlation_id\":\"agents-e2e-correlation\"},\"id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"object\":\"agent.session\",\"created_at\":1789125508,\"last_active_at\":1789125508,\"status\":\"in_progress\",\"required_actions\":[],\"error\":null,\"agent\":{\"id\":\"agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad\",\"name\":null,\"model\":\"gpt-5.6-luna\",\"reasoning\":{\"effort\":\"none\",\"summary\":null},\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"service_tier\":\"auto\",\"instructions\":\"You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna\",\"tools\":[{\"type\":\"web_search\",\"mode\":\"live\",\"context_size\":\"medium\",\"allowed_domains\":null,\"location\":null}],\"multi_agent\":{\"enabled\":false,\"max_concurrent_subagents\":null}},\"environment\":{\"type\":\"none\"},\"vault_ids\":[],\"usage\":null}}", + "event: agent.session.turn.created\ndata: {\"type\":\"agent.session.turn.created\",\"event_id\":\"evt_87681e2e4a2b470e986c30869a57a8235c349da455c54797b1\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"turn\":{\"id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"object\":\"agent.session.turn\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"agent_id\":\"agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad\",\"subagent_id\":null,\"status\":\"queued\",\"created_at\":1789125511,\"started_at\":null,\"completed_at\":null,\"error\":null,\"usage\":null}}", + "event: agent.session.turn.item.added\ndata: {\"type\":\"agent.session.turn.item.added\",\"event_id\":\"evt_3e3aa15ed8164c22af8d98c12f9ac7012c52f492acff46bfbd\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"output_index\":null,\"item\":{\"type\":\"message\",\"id\":\"msg_e9e732621495fdbc83229e8052ff025d1e7db9580130517e82\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Use web search once for Vienna weather, then answer exactly: Sunny in Vienna\"}],\"status\":\"completed\",\"phase\":null}}", + "event: agent.session.in_progress\ndata: {\"type\":\"agent.session.in_progress\",\"event_id\":\"evt_afc3e19ad879464dbfe043c51293383050323b216d584c458e\",\"session\":{\"metadata\":{\"braintrust_correlation_id\":\"agents-e2e-correlation\"},\"id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"object\":\"agent.session\",\"created_at\":1789125508,\"last_active_at\":1789125508,\"status\":\"in_progress\",\"required_actions\":[],\"error\":null,\"agent\":{\"id\":\"agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad\",\"name\":null,\"model\":\"gpt-5.6-luna\",\"reasoning\":{\"effort\":\"none\",\"summary\":null},\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"service_tier\":\"auto\",\"instructions\":\"You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna\",\"tools\":[{\"type\":\"web_search\",\"mode\":\"live\",\"context_size\":\"medium\",\"allowed_domains\":null,\"location\":null}],\"multi_agent\":{\"enabled\":false,\"max_concurrent_subagents\":null}},\"environment\":{\"type\":\"none\"},\"vault_ids\":[],\"usage\":null}}", + "event: agent.session.turn.in_progress\ndata: {\"type\":\"agent.session.turn.in_progress\",\"event_id\":\"evt_09d03742146a40079750d2919f9c79ce84439eb416c545a398\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"turn\":{\"id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"object\":\"agent.session.turn\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"agent_id\":\"agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad\",\"subagent_id\":null,\"status\":\"in_progress\",\"created_at\":1789125511,\"started_at\":1789125511,\"completed_at\":null,\"error\":null,\"usage\":null}}", + "event: agent.session.turn.item.added\ndata: {\"type\":\"agent.session.turn.item.added\",\"event_id\":\"evt_4b7385a801de4a6fb20add136d871ff09713865086d7468eb4\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"output_index\":0,\"item\":{\"type\":\"web_search_call\",\"id\":\"exec-e6e064e0-faa6-4cfd-9892-13ccb553983d\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"status\":\"in_progress\",\"action\":{\"type\":\"search\",\"query\":\"\",\"queries\":null}}}", + "event: agent.session.turn.item.done\ndata: {\"type\":\"agent.session.turn.item.done\",\"event_id\":\"evt_0bd2db80117e4d639f0407395398af78a9baa2f24b1f4a9ab0\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"output_index\":0,\"item\":{\"type\":\"web_search_call\",\"id\":\"exec-e6e064e0-faa6-4cfd-9892-13ccb553983d\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"status\":\"completed\",\"action\":{\"type\":\"other\"}}}", + "event: agent.session.turn.item.added\ndata: {\"type\":\"agent.session.turn.item.added\",\"event_id\":\"evt_9809d861f43d495e93fb718674d81185419a568fb7514ab2a0\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"output_index\":1,\"item\":{\"type\":\"message\",\"id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"role\":\"assistant\",\"content\":[],\"status\":\"in_progress\",\"phase\":\"final_answer\"}}", + "event: agent.session.turn.content_part.added\ndata: {\"type\":\"agent.session.turn.content_part.added\",\"event_id\":\"evt_f6cdbe6fcbc74e14905caf9466d89efc4bfaac8b44a247a5a0\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"item_id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"output_index\":1,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\"}}", + "event: agent.session.turn.output_text.delta\ndata: {\"type\":\"agent.session.turn.output_text.delta\",\"event_id\":\"evt_98ec87d5e36941a7947d6f9eac3e19ae8efe61a94e1c43049f\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"item_id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"output_index\":1,\"content_index\":0,\"delta\":\"Sunny\"}", + "event: agent.session.turn.output_text.delta\ndata: {\"type\":\"agent.session.turn.output_text.delta\",\"event_id\":\"evt_83c6091929cc47c68ee7fe8335294b9d70831f362e9d4da6ae\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"item_id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"output_index\":1,\"content_index\":0,\"delta\":\" in\"}", + "event: agent.session.turn.output_text.delta\ndata: {\"type\":\"agent.session.turn.output_text.delta\",\"event_id\":\"evt_fb483f6b0de442a2b80b6ab730ebab8fafb2556c50f74ebd83\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"item_id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"output_index\":1,\"content_index\":0,\"delta\":\" Vienna\"}", + "event: agent.session.turn.output_text.done\ndata: {\"type\":\"agent.session.turn.output_text.done\",\"event_id\":\"evt_4d14d6fa4db74db89ea585edd4d20fb20e1461d354f34cf5ad\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"item_id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"output_index\":1,\"content_index\":0,\"text\":\"Sunny in Vienna\"}", + "event: agent.session.turn.content_part.done\ndata: {\"type\":\"agent.session.turn.content_part.done\",\"event_id\":\"evt_50131ead637d47dcad2b9a25c4007a72cb0d2bdcc25748e28e\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"item_id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"output_index\":1,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"Sunny in Vienna\"}}", + "event: agent.session.turn.item.done\ndata: {\"type\":\"agent.session.turn.item.done\",\"event_id\":\"evt_5fa955358fac43f1acc8a5d54ff2990aa85c207ac9c7422084\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"output_index\":1,\"item\":{\"type\":\"message\",\"id\":\"msg_00e8337f37800baf016aa3e389dd9487d19273d5cb18760362\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Sunny in Vienna\"}],\"phase\":\"final_answer\"}}", + "event: agent.session.turn.completed\ndata: {\"type\":\"agent.session.turn.completed\",\"event_id\":\"evt_4de21872928049a5a2b9132414ce4af733cf7a0b44c2493483\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"turn_id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"turn\":{\"id\":\"turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2\",\"object\":\"agent.session.turn\",\"session_id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"agent_id\":\"agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad\",\"subagent_id\":null,\"status\":\"completed\",\"created_at\":1789125511,\"started_at\":1789125511,\"completed_at\":1789125515,\"error\":null,\"usage\":null},\"usage\":null}", + "event: agent.session.idle\ndata: {\"type\":\"agent.session.idle\",\"event_id\":\"evt_04b8c75ce34a46ceafc4782d6e7c1fc891befe70ef77443bbf\",\"session\":{\"metadata\":{\"braintrust_correlation_id\":\"agents-e2e-correlation\"},\"id\":\"sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db\",\"object\":\"agent.session\",\"created_at\":1789125508,\"last_active_at\":1789125508,\"status\":\"idle\",\"required_actions\":[],\"error\":null,\"agent\":{\"id\":\"agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad\",\"name\":null,\"model\":\"gpt-5.6-luna\",\"reasoning\":{\"effort\":\"none\",\"summary\":null},\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"service_tier\":\"auto\",\"instructions\":\"You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna\",\"tools\":[{\"type\":\"web_search\",\"mode\":\"live\",\"context_size\":\"medium\",\"allowed_domains\":null,\"location\":null}],\"multi_agent\":{\"enabled\":false,\"max_concurrent_subagents\":null}},\"environment\":{\"type\":\"none\"},\"vault_ids\":[],\"usage\":null}}" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-origin": "*", + "access-control-expose-headers": "x-request-id,openai-processing-ms,openai-version,openai-organization,openai-project, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a396458fa865ad9e-IAD", + "connection": "keep-alive", + "content-type": "text/event-stream", + "date": "Fri, 11 Sep 2026 11:18:29 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "2423", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "vary": "origin, access-control-request-method, access-control-request-headers", + "x-content-type-options": "nosniff", + "x-openai-proxy-wasm": "v0.1", + "x-request-id": "req_53ab6e36f3884f0b8fe1ee0be1818c2e" + }, + "status": 201, + "statusText": "Created" + } + } + ], + "meta": { + "createdAt": "2026-09-11T09:56:45.830Z" + } +} diff --git a/e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7.cassette.json b/e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7.cassette.json new file mode 100644 index 000000000..606486e1e --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/__cassettes__/openai-agents-api-v7.cassette.json @@ -0,0 +1,92 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "49d07f1d39cd5c3d", + "matchKey": "POST api.openai.com/v1/agents/sessions", + "recordedAt": "2026-09-11T11:18:24.375Z", + "request": { + "body": { + "kind": "json", + "value": { + "agent": { + "instructions": "You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna", + "model": "gpt-5.6-luna", + "reasoning": { + "effort": "none" + }, + "tools": [ + { + "type": "web_search" + } + ] + }, + "environment": { + "type": "none" + }, + "input": "Use web search once for Vienna weather, then answer exactly: Sunny in Vienna", + "metadata": { + "braintrust_correlation_id": "agents-e2e-correlation" + }, + "stream": true + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/agents/sessions" + }, + "response": { + "body": { + "chunks": [ + "event: agent.session.created\ndata: {\"type\":\"agent.session.created\",\"event_id\":\"evt_20a706efc00b4aceafbc3c0d4680e14570c0b8091d604b5798\",\"session\":{\"metadata\":{\"braintrust_correlation_id\":\"agents-e2e-correlation\"},\"id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"object\":\"agent.session\",\"created_at\":1789125491,\"last_active_at\":1789125491,\"status\":\"in_progress\",\"required_actions\":[],\"error\":null,\"agent\":{\"id\":\"agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f\",\"name\":null,\"model\":\"gpt-5.6-luna\",\"reasoning\":{\"effort\":\"none\",\"summary\":null},\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"service_tier\":\"auto\",\"instructions\":\"You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna\",\"tools\":[{\"type\":\"web_search\",\"mode\":\"live\",\"context_size\":\"medium\",\"allowed_domains\":null,\"location\":null}],\"multi_agent\":{\"enabled\":false,\"max_concurrent_subagents\":null}},\"environment\":{\"type\":\"none\"},\"vault_ids\":[],\"usage\":null}}", + "event: agent.session.turn.created\ndata: {\"type\":\"agent.session.turn.created\",\"event_id\":\"evt_e7f0459247c941a2a7bf86b6250fa089db78ae90a78742898c\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"turn\":{\"id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"object\":\"agent.session.turn\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"agent_id\":\"agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f\",\"subagent_id\":null,\"status\":\"queued\",\"created_at\":1789125496,\"started_at\":null,\"completed_at\":null,\"error\":null,\"usage\":null}}", + "event: agent.session.turn.item.added\ndata: {\"type\":\"agent.session.turn.item.added\",\"event_id\":\"evt_6b903e0167ec41f29a6dcdbf32b53138d004da2b4f2044f180\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"output_index\":null,\"item\":{\"type\":\"message\",\"id\":\"msg_4f6a34e5eb82a5ebb5556f4db1b2b2655f05c12164335971d3\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Use web search once for Vienna weather, then answer exactly: Sunny in Vienna\"}],\"status\":\"completed\",\"phase\":null}}", + "event: agent.session.in_progress\ndata: {\"type\":\"agent.session.in_progress\",\"event_id\":\"evt_36fb4618a8654e97b86401c3a9c1ca84c52e5a1dbf4a4466a3\",\"session\":{\"metadata\":{\"braintrust_correlation_id\":\"agents-e2e-correlation\"},\"id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"object\":\"agent.session\",\"created_at\":1789125491,\"last_active_at\":1789125491,\"status\":\"in_progress\",\"required_actions\":[],\"error\":null,\"agent\":{\"id\":\"agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f\",\"name\":null,\"model\":\"gpt-5.6-luna\",\"reasoning\":{\"effort\":\"none\",\"summary\":null},\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"service_tier\":\"auto\",\"instructions\":\"You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna\",\"tools\":[{\"type\":\"web_search\",\"mode\":\"live\",\"context_size\":\"medium\",\"allowed_domains\":null,\"location\":null}],\"multi_agent\":{\"enabled\":false,\"max_concurrent_subagents\":null}},\"environment\":{\"type\":\"none\"},\"vault_ids\":[],\"usage\":null}}", + "event: agent.session.turn.in_progress\ndata: {\"type\":\"agent.session.turn.in_progress\",\"event_id\":\"evt_cd97596b213b464a986a4ee759b6691f3a9fa08800974aebbb\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"turn\":{\"id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"object\":\"agent.session.turn\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"agent_id\":\"agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f\",\"subagent_id\":null,\"status\":\"in_progress\",\"created_at\":1789125496,\"started_at\":1789125496,\"completed_at\":null,\"error\":null,\"usage\":null}}", + "event: agent.session.turn.item.added\ndata: {\"type\":\"agent.session.turn.item.added\",\"event_id\":\"evt_447964f2964948cc99972886e1a81b53aab0858dd9e0496592\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"output_index\":0,\"item\":{\"type\":\"web_search_call\",\"id\":\"exec-c50e194b-76a7-44b3-85d0-14fcb6597a49\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"status\":\"in_progress\",\"action\":{\"type\":\"search\",\"query\":\"\",\"queries\":null}}}", + "event: agent.session.turn.item.done\ndata: {\"type\":\"agent.session.turn.item.done\",\"event_id\":\"evt_86bf10ed83e44090845278d61c847c4938cd297a5f3f41ddb3\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"output_index\":0,\"item\":{\"type\":\"web_search_call\",\"id\":\"exec-c50e194b-76a7-44b3-85d0-14fcb6597a49\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"status\":\"completed\",\"action\":{\"type\":\"other\"}}}", + "event: agent.session.turn.item.added\ndata: {\"type\":\"agent.session.turn.item.added\",\"event_id\":\"evt_0fee1cb3fb2a4787a53bd39a9a426e892e7816b7a75749d5a9\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"output_index\":1,\"item\":{\"type\":\"message\",\"id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"role\":\"assistant\",\"content\":[],\"status\":\"in_progress\",\"phase\":\"final_answer\"}}", + "event: agent.session.turn.content_part.added\ndata: {\"type\":\"agent.session.turn.content_part.added\",\"event_id\":\"evt_d75be341a5bc4062b62faf85ab844dfcf7494dc2b467405480\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"item_id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"output_index\":1,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\"}}", + "event: agent.session.turn.output_text.delta\ndata: {\"type\":\"agent.session.turn.output_text.delta\",\"event_id\":\"evt_73d10cd58f524858b141ddb719cbaefc552f7c715a50466184\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"item_id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"output_index\":1,\"content_index\":0,\"delta\":\"Sunny\"}", + "event: agent.session.turn.output_text.delta\ndata: {\"type\":\"agent.session.turn.output_text.delta\",\"event_id\":\"evt_4ef2c8a3de6e4a44838bda7fcc5764db09798b7ee17d445485\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"item_id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"output_index\":1,\"content_index\":0,\"delta\":\" in\"}", + "event: agent.session.turn.output_text.delta\ndata: {\"type\":\"agent.session.turn.output_text.delta\",\"event_id\":\"evt_69fb0c9587674f3395af04fd884b24624a6c65623f6f4526b0\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"item_id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"output_index\":1,\"content_index\":0,\"delta\":\" Vienna\"}", + "event: agent.session.turn.output_text.done\ndata: {\"type\":\"agent.session.turn.output_text.done\",\"event_id\":\"evt_484a57d1a4d7474f92f70fa43a0c7e815444f72923534dbda0\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"item_id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"output_index\":1,\"content_index\":0,\"text\":\"Sunny in Vienna\"}", + "event: agent.session.turn.content_part.done\ndata: {\"type\":\"agent.session.turn.content_part.done\",\"event_id\":\"evt_1006d86344314f6cb0ae5e019d9e3eaa954886a8f06f479495\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"item_id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"output_index\":1,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"Sunny in Vienna\"}}", + "event: agent.session.turn.item.done\ndata: {\"type\":\"agent.session.turn.item.done\",\"event_id\":\"evt_0ba0d293ef3844d1b309e1f124b4d7bfa7942ff5751c4b1bb2\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"output_index\":1,\"item\":{\"type\":\"message\",\"id\":\"msg_07bdbc4dd4374e52016aa3e37ad09087d19973f0307b9aff4f\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"Sunny in Vienna\"}],\"phase\":\"final_answer\"}}", + "event: agent.session.turn.completed\ndata: {\"type\":\"agent.session.turn.completed\",\"event_id\":\"evt_dd98ee77802b4ae5a9bb91032a9b77a715aa19be3c914bd4a5\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"turn_id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"turn\":{\"id\":\"turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0\",\"object\":\"agent.session.turn\",\"session_id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"agent_id\":\"agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f\",\"subagent_id\":null,\"status\":\"completed\",\"created_at\":1789125496,\"started_at\":1789125496,\"completed_at\":1789125500,\"error\":null,\"usage\":null},\"usage\":null}", + "event: agent.session.idle\ndata: {\"type\":\"agent.session.idle\",\"event_id\":\"evt_78a083a7aeb74ed3983fa1aeb45f36997315c2f231a847a0ab\",\"session\":{\"metadata\":{\"braintrust_correlation_id\":\"agents-e2e-correlation\"},\"id\":\"sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2\",\"object\":\"agent.session\",\"created_at\":1789125491,\"last_active_at\":1789125491,\"status\":\"idle\",\"required_actions\":[],\"error\":null,\"agent\":{\"id\":\"agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f\",\"name\":null,\"model\":\"gpt-5.6-luna\",\"reasoning\":{\"effort\":\"none\",\"summary\":null},\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"service_tier\":\"auto\",\"instructions\":\"You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna\",\"tools\":[{\"type\":\"web_search\",\"mode\":\"live\",\"context_size\":\"medium\",\"allowed_domains\":null,\"location\":null}],\"multi_agent\":{\"enabled\":false,\"max_concurrent_subagents\":null}},\"environment\":{\"type\":\"none\"},\"vault_ids\":[],\"usage\":null}}" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-origin": "*", + "access-control-expose-headers": "x-request-id,openai-processing-ms,openai-version,openai-organization,openai-project, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cache-control": "no-cache", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a39645261b75ad9e-IAD", + "connection": "keep-alive", + "content-type": "text/event-stream", + "date": "Fri, 11 Sep 2026 11:18:13 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "3015", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "vary": "origin, access-control-request-method, access-control-request-headers", + "x-content-type-options": "nosniff", + "x-openai-proxy-wasm": "v0.1", + "x-request-id": "req_3a7a25e3ffce4d24a21386904a27a1af" + }, + "status": 201, + "statusText": "Created" + } + } + ], + "meta": { + "createdAt": "2026-09-11T09:56:42.914Z" + } +} diff --git a/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.json b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.json new file mode 100644 index 000000000..513aeacd3 --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.json @@ -0,0 +1,53 @@ +{ + "span_tree": [ + { + "name": "openai-agents-api-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai.agents.turn", + "type": "task", + "children": [ + { + "name": "Web search", + "type": "tool", + "children": [], + "input": { + "queries": null, + "query": "", + "type": "other" + }, + "output": { + "type": "other" + }, + "metadata": { + "item_id": "exec-", + "provider": "openai", + "status": "completed", + "tool_type": "web_search_call", + "turn_id": "turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2" + } + } + ], + "input": "Use web search once for Vienna weather, then answer exactly: Sunny in Vienna", + "output": "Sunny in Vienna", + "metadata": { + "agent_id": "agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad", + "api": "agents", + "braintrust_correlation_id": "agents-e2e-correlation", + "model": "gpt-5.6-luna", + "provider": "openai", + "session_id": "sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "scenario": "openai-agents-api-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.txt b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.txt new file mode 100644 index 000000000..e413f3985 --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7-latest.span-tree.txt @@ -0,0 +1,36 @@ +span_tree: +└── openai-agents-api-instrumentation-root [task] + metadata: { + "scenario": "openai-agents-api-instrumentation", + "testRunId": "" + } + └── openai.agents.turn [task] + input: "Use web search once for Vienna weather, then answer exactly: Sunny in Vienna" + output: "Sunny in Vienna" + metadata: { + "agent_id": "agent_df435bc8b75f497698093b919cbada68a6ca027cf3ae4c46ad", + "api": "agents", + "braintrust_correlation_id": "agents-e2e-correlation", + "model": "gpt-5.6-luna", + "provider": "openai", + "session_id": "sess_0f562f48483591c5006aa3e3840810819197f2ea12b5a0b4db" + } + metrics: { + "time_to_first_token": 0 + } + └── Web search [tool] + input: { + "queries": null, + "query": "", + "type": "other" + } + output: { + "type": "other" + } + metadata: { + "item_id": "exec-", + "provider": "openai", + "status": "completed", + "tool_type": "web_search_call", + "turn_id": "turn_0f562f48483591c5006aa3e389ed788191a53f9b1d288596a2" + } diff --git a/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.json b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.json new file mode 100644 index 000000000..193ec21ee --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.json @@ -0,0 +1,53 @@ +{ + "span_tree": [ + { + "name": "openai-agents-api-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai.agents.turn", + "type": "task", + "children": [ + { + "name": "Web search", + "type": "tool", + "children": [], + "input": { + "queries": null, + "query": "", + "type": "other" + }, + "output": { + "type": "other" + }, + "metadata": { + "item_id": "exec-", + "provider": "openai", + "status": "completed", + "tool_type": "web_search_call", + "turn_id": "turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0" + } + } + ], + "input": "Use web search once for Vienna weather, then answer exactly: Sunny in Vienna", + "output": "Sunny in Vienna", + "metadata": { + "agent_id": "agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f", + "api": "agents", + "braintrust_correlation_id": "agents-e2e-correlation", + "model": "gpt-5.6-luna", + "provider": "openai", + "session_id": "sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "scenario": "openai-agents-api-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.txt b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.txt new file mode 100644 index 000000000..62b771a15 --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/__snapshots__/openai-agents-api-v7.span-tree.txt @@ -0,0 +1,36 @@ +span_tree: +└── openai-agents-api-instrumentation-root [task] + metadata: { + "scenario": "openai-agents-api-instrumentation", + "testRunId": "" + } + └── openai.agents.turn [task] + input: "Use web search once for Vienna weather, then answer exactly: Sunny in Vienna" + output: "Sunny in Vienna" + metadata: { + "agent_id": "agent_5560e9ac401c4be3a7eb1600fbdbb5ffcecbbe89173b47e59f", + "api": "agents", + "braintrust_correlation_id": "agents-e2e-correlation", + "model": "gpt-5.6-luna", + "provider": "openai", + "session_id": "sess_0fc64a67f5ebbfcf006aa3e373bef88191b89c12edfdbec0e2" + } + metrics: { + "time_to_first_token": 0 + } + └── Web search [tool] + input: { + "queries": null, + "query": "", + "type": "other" + } + output: { + "type": "other" + } + metadata: { + "item_id": "exec-", + "provider": "openai", + "status": "completed", + "tool_type": "web_search_call", + "turn_id": "turn_0fc64a67f5ebbfcf006aa3e379f5bc81918596295e24a4e9c0" + } diff --git a/e2e/scenarios/openai-agents-api-instrumentation/assertions.ts b/e2e/scenarios/openai-agents-api-instrumentation/assertions.ts new file mode 100644 index 000000000..2e21bbd38 --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/assertions.ts @@ -0,0 +1,97 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { + withScenarioHarness, + type ScenarioRunContext, +} from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { + findLatestChildSpan, + findLatestSpan, +} from "../../helpers/trace-selectors"; +import { + CORRELATION_ID, + CORRELATION_KEY, + FINAL_OUTPUT, + INPUT, + MODEL_NAME, + ROOT_NAME, + SCENARIO_NAME, + TOOL_NAME, +} from "./constants.mjs"; + +type RunOpenAIAgentsAPIScenario = (harness: { + runScenarioDir: (options: { + entry: string; + env?: Record; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +export function defineOpenAIAgentsAPIInstrumentationAssertions(options: { + name: string; + runScenario: RunOpenAIAgentsAPIScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ); + const testConfig = { timeout: options.timeoutMs }; + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures the asynchronous Agents API turn", testConfig, () => { + const scenarioRoot = findLatestSpan(events, ROOT_NAME); + const turn = findLatestChildSpan( + events, + "openai.agents.turn", + scenarioRoot?.span.id, + ); + const tool = findLatestChildSpan(events, TOOL_NAME, turn?.span.id); + + expect(scenarioRoot?.row.metadata).toMatchObject({ + scenario: SCENARIO_NAME, + }); + expect(turn).toBeDefined(); + expect(turn?.span.type).toBe("task"); + expect(turn?.input).toBe(INPUT); + expect(String(turn?.output)).toContain(FINAL_OUTPUT); + expect(turn?.row.metadata).toMatchObject({ + [CORRELATION_KEY]: CORRELATION_ID, + api: "agents", + model: MODEL_NAME, + provider: "openai", + session_id: expect.any(String), + }); + expect(turn?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + }); + expect(tool).toBeDefined(); + expect(tool?.span.type).toBe("tool"); + expect(tool?.output).toBeDefined(); + expect(tool?.row.metadata).toMatchObject({ + provider: "openai", + tool_type: "web_search_call", + }); + expect(events.some((event) => event.span.type === "llm")).toBe(false); + }); + + test("matches the span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/openai-agents-api-instrumentation/cassette-filter.mjs b/e2e/scenarios/openai-agents-api-instrumentation/cassette-filter.mjs new file mode 100644 index 000000000..deeee961d --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/cassette-filter.mjs @@ -0,0 +1,3 @@ +// @ts-check +/** @type {import("@braintrust/seinfeld").FilterSpec} */ +export const filter = ["default"]; diff --git a/e2e/scenarios/openai-agents-api-instrumentation/constants.mjs b/e2e/scenarios/openai-agents-api-instrumentation/constants.mjs new file mode 100644 index 000000000..e7126cced --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/constants.mjs @@ -0,0 +1,9 @@ +export const CORRELATION_ID = "agents-e2e-correlation"; +export const CORRELATION_KEY = "braintrust_correlation_id"; +export const FINAL_OUTPUT = "Sunny in Vienna"; +export const INPUT = + "Use web search once for Vienna weather, then answer exactly: Sunny in Vienna"; +export const MODEL_NAME = "gpt-5.6-luna"; +export const ROOT_NAME = "openai-agents-api-instrumentation-root"; +export const SCENARIO_NAME = "openai-agents-api-instrumentation"; +export const TOOL_NAME = "Web search"; diff --git a/e2e/scenarios/openai-agents-api-instrumentation/package.json b/e2e/scenarios/openai-agents-api-instrumentation/package.json new file mode 100644 index 000000000..b37cf1937 --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/package.json @@ -0,0 +1,19 @@ +{ + "name": "@braintrust/e2e-openai-agents-api-instrumentation", + "private": true, + "type": "module", + "braintrustScenario": { + "bump": { + "dependencies": { + "openai-agents-api-v7-latest": { + "package": "openai", + "range": "7" + } + } + } + }, + "dependencies": { + "openai-agents-api-v7": "npm:openai@7.15.0", + "openai-agents-api-v7-latest": "npm:openai@7.15.0" + } +} diff --git a/e2e/scenarios/openai-agents-api-instrumentation/pnpm-lock.yaml b/e2e/scenarios/openai-agents-api-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..772bdb735 --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/pnpm-lock.yaml @@ -0,0 +1,46 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + openai-agents-api-v7: + specifier: npm:openai@7.15.0 + version: openai@7.15.0 + openai-agents-api-v7-latest: + specifier: npm:openai@7.15.0 + version: openai@7.15.0 + +packages: + + openai@7.15.0: + resolution: {integrity: sha512-2DIwesnPSduw68MFx7nAdgvshrgvS8WOan5W8+tgUMlAy/tY+Q41g80Pa+w/v64rHA6rtn+dketa8QvMVxU96w==} + engines: {node: '>=22.0.0'} + peerDependencies: + '@aws-sdk/credential-provider-node': '>=3.972.0 <4' + '@smithy/hash-node': '>=4.3.0 <5' + '@smithy/signature-v4': '>=5.4.0 <6' + undici: '>=5 <9' + ws: ^8.21.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@aws-sdk/credential-provider-node': + optional: true + '@smithy/hash-node': + optional: true + '@smithy/signature-v4': + optional: true + undici: + optional: true + ws: + optional: true + zod: + optional: true + +snapshots: + + openai@7.15.0: {} diff --git a/e2e/scenarios/openai-agents-api-instrumentation/scenario.mjs b/e2e/scenarios/openai-agents-api-instrumentation/scenario.mjs new file mode 100644 index 000000000..4bd947bcb --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/scenario.mjs @@ -0,0 +1,112 @@ +import { + failOpenAIAgentsTrace, + startOpenAIAgentsTrace, + updateOpenAIAgentsTrace, +} from "braintrust"; +import { runMain, runTracedScenario } from "../../helpers/provider-runtime.mjs"; +import { + CORRELATION_ID, + CORRELATION_KEY, + FINAL_OUTPUT, + INPUT, + MODEL_NAME, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +export async function runOpenAIAgentsAPIInstrumentationScenario() { + const openAIPackageName = + process.env.OPENAI_AGENTS_API_PACKAGE_NAME ?? "openai-agents-api-v7-latest"; + const { default: OpenAI } = await import(openAIPackageName); + const client = new OpenAI(); + const params = { + agent: { + instructions: + "You must use web search exactly once before answering. After searching, answer exactly: Sunny in Vienna", + model: MODEL_NAME, + reasoning: { effort: "none" }, + tools: [{ type: "web_search" }], + }, + environment: { type: "none" }, + input: INPUT, + metadata: { [CORRELATION_KEY]: CORRELATION_ID }, + stream: true, + }; + + await runTracedScenario({ + callback: async () => { + let traceToken = startOpenAIAgentsTrace(params); + const events = await client.beta.agents.sessions.create(params); + let finalOutput = ""; + let metadataObserved = false; + let webSearchObserved = false; + const seenEventTypes = []; + const timeout = setTimeout( + () => + events.controller.abort( + new Error( + `OpenAI Agents API stream timed out after events: ${seenEventTypes.join(", ")}`, + ), + ), + process.env.BRAINTRUST_E2E_DEBUG_AGENTS === "1" ? 60_000 : 120_000, + ); + try { + for await (const event of events) { + seenEventTypes.push(event.type); + traceToken = await updateOpenAIAgentsTrace(traceToken, event); + if ( + "session" in event && + event.session.metadata?.[CORRELATION_KEY] === CORRELATION_ID + ) { + metadataObserved = true; + } + if ( + !webSearchObserved && + event.type === "agent.session.turn.item.added" && + event.item.type === "web_search_call" + ) { + webSearchObserved = true; + } else if ( + event.type === "agent.session.turn.item.done" && + event.item.type === "message" && + event.item.phase === "final_answer" + ) { + finalOutput = event.item.content.map((part) => part.text).join(""); + } + if ( + event.type === "agent.session.turn.completed" || + event.type === "agent.session.turn.failed" || + event.type === "agent.session.turn.cancelled" + ) { + break; + } + } + } catch (error) { + traceToken = await failOpenAIAgentsTrace(traceToken, error); + if (events.controller.signal.reason instanceof Error) { + throw events.controller.signal.reason; + } + throw error; + } finally { + clearTimeout(timeout); + events.controller.abort(); + } + if (!webSearchObserved) { + throw new Error("OpenAI Agents API did not use web search"); + } + if (!metadataObserved) { + throw new Error("OpenAI Agents API did not preserve session metadata"); + } + if (!finalOutput.includes(FINAL_OUTPUT)) { + throw new Error(`Unexpected OpenAI Agents API output: ${finalOutput}`); + } + }, + flushCount: 2, + flushDelayMs: 10, + metadata: { scenario: SCENARIO_NAME }, + projectNameBase: "e2e-openai-agents-api-instrumentation", + rootName: ROOT_NAME, + }); +} + +runMain(runOpenAIAgentsAPIInstrumentationScenario); diff --git a/e2e/scenarios/openai-agents-api-instrumentation/scenario.test.ts b/e2e/scenarios/openai-agents-api-instrumentation/scenario.test.ts new file mode 100644 index 000000000..732bf7df8 --- /dev/null +++ b/e2e/scenarios/openai-agents-api-instrumentation/scenario.test.ts @@ -0,0 +1,61 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineOpenAIAgentsAPIInstrumentationAssertions } from "./assertions"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const TIMEOUT_MS = 180_000; +const scenarios = await Promise.all( + [ + { + dependencyName: "openai-agents-api-v7", + variantKey: "openai-agents-api-v7", + }, + { + dependencyName: "openai-agents-api-v7-latest", + variantKey: "openai-agents-api-v7-latest", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +describe.concurrent("variants", () => { + for (const scenario of scenarios) { + describe.sequential( + `${scenario.variantKey}: OpenAI Agents API SDK ${scenario.version}`, + () => { + defineOpenAIAgentsAPIInstrumentationAssertions({ + name: "manual instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: "scenario.mjs", + env: { + OPENAI_AGENTS_API_PACKAGE_NAME: scenario.dependencyName, + }, + runContext: { + variantKey: scenario.variantKey, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + }, + snapshotName: scenario.variantKey, + testFileUrl: import.meta.url, + timeoutMs: TIMEOUT_MS, + }); + }, + ); + } +}); diff --git a/js/src/exports.ts b/js/src/exports.ts index f7a4c3026..01a32f3ba 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -135,6 +135,12 @@ export { openaiFilesCreateTraced, } from "./openai-batch"; +export { + failOpenAIAgentsTrace, + startOpenAIAgentsTrace, + updateOpenAIAgentsTrace, +} from "./openai-agents-api"; + // Internal isomorph layer for platform-specific implementations import _internalIso from "./isomorph"; export { _internalIso }; diff --git a/js/src/instrumentation/plugins/openai-agents-api-instrumentation.test.ts b/js/src/instrumentation/plugins/openai-agents-api-instrumentation.test.ts new file mode 100644 index 000000000..41cfbb087 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-agents-api-instrumentation.test.ts @@ -0,0 +1,400 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + _exportsForTestingOnly, + initLogger, + startSpan, + withCurrent, +} from "../../logger"; +import { configureNode } from "../../node/config"; +import { + failOpenAIAgentsTrace, + startOpenAIAgentsTrace, + updateOpenAIAgentsTrace, +} from "../../openai-agents-api"; +import { OpenAIPlugin } from "./openai-plugin"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +describe("OpenAI Agents API instrumentation", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + let plugin: OpenAIPlugin; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "openai-agents-api-instrumentation.test.ts", + projectId: "test-project-id", + }); + plugin = new OpenAIPlugin(); + plugin.enable(); + }); + + afterEach(() => { + plugin.disable(); + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + it("captures a resumable turn and correlates function results", async () => { + let token = startOpenAIAgentsTrace({ + input: "What is the weather in Vienna?", + agent: { model: "gpt-test", tools: [{ name: "lookup_weather" }] }, + metadata: { tenant: "test" }, + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-session", + session: { + id: "sess-1", + agent: { id: "agent-1", model: "gpt-test", name: "Weather" }, + }, + type: "agent.session.created", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-turn", + session_id: "sess-1", + turn: { + id: "turn-1", + created_at: 100, + subagent_id: null, + }, + turn_id: "turn-1", + type: "agent.session.turn.created", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-function", + item: { + id: "item-function", + arguments: { city: "Vienna" }, + call_id: "call-1", + name: "lookup_weather", + status: "completed", + turn_id: "turn-1", + type: "function_call", + }, + type: "agent.session.turn.item.done", + }); + token = await updateOpenAIAgentsTrace(token, { + delta: "streamed answer text", + event_id: "evt-first-token", + type: "agent.session.turn.output_text.delta", + }); + + expect(token).not.toContain("streamed answer text"); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-result", + item: { + id: "item-result", + call_id: "call-1", + error: null, + output: "Sunny in Vienna", + status: "completed", + turn_id: "turn-1", + type: "function_call_output", + }, + type: "agent.session.turn.item.added", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-answer", + item: { + id: "item-answer", + content: [{ type: "output_text", text: "It is sunny." }], + phase: "final_answer", + role: "assistant", + status: "completed", + turn_id: "turn-1", + type: "message", + }, + type: "agent.session.turn.item.done", + }); + const completed = { + event_id: "evt-completed", + session_id: "sess-1", + turn: { + id: "turn-1", + completed_at: 200, + error: null, + subagent_id: null, + }, + turn_id: "turn-1", + type: "agent.session.turn.completed", + usage: { + input_tokens: 10, + input_tokens_details: { cached_tokens: 3 }, + output_tokens: 4, + output_tokens_details: { reasoning_tokens: 1 }, + total_tokens: 14, + }, + }; + token = await updateOpenAIAgentsTrace(token, completed); + token = await updateOpenAIAgentsTrace(token, completed); + expect(token).toEqual(expect.any(String)); + + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(2); + const root = rows.find( + (row) => row.span_attributes?.name === "openai.agents.turn", + ); + const tool = rows.find( + (row) => row.span_attributes?.name === "lookup_weather", + ); + expect(root).toMatchObject({ + input: "What is the weather in Vienna?", + metadata: { + agent_id: "agent-1", + agent_name: "Weather", + api: "agents", + model: "gpt-test", + provider: "openai", + session_id: "sess-1", + tenant: "test", + }, + metrics: { + completion_reasoning_tokens: 1, + completion_tokens: 4, + end: 200, + prompt_cached_tokens: 3, + prompt_tokens: 10, + time_to_first_token: expect.any(Number), + tokens: 14, + }, + output: "It is sunny.", + span_attributes: { type: "task" }, + }); + expect(tool).toMatchObject({ + input: { city: "Vienna" }, + metadata: { + item_id: "item-function", + provider: "openai", + status: "completed", + tool_type: "function_call", + turn_id: "turn-1", + }, + output: "Sunny in Vienna", + span_attributes: { type: "tool" }, + span_parents: [root?.span_id], + }); + expect(rows.some((row) => row.span_attributes?.type === "llm")).toBe(false); + }); + + it("nests subagent tools under a subagent task", async () => { + let token = startOpenAIAgentsTrace({ + input: "Delegate this research", + agent: { id: "root-agent", model: "gpt-test" }, + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-subagent", + subagent: { + id: "subagent-1", + instructions: [{ type: "output_text", text: "Research Vienna" }], + name: "Researcher", + opened_at: 110, + parent_agent_id: "root-agent", + status: "active", + }, + type: "agent.session.subagent.created", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-sub-turn", + session_id: "sess-2", + turn: { + id: "turn-sub", + created_at: 111, + subagent_id: "subagent-1", + }, + turn_id: "turn-sub", + type: "agent.session.turn.created", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-mcp-added", + item: { + id: "item-mcp", + arguments: { city: "Vienna" }, + name: "search", + server_label: "maps", + status: "in_progress", + turn_id: "turn-sub", + type: "mcp_call", + }, + type: "agent.session.turn.item.added", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-mcp-done", + item: { + id: "item-mcp", + arguments: { city: "Vienna" }, + error: null, + name: "search", + output: { result: "Vienna" }, + server_label: "maps", + status: "completed", + turn_id: "turn-sub", + type: "mcp_call", + }, + type: "agent.session.turn.item.done", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-sub-closed", + subagent: { + id: "subagent-1", + closed_at: 140, + name: "Researcher", + opened_at: 110, + parent_agent_id: "root-agent", + status: "closed", + }, + type: "agent.session.subagent.closed", + }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-root-completed", + session_id: "sess-2", + turn: { + id: "turn-root", + completed_at: 150, + subagent_id: null, + }, + turn_id: "turn-root", + type: "agent.session.turn.completed", + usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 }, + }); + expect(token).toEqual(expect.any(String)); + + const rows = (await backgroundLogger.drain()) as Array>; + const root = rows.find( + (row) => row.span_attributes?.name === "openai.agents.turn", + ); + const subagent = rows.find( + (row) => row.span_attributes?.name === "openai.agents.subagent", + ); + const tool = rows.find( + (row) => row.span_attributes?.name === "maps.search", + ); + expect(subagent).toMatchObject({ + input: [{ type: "output_text", text: "Research Vienna" }], + metadata: { + agent_id: "subagent-1", + agent_name: "Researcher", + parent_agent_id: "root-agent", + }, + metrics: { end: 140, start: 110 }, + span_attributes: { type: "task" }, + span_parents: [root?.span_id], + }); + expect(tool).toMatchObject({ + input: { city: "Vienna" }, + output: { result: "Vienna" }, + span_attributes: { type: "tool" }, + span_parents: [subagent?.span_id], + }); + }); + + it("closes open work on explicit failure and rejects invalid tokens", async () => { + let token = startOpenAIAgentsTrace({ input: "fail" }); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-command", + item: { + id: "item-command", + command: "exit 1", + cwd: "/workspace", + status: "in_progress", + turn_id: "turn-fail", + type: "command_execution", + }, + type: "agent.session.turn.item.added", + }); + token = await failOpenAIAgentsTrace(token, new Error("network failed")); + token = await failOpenAIAgentsTrace(token, new Error("duplicate failure")); + + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(2); + expect( + rows.find((row) => row.span_attributes?.name === "openai.agents.turn") + ?.error, + ).toContain("network failed"); + expect( + rows.find((row) => row.span_attributes?.name === "Command execution") + ?.error, + ).toContain("network failed"); + await expect( + updateOpenAIAgentsTrace("invalid", { type: "agent.session.created" }), + ).rejects.toThrow("Invalid OpenAI Agents trace token"); + }); + + it("preserves the active Braintrust span as its parent", async () => { + const parent = startSpan({ name: "parent" }); + let token = withCurrent(parent, () => + startOpenAIAgentsTrace({ input: "hello" }), + ); + token = await updateOpenAIAgentsTrace(token, { + event_id: "evt-completed", + turn: { completed_at: 200, id: "turn-1", subagent_id: null }, + type: "agent.session.turn.completed", + }); + expect(token).toEqual(expect.any(String)); + parent.end(); + + const rows = (await backgroundLogger.drain()) as Array>; + const parentRow = rows.find( + (row) => row.span_attributes?.name === "parent", + ); + const root = rows.find( + (row) => row.span_attributes?.name === "openai.agents.turn", + ); + expect(root?.span_parents).toEqual([parentRow?.span_id]); + expect(root?.root_span_id).toBe(parentRow?.root_span_id); + }); + + it("validates broadly typed start and update arguments at runtime", async () => { + expect(() => startOpenAIAgentsTrace(null)).toThrow( + "expected an OpenAI Agents parameters object", + ); + expect(() => + startOpenAIAgentsTrace({ agent: "not-an-agent-object" }), + ).toThrow("expected agent to be an object or null"); + expect(() => startOpenAIAgentsTrace({ agent: { model: 42 } })).toThrow( + "invalid agent identity fields", + ); + expect(() => + startOpenAIAgentsTrace({ agent_id: { id: "agent-1" } }), + ).toThrow("expected agent_id to be a string"); + expect(() => + startOpenAIAgentsTrace({ metadata: ["not", "metadata"] }), + ).toThrow("expected metadata to be an object or null"); + let token = startOpenAIAgentsTrace({ + agent: { future_agent_field: true, model: "gpt-test" }, + future_session_field: { supported: true }, + input: [{ future_input_type: "provider-version-specific" }], + }); + expect(token).toEqual(expect.any(String)); + await expect(updateOpenAIAgentsTrace(token, null)).rejects.toThrow( + "expected an OpenAI Agents event object", + ); + await expect(updateOpenAIAgentsTrace(token, { type: 42 })).rejects.toThrow( + "expected an OpenAI Agents event object", + ); + await expect(updateOpenAIAgentsTrace(token, { type: "" })).rejects.toThrow( + "expected an OpenAI Agents event object", + ); + token = await updateOpenAIAgentsTrace(token, { + future_event_field: { supported: true }, + type: "agent.session.future_event", + }); + token = await failOpenAIAgentsTrace(token, new Error("done")); + expect(token).toEqual(expect.any(String)); + + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + input: [{ future_input_type: "provider-version-specific" }], + metadata: { model: "gpt-test", provider: "openai" }, + }); + }); +}); diff --git a/js/src/instrumentation/plugins/openai-agents-api-instrumentation.ts b/js/src/instrumentation/plugins/openai-agents-api-instrumentation.ts new file mode 100644 index 000000000..ef76dc665 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-agents-api-instrumentation.ts @@ -0,0 +1,816 @@ +import { debugLogger } from "../../debug-logger"; +import { + _internalStartSpanWithInitialMerge, + _internalStartSpanWithInitialMergeAndParentSpanIds, + NOOP_SPAN, + type Span, + withCurrent, +} from "../../logger"; +import type { + OpenAIAgentsOpenTool, + OpenAIAgentsTraceState, + StartOpenAIAgentsTraceArgs, +} from "../../openai-agents-api-types"; +import { parseMetricsFromUsage } from "../../openai-utils"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { getCurrentUnixTimestamp } from "../../util"; +import { isObject, SpanTypeAttribute } from "../../../util/index"; +import { SpanComponentsV4 } from "../../../util/span_identifier_v4"; +import { openAIChannels } from "./openai-channels"; +import { + deterministicDigest, + digestHex, + digestUuid, +} from "./openai-manual-instrumentation-utils"; + +const TERMINAL_TURN_EVENTS = new Set([ + "agent.session.turn.completed", + "agent.session.turn.failed", + "agent.session.turn.cancelled", +]); + +const TURN_LIFECYCLE_EVENTS = new Set([ + "agent.session.turn.created", + "agent.session.turn.in_progress", + ...TERMINAL_TURN_EVENTS, +]); + +const SESSION_EVENTS = new Set([ + "agent.session.created", + "agent.session.failed", + "agent.session.idle", + "agent.session.in_progress", + "agent.session.requires_action", +]); + +const SUBAGENT_EVENTS = new Set([ + "agent.session.subagent.active", + "agent.session.subagent.closed", + "agent.session.subagent.created", +]); + +function read(value: unknown, key: PropertyKey): unknown { + if (!isObject(value)) { + return undefined; + } + try { + return Reflect.get(value, key); + } catch { + return undefined; + } +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function recordValue(record: Record, key: string): T | undefined { + return Object.hasOwn(record, key) ? record[key] : undefined; +} + +function setRecordValue(record: Record, key: string, value: T) { + Object.defineProperty(record, key, { + configurable: true, + enumerable: true, + value, + writable: true, + }); +} + +function loggedError(value: unknown, fallback: string): Error { + if (value instanceof Error) { + return value; + } + const message = stringValue(read(value, "message")); + return new Error(message ?? (typeof value === "string" ? value : fallback)); +} + +function logInstrumentationError(context: string, error: unknown): void { + debugLogger.debug(`OpenAI Agents API instrumentation ${context}:`, error); +} + +async function childSpanIds( + rootKey: string, + kind: string, + providerId: string, +): Promise<{ rowId: string; spanId: string }> { + const [row, span] = await Promise.all([ + deterministicDigest("openai:agents:row", rootKey, kind, providerId), + deterministicDigest("openai:agents:span", rootKey, kind, providerId), + ]); + return { rowId: digestUuid(row), spanId: digestHex(span, 8) }; +} + +function traceMetadata(args: StartOpenAIAgentsTraceArgs) { + const metadata: Record = {}; + const suppliedMetadata = read(args, "metadata"); + if (isObject(suppliedMetadata)) { + try { + for (const key of Object.keys(suppliedMetadata)) { + metadata[key] = read(suppliedMetadata, key); + } + } catch (error) { + logInstrumentationError("could not read supplied metadata", error); + } + } + const agent = read(args, "agent"); + const agentId = + stringValue(read(agent, "id")) ?? stringValue(read(args, "agent_id")); + const agentName = stringValue(read(agent, "name")); + const model = stringValue(read(agent, "model")); + return { + ...metadata, + api: "agents", + ...(agentId ? { agent_id: agentId } : {}), + ...(agentName ? { agent_name: agentName } : {}), + ...(model ? { model } : {}), + provider: "openai", + }; +} + +function copyState(state: OpenAIAgentsTraceState): OpenAIAgentsTraceState { + return { + ...state, + callItems: { ...state.callItems }, + eventIds: [...state.eventIds], + openTools: { ...state.openTools }, + subagents: { ...state.subagents }, + turnSubagents: { ...state.turnSubagents }, + }; +} + +function startRootSpan(state: OpenAIAgentsTraceState): Span { + const root = SpanComponentsV4.fromStr(state.root); + const parent = SpanComponentsV4.fromStr(state.rootParent); + const rowId = stringValue(root.data.row_id); + const rootSpanId = stringValue(root.data.root_span_id); + const spanId = stringValue(root.data.span_id); + if (!rowId || !rootSpanId || !spanId) { + throw new Error("OpenAI Agents trace root is invalid"); + } + const hasParentSpan = Boolean( + parent.data.row_id && parent.data.span_id && parent.data.root_span_id, + ); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMergeAndParentSpanIds( + withSpanInstrumentationName( + { + name: "openai.agents.turn", + type: SpanTypeAttribute.TASK, + parent: state.rootParent, + ...(!hasParentSpan + ? { + parentSpanIds: { + parentSpanIds: [], + rootSpanId, + }, + } + : {}), + spanId, + startTime: state.startTime, + event: { id: rowId }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function startSubagentSpan( + state: OpenAIAgentsTraceState, + subagentId: string, + input?: unknown, + metadata?: Record, + visiting = new Set(), +): Promise { + const subagent = recordValue(state.subagents, subagentId); + let parent = state.root; + if ( + subagent?.parentAgentId && + subagent.parentAgentId !== subagentId && + recordValue(state.subagents, subagent.parentAgentId) && + !visiting.has(subagent.parentAgentId) + ) { + visiting.add(subagentId); + parent = await ( + await startSubagentSpan( + state, + subagent.parentAgentId, + undefined, + undefined, + visiting, + ) + ).export(); + } + const ids = await childSpanIds(state.rootKey, "subagent", subagentId); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { + name: "openai.agents.subagent", + type: SpanTypeAttribute.TASK, + parent, + spanId: ids.spanId, + startTime: subagent?.openedAt, + event: { + id: ids.rowId, + ...(input !== undefined ? { input } : {}), + metadata: { + ...metadata, + agent_id: subagentId, + ...(subagent?.parentAgentId + ? { parent_agent_id: subagent.parentAgentId } + : {}), + provider: "openai", + }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function parentForTurn( + state: OpenAIAgentsTraceState, + turnId: string | undefined, +): Promise { + const subagentId = turnId + ? recordValue(state.turnSubagents, turnId) + : undefined; + if (!subagentId) { + return state.root; + } + return await (await startSubagentSpan(state, subagentId)).export(); +} + +async function startToolSpan( + state: OpenAIAgentsTraceState, + tool: OpenAIAgentsOpenTool, + input?: unknown, +): Promise { + const ids = await childSpanIds(state.rootKey, "tool", tool.itemId); + return withCurrent(NOOP_SPAN, async () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { + name: tool.name, + type: SpanTypeAttribute.TOOL, + parent: await parentForTurn(state, tool.turnId), + spanId: ids.spanId, + startTime: tool.startTime, + event: { + id: ids.rowId, + ...(input !== undefined ? { input } : {}), + metadata: { + item_id: tool.itemId, + provider: "openai", + tool_type: tool.toolType, + ...(tool.turnId ? { turn_id: tool.turnId } : {}), + }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +function toolFromItem( + item: unknown, + observedAt: number, +): { input?: unknown; tool: OpenAIAgentsOpenTool } | undefined { + const itemId = stringValue(read(item, "id")); + const toolType = stringValue(read(item, "type")); + if (!itemId || !toolType) { + return undefined; + } + const turnId = stringValue(read(item, "turn_id")); + switch (toolType) { + case "function_call": + return { + input: read(item, "arguments"), + tool: { + itemId, + name: stringValue(read(item, "name")) ?? "Function call", + startTime: observedAt, + toolType, + ...(turnId ? { turnId } : {}), + }, + }; + case "mcp_call": { + const name = stringValue(read(item, "name")) ?? "MCP call"; + const server = stringValue(read(item, "server_label")); + return { + input: read(item, "arguments"), + tool: { + itemId, + name: server ? `${server}.${name}` : name, + startTime: observedAt, + toolType, + ...(turnId ? { turnId } : {}), + }, + }; + } + case "web_search_call": + return { + input: read(item, "action"), + tool: { + itemId, + name: "Web search", + startTime: observedAt, + toolType, + ...(turnId ? { turnId } : {}), + }, + }; + case "command_execution": + return { + input: { + command: read(item, "command"), + cwd: read(item, "cwd"), + }, + tool: { + itemId, + name: "Command execution", + startTime: observedAt, + toolType, + ...(turnId ? { turnId } : {}), + }, + }; + default: + return undefined; + } +} + +function textFromContent(content: unknown): string | undefined { + if (!Array.isArray(content)) { + return undefined; + } + const text: string[] = []; + for (const part of content) { + const partText = read(part, "text"); + if (read(part, "type") === "output_text" && typeof partText === "string") { + text.push(partText); + } + } + return text.length > 0 ? text.join("") : undefined; +} + +function toolOutput(item: unknown, toolType: string): unknown { + switch (toolType) { + case "function_call": + return read(item, "output"); + case "mcp_call": + return read(item, "output"); + case "web_search_call": + return read(item, "action"); + case "command_execution": + return read(item, "output"); + default: + return undefined; + } +} + +function toolFailed(item: unknown, toolType: string): Error | undefined { + const status = read(item, "status"); + const explicitError = read(item, "error"); + if (explicitError !== undefined && explicitError !== null) { + return loggedError(explicitError, `OpenAI ${toolType} failed`); + } + if (status === "failed" || status === "incomplete") { + return new Error(`OpenAI ${toolType} ${status}`); + } + if ( + toolType === "command_execution" && + typeof read(item, "exit_code") === "number" && + read(item, "exit_code") !== 0 + ) { + return new Error( + `OpenAI command exited with code ${read(item, "exit_code")}`, + ); + } + return undefined; +} + +async function completeTool( + state: OpenAIAgentsTraceState, + tool: OpenAIAgentsOpenTool, + item: unknown, + endTime: number, +): Promise { + const span = await startToolSpan(state, tool); + const output = toolOutput(item, tool.toolType); + const error = toolFailed(item, tool.toolType); + span.log({ + ...(output !== undefined ? { output } : {}), + ...(error ? { error } : {}), + metadata: { + status: read(item, "status"), + ...(tool.toolType === "command_execution" + ? { exit_code: read(item, "exit_code") } + : {}), + }, + }); + span.end({ endTime }); + Reflect.deleteProperty(state.openTools, tool.itemId); +} + +async function captureMessage( + state: OpenAIAgentsTraceState, + item: unknown, +): Promise { + if ( + read(item, "type") !== "message" || + read(item, "role") !== "assistant" || + read(item, "phase") !== "final_answer" + ) { + return; + } + const output = textFromContent(read(item, "content")); + if (output === undefined) { + return; + } + const turnId = stringValue(read(item, "turn_id")); + const subagentId = turnId + ? recordValue(state.turnSubagents, turnId) + : undefined; + if (subagentId) { + (await startSubagentSpan(state, subagentId)).log({ output }); + } else { + startRootSpan(state).log({ output }); + } +} + +async function captureItem( + state: OpenAIAgentsTraceState, + item: unknown, + isDone: boolean, + observedAt: number, +): Promise { + await captureMessage(state, item); + const itemType = stringValue(read(item, "type")); + if (itemType === "function_call_output") { + const callId = stringValue(read(item, "call_id")); + const itemId = callId ? recordValue(state.callItems, callId) : undefined; + const tool = itemId ? recordValue(state.openTools, itemId) : undefined; + if (tool) { + await completeTool(state, tool, item, observedAt); + } + return; + } + + const descriptor = toolFromItem(item, observedAt); + if (!descriptor) { + return; + } + const existing = recordValue(state.openTools, descriptor.tool.itemId); + const tool = existing ?? descriptor.tool; + setRecordValue(state.openTools, tool.itemId, tool); + if (itemType === "function_call") { + const callId = stringValue(read(item, "call_id")); + if (callId) { + setRecordValue(state.callItems, callId, tool.itemId); + } + } + await startToolSpan(state, tool, descriptor.input); + const status = read(item, "status"); + const terminalFunctionCall = + itemType === "function_call" && + (status === "failed" || status === "incomplete"); + if ( + (itemType !== "function_call" && (isDone || status !== "in_progress")) || + terminalFunctionCall + ) { + await completeTool(state, tool, item, observedAt); + } +} + +function updateSessionMetadata( + state: OpenAIAgentsTraceState, + session: unknown, +): void { + const sessionId = stringValue(read(session, "id")); + const agent = read(session, "agent"); + startRootSpan(state).log({ + metadata: { + ...(sessionId ? { session_id: sessionId } : {}), + ...(stringValue(read(agent, "id")) + ? { agent_id: read(agent, "id") } + : {}), + ...(stringValue(read(agent, "name")) + ? { agent_name: read(agent, "name") } + : {}), + ...(stringValue(read(agent, "model")) + ? { model: read(agent, "model") } + : {}), + }, + }); +} + +async function captureSubagent( + state: OpenAIAgentsTraceState, + eventType: string, + subagent: unknown, + observedAt: number, +): Promise { + const subagentId = stringValue(read(subagent, "id")); + if (!subagentId) { + return; + } + const existing = recordValue(state.subagents, subagentId); + const observedOpenedAt = read(subagent, "opened_at"); + const openedAt = + typeof observedOpenedAt === "number" && + Number.isFinite(observedOpenedAt) && + observedOpenedAt >= 0 + ? observedOpenedAt + : (existing?.openedAt ?? observedAt); + const observedClosedAt = read(subagent, "closed_at"); + const closedAt = + typeof observedClosedAt === "number" && + Number.isFinite(observedClosedAt) && + observedClosedAt >= 0 + ? observedClosedAt + : existing?.closedAt; + const parentAgentId = + stringValue(read(subagent, "parent_agent_id")) ?? existing?.parentAgentId; + setRecordValue(state.subagents, subagentId, { + ...(closedAt !== undefined ? { closedAt } : {}), + openedAt, + ...(parentAgentId ? { parentAgentId } : {}), + }); + const span = await startSubagentSpan( + state, + subagentId, + read(subagent, "instructions"), + { + ...(stringValue(read(subagent, "name")) + ? { agent_name: read(subagent, "name") } + : {}), + status: read(subagent, "status"), + }, + ); + if (eventType === "agent.session.subagent.closed") { + span.end({ endTime: closedAt ?? observedAt }); + } +} + +async function closeTrace( + state: OpenAIAgentsTraceState, + endTime: number, + error?: Error, + usage?: unknown, +): Promise { + if (state.ended) { + return; + } + for (const tool of Object.values(state.openTools)) { + const span = await startToolSpan(state, tool); + span.log({ + error: + error ?? new Error("OpenAI Agents turn ended before tool completion"), + }); + span.end({ endTime }); + } + state.openTools = {}; + for (const subagentId of Object.keys(state.subagents)) { + const subagent = state.subagents[subagentId]; + const span = await startSubagentSpan(state, subagentId); + span.end({ endTime: subagent.closedAt ?? endTime }); + } + const root = startRootSpan(state); + root.log({ + ...(error ? { error } : {}), + metrics: parseMetricsFromUsage(usage), + }); + root.end({ endTime }); + state.ended = true; +} + +async function captureTurnLifecycle( + state: OpenAIAgentsTraceState, + event: unknown, + eventType: string, + observedAt: number, +): Promise { + const turn = read(event, "turn"); + const turnId = + stringValue(read(event, "turn_id")) ?? stringValue(read(turn, "id")); + const subagentId = stringValue(read(turn, "subagent_id")); + if (turnId && subagentId) { + setRecordValue(state.turnSubagents, turnId, subagentId); + if (!recordValue(state.subagents, subagentId)) { + const createdAt = read(turn, "created_at"); + setRecordValue(state.subagents, subagentId, { + openedAt: + typeof createdAt === "number" && + Number.isFinite(createdAt) && + createdAt >= 0 + ? createdAt + : observedAt, + }); + } + await startSubagentSpan(state, subagentId); + } + const sessionId = stringValue(read(event, "session_id")); + if (sessionId) { + startRootSpan(state).log({ metadata: { session_id: sessionId } }); + } + if (!TERMINAL_TURN_EVENTS.has(eventType) || subagentId) { + if (subagentId && eventType === "agent.session.turn.failed") { + (await startSubagentSpan(state, subagentId)).log({ + error: loggedError(read(turn, "error"), "OpenAI subagent turn failed"), + }); + } + return; + } + const completedAt = read(turn, "completed_at"); + const endTime = + typeof completedAt === "number" && + Number.isFinite(completedAt) && + completedAt >= 0 + ? completedAt + : observedAt; + const usage = read(event, "usage") ?? read(turn, "usage"); + const error = + eventType === "agent.session.turn.completed" + ? undefined + : loggedError( + read(turn, "error"), + eventType === "agent.session.turn.cancelled" + ? "OpenAI Agents turn cancelled" + : "OpenAI Agents turn failed", + ); + await closeTrace(state, endTime, error, usage); +} + +async function captureEvent( + state: OpenAIAgentsTraceState, + event: unknown, +): Promise { + if (state.ended) { + return state; + } + const eventType = stringValue(read(event, "type")); + const eventId = stringValue(read(event, "event_id")); + if (!eventType || (eventId && state.eventIds.includes(eventId))) { + return state; + } + const next = copyState(state); + const observedAt = getCurrentUnixTimestamp(); + let handled = false; + if (SESSION_EVENTS.has(eventType)) { + const session = read(event, "session"); + if (session) { + updateSessionMetadata(next, session); + } + handled = true; + } + if (eventType === "agent.session.turn.item.added") { + await captureItem(next, read(event, "item"), false, observedAt); + handled = true; + } else if (eventType === "agent.session.turn.item.done") { + await captureItem(next, read(event, "item"), true, observedAt); + handled = true; + } else if (TURN_LIFECYCLE_EVENTS.has(eventType)) { + await captureTurnLifecycle(next, event, eventType, observedAt); + handled = true; + } else if (SUBAGENT_EVENTS.has(eventType)) { + await captureSubagent(next, eventType, read(event, "subagent"), observedAt); + handled = true; + } else if ( + eventType === "agent.session.turn.output_text.delta" && + next.firstTokenAt === undefined && + stringValue(read(event, "delta")) + ) { + next.firstTokenAt = observedAt; + startRootSpan(next).log({ + metrics: { time_to_first_token: observedAt - next.startTime }, + }); + handled = true; + } else if (eventType === "agent.session.failed") { + const session = read(event, "session"); + const lastActiveAt = read(session, "last_active_at"); + const endTime = + typeof lastActiveAt === "number" && + Number.isFinite(lastActiveAt) && + lastActiveAt >= 0 + ? lastActiveAt + : observedAt; + await closeTrace( + next, + endTime, + loggedError(read(session, "error"), "OpenAI Agents session failed"), + read(session, "usage"), + ); + handled = true; + } else if (eventType === "error") { + await closeTrace( + next, + observedAt, + loggedError(read(event, "error"), "OpenAI Agents session failed"), + ); + handled = true; + } + if (!handled) { + return state; + } + if (eventId) { + next.eventIds.push(eventId); + } + return next; +} + +export const interceptOpenAIAgentsTraceStart: Parameters< + typeof openAIChannels.agentsTraceStart.intercept +>[0] = async (target, thisArg, args) => { + const fallback = await Reflect.apply(target, thisArg, args); + try { + const state = args[0].state; + const parent = state.rootParent; + const traceArgs = args[0].args; + const parentComponents = SpanComponentsV4.fromStr(parent); + const hasParentSpan = Boolean( + parentComponents.data.row_id && + parentComponents.data.span_id && + parentComponents.data.root_span_id, + ); + const rootComponents = SpanComponentsV4.fromStr(state.root); + const rowId = stringValue(rootComponents.data.row_id); + const rootSpanId = stringValue(rootComponents.data.root_span_id); + const spanId = stringValue(rootComponents.data.span_id); + if (!rowId || !rootSpanId || !spanId) { + throw new Error("OpenAI Agents trace root is invalid"); + } + withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMergeAndParentSpanIds( + withSpanInstrumentationName( + { + name: "openai.agents.turn", + type: SpanTypeAttribute.TASK, + parent, + ...(!hasParentSpan + ? { + parentSpanIds: { + parentSpanIds: [], + rootSpanId, + }, + } + : {}), + spanId, + startTime: state.startTime, + event: { + id: rowId, + input: read(traceArgs, "input"), + metadata: traceMetadata(traceArgs), + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); + return state; + } catch (error) { + logInstrumentationError("could not start trace", error); + return fallback; + } +}; + +export const interceptOpenAIAgentsTraceCapture: Parameters< + typeof openAIChannels.agentsTraceCapture.intercept +>[0] = async (target, thisArg, args) => { + const fallback = await Reflect.apply(target, thisArg, args); + if (!args[0].state) { + return fallback; + } + try { + return await captureEvent(args[0].state, args[0].event); + } catch (error) { + logInstrumentationError("could not capture event", error); + return fallback; + } +}; + +export const interceptOpenAIAgentsTraceFail: Parameters< + typeof openAIChannels.agentsTraceFail.intercept +>[0] = async (target, thisArg, args) => { + const fallback = await Reflect.apply(target, thisArg, args); + if (!args[0].state) { + return fallback; + } + try { + const next = copyState(args[0].state); + await closeTrace( + next, + getCurrentUnixTimestamp(), + loggedError(args[0].error, "OpenAI Agents request failed"), + ); + return next; + } catch (error) { + logInstrumentationError("could not fail trace", error); + return fallback; + } +}; diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts index 560ac51ca..5bd662620 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -25,6 +25,11 @@ import { extractOpenAIBatchInput, processImagesInOutput, } from "./openai-span-data"; +import { + deterministicDigest, + digestHex, + digestUuid, +} from "./openai-manual-instrumentation-utils"; const SUPPORTED_ENDPOINTS = new Set(["/v1/chat/completions", "/v1/responses"]); const TERMINAL_STATUSES = new Set([ @@ -113,26 +118,6 @@ async function exportParent( return undefined; } -async function deterministicDigest(namespace: string, ...parts: string[]) { - const encoded = new TextEncoder().encode( - [namespace, ...parts].map((part) => `${part.length}:${part}`).join("\0"), - ); - return new Uint8Array( - await globalThis.crypto.subtle.digest("SHA-256", encoded), - ); -} - -function digestHex(bytes: Uint8Array, length: number): string { - return Array.from(bytes.slice(0, length)) - .map((byte) => byte.toString(16).padStart(2, "0")) - .join(""); -} - -function digestUuid(bytes: Uint8Array): string { - const hex = digestHex(bytes, 16); - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; -} - async function batchSpanIds(inputFileId: string): Promise<{ rowId: string; spanId: string; diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index 5ef873eca..d0daae63d 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -24,6 +24,10 @@ import type { OpenAIFileLike, OpenAIFilesCreateTraceArgs, } from "../../openai-batch-types"; +import type { + OpenAIAgentsTraceStartChannelArgs, + OpenAIAgentsTraceState, +} from "../../openai-agents-api-types"; type OpenAIChatSpanInfo = NonNullable["span_info"]>; @@ -38,6 +42,33 @@ type OpenAIResponsesChannelExtras = OpenAIChannelExtras; export const openAIChannels = defineChannels( "openai", { + agentsTraceStart: channel< + [OpenAIAgentsTraceStartChannelArgs], + OpenAIAgentsTraceState | null, + OpenAIChannelExtras + >({ + channelName: "agents.trace.start", + kind: "async", + }), + + agentsTraceCapture: channel< + [{ event: unknown; state: OpenAIAgentsTraceState | null }], + OpenAIAgentsTraceState | null, + OpenAIChannelExtras + >({ + channelName: "agents.trace.capture", + kind: "async", + }), + + agentsTraceFail: channel< + [{ error: unknown; state: OpenAIAgentsTraceState | null }], + OpenAIAgentsTraceState | null, + OpenAIChannelExtras + >({ + channelName: "agents.trace.fail", + kind: "async", + }), + filesCreateTraced: channel< [OpenAIFilesCreateTraceArgs], OpenAIFileLike, diff --git a/js/src/instrumentation/plugins/openai-manual-instrumentation-utils.ts b/js/src/instrumentation/plugins/openai-manual-instrumentation-utils.ts new file mode 100644 index 000000000..59b0b57c8 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-manual-instrumentation-utils.ts @@ -0,0 +1,22 @@ +export async function deterministicDigest( + namespace: string, + ...parts: string[] +): Promise { + const encoded = new TextEncoder().encode( + [namespace, ...parts].map((part) => `${part.length}:${part}`).join("\0"), + ); + return new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", encoded), + ); +} + +export function digestHex(bytes: Uint8Array, length: number): string { + return Array.from(bytes.slice(0, length)) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +export function digestUuid(bytes: Uint8Array): string { + const hex = digestHex(bytes, 16); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index fa9cdcdec..d08de59ad 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -19,6 +19,11 @@ import { interceptOpenAIBatchTraceComplete, interceptOpenAIFilesCreateTraced, } from "./openai-batch-instrumentation"; +import { + interceptOpenAIAgentsTraceCapture, + interceptOpenAIAgentsTraceFail, + interceptOpenAIAgentsTraceStart, +} from "./openai-agents-api-instrumentation"; import { BRAINTRUST_CACHED_STREAM_METRIC, getCachedMetricFromHeaders, @@ -48,6 +53,13 @@ export class OpenAIPlugin extends BasePlugin { protected onEnable(): void { this.unsubscribers.push( + openAIChannels.agentsTraceStart.intercept( + interceptOpenAIAgentsTraceStart, + ), + openAIChannels.agentsTraceCapture.intercept( + interceptOpenAIAgentsTraceCapture, + ), + openAIChannels.agentsTraceFail.intercept(interceptOpenAIAgentsTraceFail), openAIChannels.filesCreateTraced.intercept( interceptOpenAIFilesCreateTraced, ), diff --git a/js/src/logger.ts b/js/src/logger.ts index 5eb438ba9..29a4ec81d 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -5950,6 +5950,61 @@ export function getSpanParentObject( return getSpanParentObjectAndPropagatedState(options).parentObject; } +/** @internal */ +export function _internalExportParentSynchronously( + parent: ReturnType, +): string | undefined { + try { + const toStr = Reflect.get(parent, "toStr"); + if (typeof toStr === "function") { + return Reflect.apply(toStr, parent, []); + } + + if ( + "getParentInfo" in parent && + typeof parent.getParentInfo === "function" + ) { + const parentInfo = parent.getParentInfo(); + if (parentInfo) { + const objectId = parentInfo.objectId.getSync().value; + if (objectId || parentInfo.computeObjectMetadataArgs) { + return new SpanComponentsV4({ + object_type: parentInfo.objectType, + ...(objectId + ? { object_id: objectId } + : { + compute_object_metadata_args: + parentInfo.computeObjectMetadataArgs, + }), + row_id: parent.id, + root_span_id: parent.rootSpanId, + span_id: parent.spanId, + }).toStr(); + } + } + } + + const getOtelParent = Reflect.get(parent, "_getOtelParent"); + if (typeof getOtelParent !== "function") { + return undefined; + } + const components = braintrustParentToComponents( + Reflect.apply(getOtelParent, parent, []), + ); + if (!components) { + return undefined; + } + return new SpanComponentsV4({ + object_type: components.objectType, + ...(components.objectId + ? { object_id: components.objectId } + : { compute_object_metadata_args: components.computeArgs ?? {} }), + }).toStr(); + } catch { + return undefined; + } +} + /** * Return the Braintrust parent string for the current logger/experiment, if any. * diff --git a/js/src/openai-agents-api-types.ts b/js/src/openai-agents-api-types.ts new file mode 100644 index 000000000..ff70bc5fe --- /dev/null +++ b/js/src/openai-agents-api-types.ts @@ -0,0 +1,50 @@ +export interface StartOpenAIAgentsTraceArgs { + input?: unknown; + agent?: { + id?: string; + model?: string; + name?: string | null; + [key: string]: unknown; + } | null; + agent_id?: string; + metadata?: Record | null; +} + +export interface OpenAIAgentsTraceStartChannelArgs { + args: StartOpenAIAgentsTraceArgs; + state: OpenAIAgentsTraceState; +} + +export type OpenAIAgentsOpenTool = { + itemId: string; + name: string; + startTime: number; + toolType: string; + turnId?: string; +}; + +export type OpenAIAgentsSubagent = { + closedAt?: number; + openedAt: number; + parentAgentId?: string; +}; + +export type OpenAIAgentsTraceState = { + callItems: Record; + ended: boolean; + eventIds: string[]; + firstTokenAt?: number; + openTools: Record; + root: string; + rootKey: string; + rootParent: string; + startTime: number; + subagents: Record; + turnSubagents: Record; + version: 1; +}; + +export type OpenAIAgentsTraceToken = { + state: OpenAIAgentsTraceState | null; + version: 1; +}; diff --git a/js/src/openai-agents-api.ts b/js/src/openai-agents-api.ts new file mode 100644 index 000000000..f8cdbebf7 --- /dev/null +++ b/js/src/openai-agents-api.ts @@ -0,0 +1,307 @@ +import { debugLogger } from "./debug-logger"; +import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import { resolveUseLegacyUuidIds } from "./id-gen"; +import { + _internalExportParentSynchronously, + getSpanParentObject, + newId, +} from "./logger"; +import { getCurrentUnixTimestamp } from "./util"; +import { SpanComponentsV4 } from "../util/span_identifier_v4"; +import type { + OpenAIAgentsTraceState, + OpenAIAgentsTraceToken, + StartOpenAIAgentsTraceArgs, +} from "./openai-agents-api-types"; + +const TOKEN_PREFIX = "bt-openai-agents-v1:"; +const MAX_TOKEN_LENGTH = 1_000_000; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validateStartArgs(value: unknown): StartOpenAIAgentsTraceArgs { + if (!isRecord(value)) { + throw new TypeError( + "startOpenAIAgentsTrace expected an OpenAI Agents parameters object", + ); + } + const agent = value.agent; + if (agent !== undefined && agent !== null && !isRecord(agent)) { + throw new TypeError( + "startOpenAIAgentsTrace expected agent to be an object or null", + ); + } + if ( + isRecord(agent) && + ((agent.id !== undefined && typeof agent.id !== "string") || + (agent.model !== undefined && typeof agent.model !== "string") || + (agent.name !== undefined && + agent.name !== null && + typeof agent.name !== "string")) + ) { + throw new TypeError( + "startOpenAIAgentsTrace received invalid agent identity fields", + ); + } + if (value.agent_id !== undefined && typeof value.agent_id !== "string") { + throw new TypeError( + "startOpenAIAgentsTrace expected agent_id to be a string", + ); + } + if ( + value.metadata !== undefined && + value.metadata !== null && + !isRecord(value.metadata) + ) { + throw new TypeError( + "startOpenAIAgentsTrace expected metadata to be an object or null", + ); + } + return value; +} + +function validTraceState(value: unknown): value is OpenAIAgentsTraceState { + if (!isRecord(value)) { + return false; + } + return ( + value.version === 1 && + typeof value.root === "string" && + value.root.length > 0 && + typeof value.rootKey === "string" && + value.rootKey.length > 0 && + typeof value.rootParent === "string" && + value.rootParent.length > 0 && + typeof value.ended === "boolean" && + typeof value.startTime === "number" && + Number.isFinite(value.startTime) && + (value.firstTokenAt === undefined || + (typeof value.firstTokenAt === "number" && + Number.isFinite(value.firstTokenAt))) && + Array.isArray(value.eventIds) && + value.eventIds.every((eventId) => typeof eventId === "string") && + isRecord(value.callItems) && + Object.values(value.callItems).every( + (itemId) => typeof itemId === "string", + ) && + isRecord(value.openTools) && + Object.values(value.openTools).every( + (tool) => + isRecord(tool) && + typeof tool.itemId === "string" && + typeof tool.name === "string" && + typeof tool.startTime === "number" && + Number.isFinite(tool.startTime) && + typeof tool.toolType === "string" && + (tool.turnId === undefined || typeof tool.turnId === "string"), + ) && + isRecord(value.subagents) && + Object.values(value.subagents).every( + (subagent) => + isRecord(subagent) && + typeof subagent.openedAt === "number" && + Number.isFinite(subagent.openedAt) && + (subagent.closedAt === undefined || + (typeof subagent.closedAt === "number" && + Number.isFinite(subagent.closedAt))) && + (subagent.parentAgentId === undefined || + typeof subagent.parentAgentId === "string"), + ) && + isRecord(value.turnSubagents) && + Object.values(value.turnSubagents).every( + (subagentId) => typeof subagentId === "string", + ) + ); +} + +function decodeToken(token: string): OpenAIAgentsTraceState | null { + if ( + typeof token !== "string" || + !token.startsWith(TOKEN_PREFIX) || + token.length > MAX_TOKEN_LENGTH + ) { + throw new TypeError("Invalid OpenAI Agents trace token"); + } + let decoded: unknown; + try { + decoded = JSON.parse(token.slice(TOKEN_PREFIX.length)); + } catch { + throw new TypeError("Invalid OpenAI Agents trace token"); + } + if (!isRecord(decoded) || decoded.version !== 1) { + throw new TypeError("Invalid OpenAI Agents trace token"); + } + if (decoded.state === null) { + return null; + } + if (!validTraceState(decoded.state)) { + throw new TypeError("Invalid OpenAI Agents trace token"); + } + return decoded.state; +} + +function encodeToken(state: OpenAIAgentsTraceState | null): string { + const token: OpenAIAgentsTraceToken = { + state, + version: 1, + }; + const encoded = `${TOKEN_PREFIX}${JSON.stringify(token)}`; + if (encoded.length > MAX_TOKEN_LENGTH) { + throw new RangeError("OpenAI Agents trace token is too large"); + } + return encoded; +} + +function validateEvent(event: unknown): unknown { + let eventType: unknown; + try { + eventType = isRecord(event) ? Reflect.get(event, "type") : undefined; + } catch { + eventType = undefined; + } + if (typeof eventType !== "string" || eventType.length === 0) { + throw new TypeError( + "updateOpenAIAgentsTrace expected an OpenAI Agents event object", + ); + } + return event; +} + +/** + * Start a manual trace for one asynchronous OpenAI Agents API turn. + * + * This helper performs no OpenAI API requests. Pass it the same `input`, + * `agent`, and `agent_id` values used to create or continue the turn. `metadata` + * is copied to the Braintrust span and can also contain an application + * correlation ID when the same parameters are sent to OpenAI. The returned + * token can be persisted immediately before submission and passed with later events to + * `updateOpenAIAgentsTrace`. The trace closes when a root turn reaches a + * terminal state. + */ +export function startOpenAIAgentsTrace( + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Provider types vary between OpenAI SDK versions; validate the narrow surface at runtime. + args: any, +): string { + const validatedArgs = validateStartArgs(args); + const parent = _internalExportParentSynchronously(getSpanParentObject()); + if (!parent) { + return encodeToken(null); + } + const parentComponents = SpanComponentsV4.fromStr(parent); + const rowId = newId(); + const useLegacyIds = resolveUseLegacyUuidIds(); + const spanId = useLegacyIds + ? newId() + : newId().replaceAll("-", "").slice(0, 16); + const rootSpanId = useLegacyIds ? spanId : newId().replaceAll("-", ""); + const root = new SpanComponentsV4({ + object_type: parentComponents.data.object_type, + ...(parentComponents.data.object_id + ? { object_id: parentComponents.data.object_id } + : { + compute_object_metadata_args: + parentComponents.data.compute_object_metadata_args ?? {}, + }), + propagated_event: parentComponents.data.propagated_event, + root_span_id: rootSpanId, + row_id: rowId, + span_id: spanId, + }).toStr(); + const state: OpenAIAgentsTraceState = { + callItems: {}, + ended: false, + eventIds: [], + openTools: {}, + root, + rootKey: spanId, + rootParent: parent, + startTime: getCurrentUnixTimestamp(), + subagents: {}, + turnSubagents: {}, + version: 1, + }; + try { + void openAIChannels.agentsTraceStart + .invoke( + async () => state, + undefined, + [{ args: validatedArgs, state }], + {}, + ) + .catch((error) => { + debugLogger.debug( + "OpenAI Agents API instrumentation could not start:", + error, + ); + }); + } catch (error) { + debugLogger.debug( + "OpenAI Agents API instrumentation could not start:", + error, + ); + } + return encodeToken(state); +} + +/** + * Apply an OpenAI Agents event to a trace and return its next resumable token. + * + * This helper performs no OpenAI API requests. It is intended for stream, + * webhook, and worker processes that observe events from a turn. Events should + * be applied in order, passing the returned token to the next call. + */ +export async function updateOpenAIAgentsTrace( + token: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Provider event types vary between OpenAI SDK versions; validate the narrow surface at runtime. + event: any, +): Promise { + const state = decodeToken(token); + const validatedEvent = validateEvent(event); + try { + return encodeToken( + await openAIChannels.agentsTraceCapture.invoke( + async () => state, + undefined, + [{ state, event: validatedEvent }], + {}, + ), + ); + } catch (error) { + debugLogger.debug( + "OpenAI Agents API instrumentation could not update:", + error, + ); + return token; + } +} + +/** + * End a manual OpenAI Agents API trace after submission or observation fails. + * + * This helper performs no OpenAI API requests. It returns a terminal token so + * repeated delivery can remain idempotent. + */ +export async function failOpenAIAgentsTrace( + token: string, + error: unknown, +): Promise { + const state = decodeToken(token); + try { + return encodeToken( + await openAIChannels.agentsTraceFail.invoke( + async () => state, + undefined, + [{ state, error }], + {}, + ), + ); + } catch (instrumentationError) { + debugLogger.debug( + "OpenAI Agents API instrumentation could not record failure:", + instrumentationError, + ); + return token; + } +} diff --git a/js/src/wrappers/ai-sdk/harness-agent-context.ts b/js/src/wrappers/ai-sdk/harness-agent-context.ts index fc0b6171a..78d60e912 100644 --- a/js/src/wrappers/ai-sdk/harness-agent-context.ts +++ b/js/src/wrappers/ai-sdk/harness-agent-context.ts @@ -2,6 +2,7 @@ import iso from "../../isomorph"; import type { IsoAsyncLocalStorage, IsoTracingChannel } from "../../isomorph"; import { _internalGetGlobalState, + _internalExportParentSynchronously, currentSpan, startSpan, updateSpan, @@ -117,32 +118,10 @@ function continuationParentFromCreateSessionParams( return context.parent; } -function exportSpanSynchronously(span: Span): string | undefined { - const parentInfo = span.getParentInfo(); - if (!parentInfo) { - return undefined; - } - const objectId = parentInfo.objectId.getSync().value; - if (!objectId && !parentInfo.computeObjectMetadataArgs) { - return undefined; - } - - return new SpanComponentsV4({ - object_type: parentInfo.objectType, - ...(objectId - ? { object_id: objectId } - : { - compute_object_metadata_args: - parentInfo.computeObjectMetadataArgs ?? {}, - }), - row_id: span.id, - root_span_id: span.rootSpanId, - span_id: span.spanId, - }).toStr(); -} - function exportedParent(parent: HarnessTurnParent): string | undefined { - return typeof parent === "string" ? parent : exportSpanSynchronously(parent); + return typeof parent === "string" + ? parent + : _internalExportParentSynchronously(parent); } function addSerializedContext(args: { From d3506ab1a33aa5c156f9652aab8208e2c0aadbfb Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:30:23 +0000 Subject: [PATCH 2/2] Update PR #2471 --- .changeset/openai-agents-api-instrumentation.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/openai-agents-api-instrumentation.md diff --git a/.changeset/openai-agents-api-instrumentation.md b/.changeset/openai-agents-api-instrumentation.md new file mode 100644 index 000000000..ed158e5f5 --- /dev/null +++ b/.changeset/openai-agents-api-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Instrument OpenAI agents api (`openai.beta.agents`)