test(e2e): Make the node-flue suite deterministic

Three things made it flaky on a Playwright retry, which is why the dev run went
red then green:

- `DataLoader` was a module-level singleton, so it cached keys 1-3 and a second
  `count_items` call skipped the batch function and emitted no span. Constructed
  per execution now, matching node-eve.
- Conversation ids were fixed strings, so `runAgentTurn` saw a settlement from an
  earlier run and returned before the new turn finished. Each turn gets its own.
- The waits matched any agent turn, so a leftover trace could satisfy the wrong
  test. Each now anchors on its own tool via `gen_ai.tool.name`.

The dataloader test asserts a shared trace rather than the exact parent: the
model may call the tool more than once, and the span that ran the loader is not
reliably the one found by name.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: RulaKhaled

dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts

@@ -2,7 +2,7 @@
 import { useModel, useTool } from '@flue/runtime';
 import * as Sentry from '@sentry/node';
 import * as v from 'valibot';
-import { itemLoader } from '../loaders.ts';
+import DataLoader from 'dataloader';
 
 // The `'use agent'` directive is how `@flue/vite` finds this module and binds an identity to it at
 // build time. That binding is the part a hand-written scenario cannot reproduce, so it is the main
@@ -31,8 +31,11 @@ export function Hello() {
     name: 'count_items',
     description: 'Count items by loading them. Call this when the user asks to count items.',
     input: v.object({}),
+    // Constructed per execution, like node-eve does: a module-level loader caches its keys, so a
+    // second call would skip the batch function and emit no span.
     run: async () => {
-      const doubled = await Promise.all([itemLoader.load(1), itemLoader.load(2), itemLoader.load(3)]);
+      const loader = new DataLoader<number, number>(async keys => keys.map(key => key * 2));
+      const doubled = await Promise.all([loader.load(1), loader.load(2), loader.load(3)]);
       return `Loaded ${doubled.length} items: ${doubled.join(', ')}.`;
     },
   });

dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts

@@ -1,6 +1,6 @@
 import { expect, test } from '@playwright/test';
 import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
-import { runAgentTurn } from './utils';
+import { newConversationId, runAgentTurn } from './utils';
 
 const APP = 'node-flue';
 
@@ -17,15 +17,22 @@ const isDataloaderSpan = (span: { attributes?: Record<string, { value?: unknown
  * spans, rather than in a trace of its own.
  */
 test('captures orchestrion-instrumented dataloader spans in the same trace as the AI spans', async ({ baseURL }) => {
-  const spansPromise = collectStreamedSpans(APP, spansOfTrace => spansOfTrace.some(isDataloaderSpan));
+  const spansPromise = collectStreamedSpans(
+    APP,
+    spansOfTrace =>
+      spansOfTrace.some(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items') &&
+      spansOfTrace.some(isDataloaderSpan),
+  );
 
-  await runAgentTurn(baseURL!, 'dataloader-conversation', 'Please call count_items to count the items.');
+  await runAgentTurn(baseURL!, newConversationId('dataloader'), 'Please call count_items to count the items.');
 
   const spans = await spansPromise;
-  const executeTool = spans.find(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items');
   const dataloaderSpan = spans.find(isDataloaderSpan);
+  const toolSpan = spans.find(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items');
 
+  // Sharing the trace is the point: the orchestrion span is captured alongside the AI spans rather
+  // than in a trace of its own. Not asserting the exact parent — the model may call the tool more
+  // than once, and the span that ran the loader is not reliably the one found here.
   expect(getSpanOp(dataloaderSpan!)).toBe('cache.get');
-  expect(dataloaderSpan?.trace_id).toBe(executeTool?.trace_id);
-  expect(dataloaderSpan?.parent_span_id).toBe(executeTool?.span_id);
+  expect(dataloaderSpan?.trace_id).toBe(toolSpan?.trace_id);
 });

dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts

@@ -1,6 +1,6 @@
 import { expect, test } from '@playwright/test';
-import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils';
-import { runAgentTurn } from './utils';
+import { collectStreamedSpans, waitForError } from '@sentry-internal/test-utils';
+import { newConversationId, runAgentTurn } from './utils';
 
 const APP = 'node-flue';
 
@@ -10,10 +10,10 @@ test('captures an error thrown inside a Flue tool and marks its span errored', a
     event => event.exception?.values?.[0]?.value === 'Intentional flue tool failure',
   );
   const spansPromise = collectStreamedSpans(APP, spansOfTrace =>
-    spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'),
+    spansOfTrace.some(span => span.attributes?.['gen_ai.tool.name']?.value === 'fail_now'),
   );
 
-  await runAgentTurn(baseURL!, 'failure-conversation', 'Please call fail_now to trigger a failure.');
+  await runAgentTurn(baseURL!, newConversationId('failure'), 'Please call fail_now to trigger a failure.');
 
   const error = await errorPromise;
   expect(error.exception?.values?.[0]?.value).toBe('Intentional flue tool failure');

dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts

@@ -1,17 +1,28 @@
 import { expect, test } from '@playwright/test';
 import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
-import { runAgentTurn } from './utils';
+import { newConversationId, runAgentTurn } from './utils';
 
 const APP = 'node-flue';
 
-const hasOps = (ops: string[]) => (spansOfTrace: { attributes?: Record<string, { value?: unknown }> }[]) =>
+type SpanLike = { name?: string; attributes?: Record<string, { value?: unknown }> };
+
+const hasOps = (ops: string[]) => (spansOfTrace: SpanLike[]) =>
   ops.every(op => spansOfTrace.some(span => getSpanOp(span) === op));
 
+// Anchored on the tool the test drives, so a leftover trace from another test cannot satisfy it.
+const usedTool = (toolName: string) => (spansOfTrace: SpanLike[]) =>
+  spansOfTrace.some(span => span.attributes?.['gen_ai.tool.name']?.value === toolName);
+
 test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn', async ({ baseURL }) => {
   // The trace flushes across several envelopes, so accumulate it rather than asserting on one.
-  const spansPromise = collectStreamedSpans(APP, hasOps(['gen_ai.invoke_agent', 'gen_ai.chat', 'gen_ai.execute_tool']));
+  const spansPromise = collectStreamedSpans(
+    APP,
+    spansOfTrace =>
+      hasOps(['gen_ai.invoke_agent', 'gen_ai.chat', 'gen_ai.execute_tool'])(spansOfTrace) &&
+      usedTool('get_weather')(spansOfTrace),
+  );
 
-  await runAgentTurn(baseURL!, 'weather-conversation', 'What is the weather in Paris?');
+  await runAgentTurn(baseURL!, newConversationId('weather'), 'What is the weather in Paris?');
 
   const spans = await spansPromise;
   const invokeAgent = spans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent');
@@ -40,12 +51,10 @@ test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn'
 test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => {
   const spansPromise = collectStreamedSpans(
     APP,
-    spansOfTrace =>
-      spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool') &&
-      spansOfTrace.some(span => span.name === 'resolve-weather'),
+    spansOfTrace => usedTool('get_weather')(spansOfTrace) && spansOfTrace.some(span => span.name === 'resolve-weather'),
   );
 
-  await runAgentTurn(baseURL!, 'manual-span-conversation', 'What is the weather in Berlin?');
+  await runAgentTurn(baseURL!, newConversationId('manual-span'), 'What is the weather in Berlin?');
 
   const spans = await spansPromise;
   const executeTool = spans.find(span => getSpanOp(span) === 'gen_ai.execute_tool');
@@ -66,7 +75,7 @@ test('nests the provider HTTP call inside the chat span', async ({ baseURL }) =>
       spansOfTrace.some(span => getSpanOp(span) === 'http.client'),
   );
 
-  await runAgentTurn(baseURL!, 'provider-http-conversation', 'Say hello.');
+  await runAgentTurn(baseURL!, newConversationId('provider-http'), 'Say hello.');
 
   const spans = await spansPromise;
   const chat = spans.find(span => getSpanOp(span) === 'gen_ai.chat');

dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts

@@ -1,5 +1,16 @@
 import { expect } from '@playwright/test';
 
+/**
+ * A conversation id nothing has used yet.
+ *
+ * `runAgentTurn` waits for the conversation to report a settlement, so a fixed id that already has
+ * one — a Playwright retry, or the `test:dev` run hitting the record `test:prod` left behind —
+ * would return before the new turn finished.
+ */
+export function newConversationId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
 /**
  * Run one agent turn over Flue's agent router and wait for it to settle.
  *