fix(otel): scope deterministic ID generation

examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java

@@ -17,7 +17,7 @@
  * <p>This handler configures the OTel plugin with:
  *
  * <ul>
- *   <li>Deterministic trace/span IDs (all invocations of the same execution share one trace)
+ *   <li>Deterministic Workflow trace/span IDs with provider-generated Invocation roots
  *   <li>MDC log enrichment (traceId, spanId, traceSampled in every log line)
  *   <li>Logging exporter (spans printed to stdout → CloudWatch Logs)
  * </ul>
@@ -28,11 +28,13 @@
  * <p>Expected trace structure:
  *
  * <pre>
- * durable.invocation
- * ├── durable.step:create-greeting [attempt 1]
- * ├── durable.step:create-greeting (operation, backfilled)
- * ├── durable.step:transform [attempt 1]
- * └── durable.step:transform (operation, backfilled)
+ * Workflow
+ *
+ * Invocation
+ * ├── create-greeting
+ * │   └── create-greeting attempt 1
+ * └── transform
+ *     └── transform attempt 1
  * </pre>
  */
 public class OtelExample extends DurableHandler<GreetingRequest, String> {

examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExample.java

@@ -13,7 +13,8 @@
  * OTel + X-Ray example using the ExecutionOtelPlugin with the no-arg constructor.
  *
  * <p>{@link ExecutionOtelPlugin#ExecutionOtelPlugin()} uses the global provider initialized by the ADOT Java agent. The
- * ExecutionOtelPlugin renders the Workflow span as the trace root with operations as siblings of the invocation span.
+ * ExecutionOtelPlugin renders the Workflow span as the durable trace root with operations beneath it. Operations link
+ * to the Invocation span in the ambient Lambda trace.
  */
 @ExampleTemplate(tracing = true, javaAgent = true)
 public class OtelXRayExecutionStepExample extends DurableHandler<GreetingRequest, String> {

examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java

@@ -28,15 +28,18 @@
  *   <li>{@code OTEL_JAVAAGENT_EXTENSIONS} pointing at the OTel plugin jar
  * </ul>
  *
- * <p>Expected trace structure in X-Ray (all under one trace ID — backend propagates same Root):
+ * <p>Expected trace structure in X-Ray:
  *
  * <pre>
- * Trace (single trace ID across both invocations)
- * ├── invocation (invocation 1)
+ * Workflow trace:
+ * Workflow
+ *
+ * Ambient invocation trace:
+ * ├── Invocation (invocation 1)
  * │   ├── before-wait
  * │   │   └── before-wait attempt 1
  * │   └── pause (ended as PENDING)
- * └── invocation (invocation 2)
+ * └── Invocation (invocation 2)
  *     ├── pause (completed)
  *     └── after-wait
  *         └── after-wait attempt 1

examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java

@@ -7,8 +7,8 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -22,6 +22,7 @@
 import software.amazon.awssdk.services.xray.model.GetTraceSummariesRequest;
 import software.amazon.awssdk.services.xray.model.Segment;
 import software.amazon.awssdk.services.xray.model.TimeRangeType;
+import software.amazon.awssdk.services.xray.model.Trace;
 import software.amazon.awssdk.services.xray.model.TraceSummary;
 import software.amazon.lambda.durable.examples.types.GreetingRequest;
 import software.amazon.lambda.durable.model.ExecutionStatus;
@@ -41,10 +42,10 @@
  * <p>After invoking the function, the test queries the X-Ray API to verify:
  *
  * <ul>
- *   <li>A single trace exists for the execution (deterministic trace ID works)
+ *   <li>A deterministic Workflow trace exists separately from the ambient Invocation trace
  *   <li>Expected span/segment names are present
  *   <li>Parent-child nesting is correct
- *   <li>Multi-invocation scenarios produce one unified trace
+ *   <li>Multi-invocation scenarios retain Workflow correlation
  * </ul>
  *
  * <p>Enable with: {@code -Dtest.cloud.enabled=true}
@@ -112,7 +113,7 @@ private static String arn(String functionName) {
     // ─── Test: Simple Steps (Single Invocation) ──────────────────────────
 
     @Test
-    void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
+    void simpleSteps_producesCorrelatedWorkflowAndInvocationTracesInXRay() throws Exception {
         var startTime = Instant.now();
 
         // 1. Invoke the function (use unique input to avoid stale executions)
@@ -128,11 +129,12 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
         Thread.sleep(XRAY_INGESTION_DELAY.toMillis());
 
         // 3. Query X-Ray for the trace, retrying until durable spans appear
-        var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "create-greeting");
+        var invocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "create-greeting");
+        var workflowTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "Workflow");
 
         // 5. Verify span structure
         var segmentDocuments =
-                durableTrace.segments().stream().map(Segment::document).toList();
+                invocationTrace.segments().stream().map(Segment::document).toList();
         var allSegmentText = String.join("\n", segmentDocuments);
 
         // Verify expected span names appear in the trace
@@ -142,19 +144,20 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
         assertTrue(allSegmentText.contains("create-greeting"), "Expected create-greeting span in trace");
         assertTrue(allSegmentText.contains("transform"), "Expected transform span in trace");
 
-        // Verify all segments share the same trace ID (single unified trace)
-        var uniqueTraceIds =
-                durableTrace.segments().stream().map(seg -> durableTrace.id()).collect(Collectors.toSet());
-        assertEquals(1, uniqueTraceIds.size(), "All segments should belong to a single trace");
+        assertNotEquals(
+                workflowTrace.id(),
+                invocationTrace.id(),
+                "Workflow and Invocation spans must not create disconnected roots in one trace");
 
         System.out.println("✅ Simple steps test passed — "
-                + durableTrace.segments().size() + " segments in trace " + durableTrace.id());
+                + invocationTrace.segments().size() + " invocation segments correlated with Workflow trace "
+                + workflowTrace.id());
     }
 
     // ─── Test: Wait + Resume (Multi-Invocation) ─────────────────────────
 
     @Test
-    void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
+    void waitAndResume_preservesWorkflowCorrelationAcrossInvocations() throws Exception {
         var startTime = Instant.now();
 
         // 1. Invoke the function — will suspend on wait, then resume automatically
@@ -172,14 +175,22 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
         Thread.sleep(XRAY_INGESTION_DELAY.plus(Duration.ofSeconds(5)).toMillis());
 
         // 3. Query X-Ray for the trace, retrying until durable spans appear
-        var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "before-wait");
+        var firstInvocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "before-wait");
+        var secondInvocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "after-wait");
+        var workflowTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "Workflow");
 
         // 4. Verify multi-invocation trace structure
-        var segmentDocuments =
-                durableTrace.segments().stream().map(Segment::document).toList();
+        var segmentDocuments = new ArrayList<String>();
+        segmentDocuments.addAll(
+                firstInvocationTrace.segments().stream().map(Segment::document).toList());
+        if (!firstInvocationTrace.id().equals(secondInvocationTrace.id())) {
+            segmentDocuments.addAll(secondInvocationTrace.segments().stream()
+                    .map(Segment::document)
+                    .toList());
+        }
         var allSegmentText = String.join("\n", segmentDocuments);
 
-        // Verify spans from BOTH invocations appear in the same trace
+        // Verify spans from both invocations were exported.
         assertTrue(allSegmentText.contains("before-wait"), "Expected before-wait span from first invocation");
         assertTrue(allSegmentText.contains("after-wait"), "Expected after-wait span from second invocation");
         assertTrue(allSegmentText.contains("pause"), "Expected wait:pause span in trace");
@@ -190,24 +201,20 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
                 invocationCount >= 2,
                 "Expected at least 2 invocation spans (multi-invocation), got " + invocationCount);
 
-        // Critical assertion: all segments under ONE trace (deterministic ID worked)
-        assertEquals(
-                1,
-                Set.of(durableTrace.id()).size(),
-                "All segments should belong to a single trace — deterministic trace ID must work across invocations");
+        assertNotEquals(workflowTrace.id(), firstInvocationTrace.id());
+        assertNotEquals(workflowTrace.id(), secondInvocationTrace.id());
 
-        System.out.println(
-                "✅ Wait + resume test passed — " + durableTrace.segments().size() + " segments across "
-                        + invocationCount + " invocations in trace " + durableTrace.id());
+        System.out.println("✅ Wait + resume test passed — "
+                + invocationCount
+                + " invocations correlated with Workflow trace "
+                + workflowTrace.id());
     }
 
     // ─── Helpers ─────────────────────────────────────────────────────────
 
     /** Queries X-Ray for traces with retry logic to handle eventual consistency. */
     private List<TraceSummary> queryTracesWithRetry(Instant startTime, Instant endTime, String functionName)
             throws InterruptedException {
-        // Query by durable.invocation service — our spans are in a separate trace from Lambda's
-        // built-in X-Ray segment (durable backend propagates its own trace root)
         // Filter by the Lambda function's service name — each function has a unique one.
         // This avoids picking up traces from other durable functions that share service.name="invocation".
         var filterExpression = "service(\"" + functionNamePrefix + functionName + "\")";
@@ -285,8 +292,8 @@ private static String summarizeSegments(List<String> segmentDocuments) {
      * Queries X-Ray for a trace containing durable spans, retrying until the expected span appears or timeout is
      * reached. Handles eventual consistency where the trace exists but OTLP-exported spans haven't been ingested yet.
      */
-    private software.amazon.awssdk.services.xray.model.Trace queryTraceWithDurableSpans(
-            Instant startTime, String functionName, String expectedSpanName) throws InterruptedException {
+    private Trace queryTraceWithDurableSpans(Instant startTime, String functionName, String expectedSpanName)
+            throws InterruptedException {
         var maxAttempts = 5;
         for (int attempt = 1; attempt <= maxAttempts; attempt++) {
             var traces = queryTracesWithRetry(startTime, Instant.now(), functionName);
@@ -295,7 +302,7 @@ private software.amazon.awssdk.services.xray.model.Trace queryTraceWithDurableSp
             }
 
             var traceIds = traces.stream().map(TraceSummary::id).toList();
-            var allTraces = new java.util.ArrayList<software.amazon.awssdk.services.xray.model.Trace>();
+            var allTraces = new ArrayList<Trace>();
             for (int i = 0; i < traceIds.size(); i += 5) {
                 var batch = traceIds.subList(i, Math.min(i + 5, traceIds.size()));
                 var batchResult = xrayClient.batchGetTraces(