From 6ff3ca8876ffc0dc38152b305da27a1c341af36c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Sat, 15 Aug 2026 18:27:36 +0000 Subject: [PATCH 1/3] fix(otel): scope deterministic ID generation --- .../durable/examples/general/OtelExample.java | 14 +- .../otel/OtelXRayExecutionStepExample.java | 3 +- .../examples/otel/OtelXRayWaitExample.java | 11 +- .../CloudBasedOtelIntegrationTest.java | 65 ++-- otel-plugin/README.md | 49 +-- otel-plugin/pom.xml | 16 +- .../otel/DeterministicIdGenerator.java | 235 +++++++++---- .../durable/otel/ExecutionOtelPlugin.java | 70 ++-- .../lambda/durable/otel/ExtractedContext.java | 5 +- .../durable/otel/InvocationOtelPlugin.java | 107 ++---- ...inAutoConfigurationCustomizerProvider.java | 14 +- .../lambda/durable/otel/OtelPluginConfig.java | 51 +-- .../durable/otel/OtelPluginSupport.java | 103 ------ .../lambda/durable/otel/ProviderSource.java | 13 +- .../durable/otel/XRayContextExtractor.java | 6 +- .../otel/DeterministicIdGeneratorTest.java | 182 +++++++++- .../durable/otel/ExecutionOtelPluginTest.java | 54 ++- .../InvocationOtelPluginIntegrationTest.java | 22 +- .../otel/InvocationOtelPluginTest.java | 322 +++++++++++++----- 19 files changed, 794 insertions(+), 548 deletions(-) diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java index d8135975b..59294be1f 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/general/OtelExample.java @@ -17,7 +17,7 @@ *

This handler configures the OTel plugin with: * *

@@ -28,11 +28,13 @@ *

Expected trace structure: * * 

- * durable.invocation
- * ├── durable.step:create-greeting [attempt 1]
- * ├── durable.step:create-greeting (operation, backfilled)
- * ├── durable.step:transform [attempt 1]
- * └── durable.step:transform (operation, backfilled)
+ * Workflow
+ *
+ * Invocation
+ * ├── create-greeting
+ * │   └── create-greeting attempt 1
+ * └── transform
+ *     └── transform attempt 1
  *
*/ public class OtelExample extends DurableHandler { diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExample.java index f01bfa122..7c30bb4f5 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExample.java @@ -13,7 +13,8 @@ * OTel + X-Ray example using the ExecutionOtelPlugin with the no-arg constructor. * *

{@link ExecutionOtelPlugin#ExecutionOtelPlugin()} uses the global provider initialized by the ADOT Java agent. The - * ExecutionOtelPlugin renders the Workflow span as the trace root with operations as siblings of the invocation span. + * ExecutionOtelPlugin renders the Workflow span as the durable trace root with operations beneath it. Operations link + * to the Invocation span in the ambient Lambda trace. */ @ExampleTemplate(tracing = true, javaAgent = true) public class OtelXRayExecutionStepExample extends DurableHandler { diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java index aac7c95d8..3d593ac0a 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java @@ -28,15 +28,18 @@ *

  • {@code OTEL_JAVAAGENT_EXTENSIONS} pointing at the OTel plugin jar * * - *

    Expected trace structure in X-Ray (all under one trace ID — backend propagates same Root): + *

    Expected trace structure in X-Ray: * * 

    - * Trace (single trace ID across both invocations)
- * ├── invocation (invocation 1)
+ * Workflow trace:
+ * Workflow
+ *
+ * Ambient invocation trace:
+ * ├── Invocation (invocation 1)
  * │   ├── before-wait
  * │   │   └── before-wait attempt 1
  * │   └── pause (ended as PENDING)
- * └── invocation (invocation 2)
+ * └── Invocation (invocation 2)
  *     ├── pause (completed)
  *     └── after-wait
  *         └── after-wait attempt 1
diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java
index 8f87e71f5..023025a46 100644
--- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java
+++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java
@@ -7,8 +7,8 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -22,6 +22,7 @@
 import software.amazon.awssdk.services.xray.model.GetTraceSummariesRequest;
 import software.amazon.awssdk.services.xray.model.Segment;
 import software.amazon.awssdk.services.xray.model.TimeRangeType;
+import software.amazon.awssdk.services.xray.model.Trace;
 import software.amazon.awssdk.services.xray.model.TraceSummary;
 import software.amazon.lambda.durable.examples.types.GreetingRequest;
 import software.amazon.lambda.durable.model.ExecutionStatus;
@@ -41,10 +42,10 @@
  * 
    After invoking the function, the test queries the X-Ray API to verify:
  *
  * 
      
- *   
    • A single trace exists for the execution (deterministic trace ID works)
+ *   
    • A deterministic Workflow trace exists separately from the ambient Invocation trace
  *   
    • Expected span/segment names are present
  *   
    • Parent-child nesting is correct
- *   
    • Multi-invocation scenarios produce one unified trace
+ *   
    • Multi-invocation scenarios retain Workflow correlation
  * 
    
  *
  * 
    Enable with: {@code -Dtest.cloud.enabled=true}
@@ -112,7 +113,7 @@ private static String arn(String functionName) {
     // ─── Test: Simple Steps (Single Invocation) ──────────────────────────
 
     @Test
-    void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
+    void simpleSteps_producesCorrelatedWorkflowAndInvocationTracesInXRay() throws Exception {
         var startTime = Instant.now();
 
         // 1. Invoke the function (use unique input to avoid stale executions)
@@ -128,11 +129,12 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
         Thread.sleep(XRAY_INGESTION_DELAY.toMillis());
 
         // 3. Query X-Ray for the trace, retrying until durable spans appear
-        var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "create-greeting");
+        var invocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "create-greeting");
+        var workflowTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "Workflow");
 
         // 5. Verify span structure
         var segmentDocuments =
-                durableTrace.segments().stream().map(Segment::document).toList();
+                invocationTrace.segments().stream().map(Segment::document).toList();
         var allSegmentText = String.join("\n", segmentDocuments);
 
         // Verify expected span names appear in the trace
@@ -142,19 +144,20 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
         assertTrue(allSegmentText.contains("create-greeting"), "Expected create-greeting span in trace");
         assertTrue(allSegmentText.contains("transform"), "Expected transform span in trace");
 
-        // Verify all segments share the same trace ID (single unified trace)
-        var uniqueTraceIds =
-                durableTrace.segments().stream().map(seg -> durableTrace.id()).collect(Collectors.toSet());
-        assertEquals(1, uniqueTraceIds.size(), "All segments should belong to a single trace");
+        assertNotEquals(
+                workflowTrace.id(),
+                invocationTrace.id(),
+                "Workflow and Invocation spans must not create disconnected roots in one trace");
 
         System.out.println("✅ Simple steps test passed — "
-                + durableTrace.segments().size() + " segments in trace " + durableTrace.id());
+                + invocationTrace.segments().size() + " invocation segments correlated with Workflow trace "
+                + workflowTrace.id());
     }
 
     // ─── Test: Wait + Resume (Multi-Invocation) ─────────────────────────
 
     @Test
-    void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
+    void waitAndResume_preservesWorkflowCorrelationAcrossInvocations() throws Exception {
         var startTime = Instant.now();
 
         // 1. Invoke the function — will suspend on wait, then resume automatically
@@ -172,14 +175,22 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
         Thread.sleep(XRAY_INGESTION_DELAY.plus(Duration.ofSeconds(5)).toMillis());
 
         // 3. Query X-Ray for the trace, retrying until durable spans appear
-        var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "before-wait");
+        var firstInvocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "before-wait");
+        var secondInvocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "after-wait");
+        var workflowTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "Workflow");
 
         // 4. Verify multi-invocation trace structure
-        var segmentDocuments =
-                durableTrace.segments().stream().map(Segment::document).toList();
+        var segmentDocuments = new ArrayList();
+        segmentDocuments.addAll(
+                firstInvocationTrace.segments().stream().map(Segment::document).toList());
+        if (!firstInvocationTrace.id().equals(secondInvocationTrace.id())) {
+            segmentDocuments.addAll(secondInvocationTrace.segments().stream()
+                    .map(Segment::document)
+                    .toList());
+        }
         var allSegmentText = String.join("\n", segmentDocuments);
 
-        // Verify spans from BOTH invocations appear in the same trace
+        // Verify spans from both invocations were exported.
         assertTrue(allSegmentText.contains("before-wait"), "Expected before-wait span from first invocation");
         assertTrue(allSegmentText.contains("after-wait"), "Expected after-wait span from second invocation");
         assertTrue(allSegmentText.contains("pause"), "Expected wait:pause span in trace");
@@ -190,15 +201,13 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
                 invocationCount >= 2,
                 "Expected at least 2 invocation spans (multi-invocation), got " + invocationCount);
 
-        // Critical assertion: all segments under ONE trace (deterministic ID worked)
-        assertEquals(
-                1,
-                Set.of(durableTrace.id()).size(),
-                "All segments should belong to a single trace — deterministic trace ID must work across invocations");
+        assertNotEquals(workflowTrace.id(), firstInvocationTrace.id());
+        assertNotEquals(workflowTrace.id(), secondInvocationTrace.id());
 
-        System.out.println(
-                "✅ Wait + resume test passed — " + durableTrace.segments().size() + " segments across "
-                        + invocationCount + " invocations in trace " + durableTrace.id());
+        System.out.println("✅ Wait + resume test passed — "
+                + invocationCount
+                + " invocations correlated with Workflow trace "
+                + workflowTrace.id());
     }
 
     // ─── Helpers ─────────────────────────────────────────────────────────
@@ -206,8 +215,6 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
     /** Queries X-Ray for traces with retry logic to handle eventual consistency. */
     private List queryTracesWithRetry(Instant startTime, Instant endTime, String functionName)
             throws InterruptedException {
-        // Query by durable.invocation service — our spans are in a separate trace from Lambda's
-        // built-in X-Ray segment (durable backend propagates its own trace root)
         // Filter by the Lambda function's service name — each function has a unique one.
         // This avoids picking up traces from other durable functions that share service.name="invocation".
         var filterExpression = "service(\"" + functionNamePrefix + functionName + "\")";
@@ -285,8 +292,8 @@ private static String summarizeSegments(List segmentDocuments) {
      * Queries X-Ray for a trace containing durable spans, retrying until the expected span appears or timeout is
      * reached. Handles eventual consistency where the trace exists but OTLP-exported spans haven't been ingested yet.
      */
-    private software.amazon.awssdk.services.xray.model.Trace queryTraceWithDurableSpans(
-            Instant startTime, String functionName, String expectedSpanName) throws InterruptedException {
+    private Trace queryTraceWithDurableSpans(Instant startTime, String functionName, String expectedSpanName)
+            throws InterruptedException {
         var maxAttempts = 5;
         for (int attempt = 1; attempt <= maxAttempts; attempt++) {
             var traces = queryTracesWithRetry(startTime, Instant.now(), functionName);
@@ -295,7 +302,7 @@ private software.amazon.awssdk.services.xray.model.Trace queryTraceWithDurableSp
             }
 
             var traceIds = traces.stream().map(TraceSummary::id).toList();
-            var allTraces = new java.util.ArrayList();
+            var allTraces = new ArrayList();
             for (int i = 0; i < traceIds.size(); i += 5) {
                 var batch = traceIds.subList(i, Math.min(i + 5, traceIds.size()));
                 var batchResult = xrayClient.batchGetTraces(
diff --git a/otel-plugin/README.md b/otel-plugin/README.md
index 8e781e217..c14577d64 100644
--- a/otel-plugin/README.md
+++ b/otel-plugin/README.md
@@ -1,10 +1,12 @@
 # AWS Durable Execution SDK - OpenTelemetry Plugin
 
-OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK for Java. Emits distributed traces that correlate across multiple Lambda invocations of a single durable execution, producing deterministic span and trace IDs so that spans from different invocations are stitched into a single coherent trace.
+OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK for Java. Emits a deterministic Workflow trace for durable-execution correlation while keeping each Invocation span in the ambient Lambda trace.
 
 ## Features
 
-- **Deterministic Trace IDs**: All invocations of the same durable execution share a single trace, derived from the X-Ray trace header or execution ARN
+- **Deterministic Workflow Traces**: Workflow trace IDs are derived from the execution start time and ARN; stable span IDs are derived from the ARN
+- **Ambient Invocation Traces**: Invocation spans inherit the active Lambda/X-Ray context, or receive a fresh provider-generated root trace ID
+- **Scoped ID Generation**: Unrelated instrumentation scopes retain their provider's normal root trace ID generation
 - **Span-per-Operation**: Each durable operation (step, wait, map, etc.) gets its own span with accurate timing
 - **Attempt Spans**: Each user function execution (step attempt, child context run) gets a span, including retries
 - **Log Correlation**: Injects `trace_id`, `span_id`, and `traceSampled` into SLF4J MDC for end-to-end observability
@@ -90,7 +92,7 @@ Build the plugin layer ZIP with the OTel plugin JAR at `java/lib/aws-durable-exe
 
 ### 2. AWS X-Ray Active Tracing
 
-Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to derive deterministic trace IDs that remain consistent across all invocations of the same durable execution.
+Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to parent Invocation spans to the ambient Lambda/X-Ray trace. The Workflow trace remains independent and deterministic.
 
 **AWS Console:** Lambda > Configuration > Monitoring and operations tools > Active tracing > Enable
 
@@ -155,23 +157,29 @@ The function's execution role needs the `AWSXRayDaemonWriteAccess` managed polic
 
 ## Trace Structure
 
-The plugin creates spans at four levels:
+With `InvocationOtelPlugin`, the plugin creates two correlated traces:
 
 ```
-Workflow (deterministic ID, exported once on terminal invocation)
-Invocation
-├── fetch-data
-│   └── fetch-data attempt 1
-├── cool-down
-└── process
-    └── process attempt 1
+Workflow trace:
+Workflow (deterministic trace/span IDs, exported once)
+
+Ambient invocation trace:
+Lambda/X-Ray parent
+└── Invocation
+    ├── fetch-data
+    │   └── fetch-data attempt 1
+    ├── cool-down
+    └── process
+        └── process attempt 1
 ```
 
-- **Workflow span** — one logical span per durable execution with a deterministic ID derived from the ARN. Exported only on the terminal invocation (SUCCEEDED/FAILED). Serves as a correlation anchor across invocations.
-- **Invocation span** — one per Lambda invocation
+- **Workflow span** — one logical root per durable execution with a deterministic, X-Ray-compatible trace ID derived from the execution start time and ARN, plus a stable span ID derived from the ARN. Exported only on the terminal invocation (SUCCEEDED/FAILED).
+- **Invocation span** — one per Lambda invocation, parented to ambient context when available
 - **Operation span** — one per durable operation, named after your step/wait names
 - **Attempt span** — one per user function execution (retries produce additional attempt spans)
 
+Operation and attempt spans link to the Workflow span. `ExecutionOtelPlugin` reverses that relationship: operations are children of Workflow and link to the current Invocation span.
+
 ## Span Attributes
 
 ### Invocation Span
@@ -255,8 +263,9 @@ new InvocationOtelPlugin(
 
 ### ExecutionOtelPlugin
 
-The `ExecutionOtelPlugin` renders the Workflow span as the trace root with operations as siblings of the invocation
-span. It takes the same `(SdkTracerProviderBuilder, OtelPluginConfig)` constructor:
+The `ExecutionOtelPlugin` renders the Workflow span as the durable trace root with operations beneath it. Invocation
+spans remain in the ambient Lambda trace, and operations link to the Invocation that ran them. It takes the same
+`(SdkTracerProviderBuilder, OtelPluginConfig)` constructor:
 
 ```java
 // Default: ADOT Java agent global provider, X-Ray context extraction, MDC enabled
@@ -295,7 +304,7 @@ The plugin's spans do not appear as nested subsegments of the Lambda platform se
 
 ### Workflow Span
 
-The Workflow span appears as a separate root segment in the X-Ray trace because it uses `setNoParent()` with a deterministic span ID. This is expected — it serves as a correlation anchor across invocations.
+The Workflow span appears in a separate deterministic trace because it uses `setNoParent()`. Invocation spans remain in the ambient Lambda/X-Ray trace. Links correlate durable operations with the other trace.
 
 ## Verification
 
@@ -304,10 +313,10 @@ After deploying your function with the plugin configured:
 1. **Invoke your durable function** — trigger at least one execution that includes multiple steps or a wait/resume cycle.
 
 2. **Check CloudWatch console** — Navigate to CloudWatch > Traces. Enable "Group by nodes" to see:
-   - A Workflow span covering the entire execution
-   - An Invocation span per Lambda invocation
+   - A deterministic Workflow trace covering the entire execution
+   - Ambient Lambda traces containing one Invocation span per Lambda invocation
    - Child spans for each durable operation (named after your step names)
-   - All invocations of the same execution grouped under one trace ID
+   - Links between durable Workflow/operation spans and Invocation spans
 
 3. **Check log correlation** — Verify that the Logs section at the bottom of the trace view shows both platform logs and application logs correlated with the trace.
 
@@ -316,7 +325,7 @@ After deploying your function with the plugin configured:
 | Symptom | Likely Cause |
 |---------|-------------|
 | No traces appear | ADOT layer not added, or `AWS_LAMBDA_EXEC_WRAPPER` not set |
-| Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function |
+| Invocation spans are not parented to Lambda | X-Ray active tracing not enabled on the Lambda function |
 | Missing spans for some operations | Sampling is configured below 1.0 |
 | `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled |
 | Plugin spans missing but Lambda/runtime spans appear | Plugin jar not configured in `OTEL_JAVAAGENT_EXTENSIONS` |
diff --git a/otel-plugin/pom.xml b/otel-plugin/pom.xml
index 7c7ae1972..b117c3fa5 100644
--- a/otel-plugin/pom.xml
+++ b/otel-plugin/pom.xml
@@ -33,19 +33,12 @@
             ${opentelemetry.version}
         
 
-        
+        
         
             io.opentelemetry
             opentelemetry-sdk
             ${opentelemetry.version}
         
-        
-        
-            io.opentelemetry
-            opentelemetry-exporter-otlp
-            ${opentelemetry.version}
-        
         
         
             io.opentelemetry
@@ -61,13 +54,6 @@
             ${opentelemetry.version}
         
 
-        
-        
-            io.opentelemetry.semconv
-            opentelemetry-semconv
-            1.43.0
-        
-
         
         
             org.slf4j
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java
index ee15f2c14..6fc101b93 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java
@@ -2,42 +2,60 @@
 // SPDX-License-Identifier: Apache-2.0
 package software.amazon.lambda.durable.otel;
 
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanBuilder;
 import io.opentelemetry.sdk.trace.IdGenerator;
+import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.concurrent.atomic.AtomicReference;
+import java.time.Instant;
 
 /**
  * Generates deterministic trace and span IDs for durable execution observability.
  *
- * 
    Trace ID resolution order:
+ * 
    The durable plugins use short-lived ID overrides around their own {@link SpanBuilder#startSpan()} calls. Outside
+ * those scopes, generation delegates to the fallback generator so unrelated instrumentation keeps normal root trace ID
+ * generation. Scoped values are also bridged through thread-keyed system properties because the application plugin and
+ * Java-agent extension may load this class in different class loaders.
  *
- * 
      
- *   
    1. If an extracted trace ID is set (from {@code _X_AMZN_TRACE_ID}), use it. The durable execution backend
- *       propagates the same Root to all invocations, so this naturally unifies the trace.
- *   
    2. If no extracted trace ID is available (local tests, non-Lambda environments), derive a deterministic trace ID
- *       from the execution ARN using SHA-256.
- *   
    3. If neither is set, fall back to random generation.
- * 
    
- *
- * 
    Span IDs for operations are deterministic (derived from execution ARN + operation ID), ensuring the same operation
- * produces the same span across invocations. When no pending operation ID is set, falls back to random generation.
+ * 
    The existing setter methods remain available for callers that use this class directly. Plugin code does not use
+ * that persistent mode.
  */
 public class DeterministicIdGenerator implements IdGenerator {
 
-    private static final IdGenerator RANDOM = IdGenerator.random();
     private static final String PROPERTY_PREFIX = "software.amazon.lambda.durable.otel.";
-    private static final String EXTRACTED_TRACE_ID_PROPERTY = PROPERTY_PREFIX + "extractedTraceId";
-    private static final String DURABLE_EXECUTION_ARN_PROPERTY = PROPERTY_PREFIX + "durableExecutionArn";
-    private static final String PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingSpanOperationId.";
-    private static final String PENDING_RAW_SPAN_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingRawSpanId.";
+    private static final String SCOPED_TRACE_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "scopedTraceId.";
+    private static final String SCOPED_SPAN_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "scopedSpanId.";
 
-    private final AtomicReference extractedTraceId = new AtomicReference<>(null);
-    private final AtomicReference arnDerivedTraceId = new AtomicReference<>(null);
+    private final IdGenerator fallbackIdGenerator;
+    private final ThreadLocal extractedTraceId = new ThreadLocal<>();
+    private final ThreadLocal arnDerivedTraceId = new ThreadLocal<>();
     private final ThreadLocal pendingSpanOperationId = new ThreadLocal<>();
     private final ThreadLocal pendingRawSpanId = new ThreadLocal<>();
-    private final AtomicReference durableExecutionArn = new AtomicReference<>(null);
+    private final ThreadLocal scopedIds = new ThreadLocal<>();
+    private final ThreadLocal durableExecutionArn = new ThreadLocal<>();
+
+    /** Creates a generator that delegates non-overridden IDs to OpenTelemetry's random generator. */
+    public DeterministicIdGenerator() {
+        this(IdGenerator.random());
+    }
+
+    DeterministicIdGenerator(IdGenerator fallbackIdGenerator) {
+        this.fallbackIdGenerator = fallbackIdGenerator;
+    }
+
+    // The SDK builder exposes only a setter. Read its configured generator so unrelated spans retain the caller's
+    // existing ID policy while durable spans use short-lived overrides.
+    static DeterministicIdGenerator installOn(SdkTracerProviderBuilder builder) {
+        var currentGenerator = configuredIdGenerator(builder);
+        if (currentGenerator instanceof DeterministicIdGenerator deterministicIdGenerator) {
+            return deterministicIdGenerator;
+        }
+        var deterministicIdGenerator = new DeterministicIdGenerator(currentGenerator);
+        builder.setIdGenerator(deterministicIdGenerator);
+        return deterministicIdGenerator;
+    }
 
     /**
      * Sets an externally extracted trace ID (e.g., from the X-Ray trace header). This takes highest priority for trace
@@ -46,8 +64,7 @@ public class DeterministicIdGenerator implements IdGenerator {
      * @param traceId 32-char lowercase hex trace ID
      */
     public void setExtractedTraceId(String traceId) {
-        this.extractedTraceId.set(traceId);
-        setOrClearProperty(EXTRACTED_TRACE_ID_PROPERTY, traceId);
+        setOrRemove(extractedTraceId, traceId);
     }
 
     /**
@@ -57,9 +74,8 @@ public void setExtractedTraceId(String traceId) {
      * @param arn the durable execution ARN
      */
     public void setDurableExecutionArn(String arn) {
-        this.durableExecutionArn.set(arn);
-        this.arnDerivedTraceId.set(arn != null ? generateTraceIdFromArn(arn) : null);
-        setOrClearProperty(DURABLE_EXECUTION_ARN_PROPERTY, arn);
+        setOrRemove(durableExecutionArn, arn);
+        setOrRemove(arnDerivedTraceId, arn != null ? generateTraceIdFromArn(arn) : null);
     }
 
     /**
@@ -68,8 +84,7 @@ public void setDurableExecutionArn(String arn) {
      * @param operationId the operation ID to derive the span ID from
      */
     public void setNextSpanOperationId(String operationId) {
-        this.pendingSpanOperationId.set(operationId);
-        setOrClearProperty(pendingSpanOperationIdProperty(), operationId);
+        setOrRemove(pendingSpanOperationId, operationId);
     }
 
     /**
@@ -80,8 +95,7 @@ public void setNextSpanOperationId(String operationId) {
      * @param spanId a 16-char lowercase hex span ID
      */
     public void setNextSpanId(String spanId) {
-        this.pendingRawSpanId.set(spanId);
-        setOrClearProperty(pendingRawSpanIdProperty(), spanId);
+        setOrRemove(pendingRawSpanId, spanId);
     }
 
     /**
@@ -103,7 +117,16 @@ public String generateSpanIdForOperation(String operationId) {
      * @return a deterministic 16-char hex span ID
      */
     public String generateWorkflowSpanId() {
-        var arn = durableExecutionArn.get();
+        return generateWorkflowSpanId(durableExecutionArn.get());
+    }
+
+    String generateTraceIdForExecution(String arn, Instant executionStartTime) {
+        var timestamp = executionStartTime != null ? executionStartTime : Instant.now();
+        var timestampHex = String.format("%08x", timestamp.getEpochSecond() & 0xffffffffL);
+        return timestampHex + sha256(arn != null ? arn : "").substring(0, 24);
+    }
+
+    String generateWorkflowSpanId(String arn) {
         var seed = "workflow:" + (arn != null ? arn : "");
         var spanId = sha256(seed).substring(0, 16);
         if (spanId.equals("0000000000000000")) {
@@ -112,50 +135,97 @@ public String generateWorkflowSpanId() {
         return spanId;
     }
 
+    String generateSpanIdForOperation(String arn, String operationId) {
+        return generateSpanIdFromOperation(arn, operationId);
+    }
+
+    Span startSpan(SpanBuilder spanBuilder, String traceId, String spanId) {
+        try (var ignored = useIds(traceId, spanId)) {
+            return spanBuilder.startSpan();
+        }
+    }
+
+    IdScope useIds(String traceId, String spanId) {
+        var previousOverride = scopedIds.get();
+        var previousTraceId = System.getProperty(scopedTraceIdProperty());
+        var previousSpanId = System.getProperty(scopedSpanIdProperty());
+
+        scopedIds.set(new IdOverride(traceId, spanId));
+        setOrClearProperty(scopedTraceIdProperty(), traceId);
+        setOrClearProperty(scopedSpanIdProperty(), spanId);
+
+        return new IdScope() {
+            private boolean closed;
+
+            @Override
+            public void close() {
+                if (closed) {
+                    return;
+                }
+                closed = true;
+                if (previousOverride == null) {
+                    scopedIds.remove();
+                } else {
+                    scopedIds.set(previousOverride);
+                }
+                setOrClearProperty(scopedTraceIdProperty(), previousTraceId);
+                setOrClearProperty(scopedSpanIdProperty(), previousSpanId);
+            }
+        };
+    }
+
     @Override
     public String generateTraceId() {
+        var override = currentOverride();
+        if (override != null && override.traceId() != null) {
+            return override.traceId();
+        }
+
         // Priority 1: extracted from X-Ray header (backend propagates same Root across invocations)
         var extracted = extractedTraceId.get();
-        if (extracted == null) {
-            extracted = System.getProperty(EXTRACTED_TRACE_ID_PROPERTY);
-        }
         if (extracted != null) {
             return extracted;
         }
         // Priority 2: deterministic from execution ARN (local tests, non-Lambda)
         var arnDerived = arnDerivedTraceId.get();
-        if (arnDerived == null) {
-            var arn = System.getProperty(DURABLE_EXECUTION_ARN_PROPERTY);
-            arnDerived = arn != null ? generateTraceIdFromArn(arn) : null;
-        }
         if (arnDerived != null) {
             return arnDerived;
         }
         // Priority 3: random fallback
-        return RANDOM.generateTraceId();
+        return fallbackIdGenerator.generateTraceId();
     }
 
     @Override
     public String generateSpanId() {
-        var raw = pendingRawSpanId.get();
-        if (raw == null) {
-            raw = System.getProperty(pendingRawSpanIdProperty());
+        var override = currentOverride();
+        if (override != null && override.spanId() != null) {
+            consumeScopedSpanId(override);
+            return override.spanId();
         }
+
+        var raw = pendingRawSpanId.get();
         if (raw != null) {
             pendingRawSpanId.remove();
-            System.clearProperty(pendingRawSpanIdProperty());
             return raw;
         }
         var operationId = pendingSpanOperationId.get();
-        if (operationId == null) {
-            operationId = System.getProperty(pendingSpanOperationIdProperty());
-        }
         if (operationId != null) {
             pendingSpanOperationId.remove();
-            System.clearProperty(pendingSpanOperationIdProperty());
             return generateSpanIdFromOperation(operationId);
         }
-        return RANDOM.generateSpanId();
+        return fallbackIdGenerator.generateSpanId();
+    }
+
+    @Override
+    public boolean generatesRandomTraceIds() {
+        var override = currentOverride();
+        if (override != null && override.traceId() != null) {
+            return false;
+        }
+        if (extractedTraceId.get() != null || arnDerivedTraceId.get() != null) {
+            return false;
+        }
+        return fallbackIdGenerator.generatesRandomTraceIds();
     }
 
     /** Generates a deterministic trace ID from an execution ARN using SHA-256 truncated to 32 hex chars. */
@@ -169,14 +239,49 @@ private String generateTraceIdFromArn(String arn) {
      */
     private String generateSpanIdFromOperation(String operationId) {
         var arn = durableExecutionArn.get();
-        if (arn == null) {
-            arn = System.getProperty(DURABLE_EXECUTION_ARN_PROPERTY);
-        }
+        return generateSpanIdFromOperation(arn, operationId);
+    }
+
+    private String generateSpanIdFromOperation(String arn, String operationId) {
         var input = arn != null ? arn + ":" + operationId : operationId;
         var hash = sha256(input);
         return hash.substring(0, 16);
     }
 
+    private IdOverride currentOverride() {
+        var override = scopedIds.get();
+        if (override != null) {
+            return override;
+        }
+        var traceId = System.getProperty(scopedTraceIdProperty());
+        var spanId = System.getProperty(scopedSpanIdProperty());
+        return traceId != null || spanId != null ? new IdOverride(traceId, spanId) : null;
+    }
+
+    private static IdGenerator configuredIdGenerator(SdkTracerProviderBuilder builder) {
+        for (var field : builder.getClass().getDeclaredFields()) {
+            if (!IdGenerator.class.isAssignableFrom(field.getType())) {
+                continue;
+            }
+            try {
+                if (!field.trySetAccessible()) {
+                    break;
+                }
+                return (IdGenerator) field.get(builder);
+            } catch (IllegalAccessException e) {
+                throw new IllegalStateException("Unable to read the configured OpenTelemetry ID generator", e);
+            }
+        }
+        throw new IllegalStateException("Unable to locate the configured OpenTelemetry ID generator");
+    }
+
+    private void consumeScopedSpanId(IdOverride override) {
+        if (scopedIds.get() != null) {
+            scopedIds.set(new IdOverride(override.traceId(), null));
+        }
+        System.clearProperty(scopedSpanIdProperty());
+    }
+
     private static String sha256(String input) {
         try {
             var digest = MessageDigest.getInstance("SHA-256");
@@ -192,22 +297,19 @@ private static String sha256(String input) {
     }
 
     static void clearSharedStateForTest() {
-        System.clearProperty(EXTRACTED_TRACE_ID_PROPERTY);
-        System.clearProperty(DURABLE_EXECUTION_ARN_PROPERTY);
         System.getProperties().stringPropertyNames().stream()
-                .filter(name -> name.startsWith(PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX)
-                        || name.startsWith(PENDING_RAW_SPAN_ID_PROPERTY_PREFIX))
+                .filter(name -> name.startsWith(SCOPED_TRACE_ID_PROPERTY_PREFIX)
+                        || name.startsWith(SCOPED_SPAN_ID_PROPERTY_PREFIX))
                 .toList()
                 .forEach(System::clearProperty);
     }
 
-    private static String pendingSpanOperationIdProperty() {
-        return PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX
-                + Thread.currentThread().getId();
+    private static String scopedTraceIdProperty() {
+        return SCOPED_TRACE_ID_PROPERTY_PREFIX + Thread.currentThread().getId();
     }
 
-    private static String pendingRawSpanIdProperty() {
-        return PENDING_RAW_SPAN_ID_PROPERTY_PREFIX + Thread.currentThread().getId();
+    private static String scopedSpanIdProperty() {
+        return SCOPED_SPAN_ID_PROPERTY_PREFIX + Thread.currentThread().getId();
     }
 
     private static void setOrClearProperty(String key, String value) {
@@ -217,4 +319,19 @@ private static void setOrClearProperty(String key, String value) {
             System.setProperty(key, value);
         }
     }
+
+    private static void setOrRemove(ThreadLocal target, String value) {
+        if (value == null) {
+            target.remove();
+        } else {
+            target.set(value);
+        }
+    }
+
+    private record IdOverride(String traceId, String spanId) {}
+
+    interface IdScope extends AutoCloseable {
+        @Override
+        void close();
+    }
 }
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java
index 5c48dc2e2..57f65d1f9 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java
@@ -56,9 +56,8 @@
  * it. Both plugins share {@link DeterministicIdGenerator}, {@link ContextExtractor}, {@link SpanAttributes}, and
  * {@link MdcSpanEnricher}.
  *
- * 
    Trace ID resolution matches {@link InvocationOtelPlugin}: the X-Ray trace ID from {@code _X_AMZN_TRACE_ID} when
- * available (the backend propagates the same Root to all invocations, unifying the trace), else a deterministic trace
- * ID derived from the execution ARN.
+ * 
    The Workflow trace ID is derived from the execution start time and ARN, and is independent of the ambient
+ * Lambda/X-Ray trace. Invocation spans inherit the active ambient context, or extracted upstream context as a fallback.
  *
  * 
    Status mapping (parity with the Python/JS references):
  *
@@ -91,6 +90,7 @@ public class ExecutionOtelPlugin implements DurableExecutionPlugin {
     private volatile Span workflowSpan;
     private volatile Span invocationSpan;
     private volatile String durableExecutionArn;
+    private volatile String workflowTraceId;
 
     // Thread-safe storage for operation spans (keyed by operationId) — open spans that need ending
     private final ConcurrentHashMap operationSpans = new ConcurrentHashMap<>();
@@ -109,7 +109,7 @@ public class ExecutionOtelPlugin implements DurableExecutionPlugin {
      * 
    Uses the provided tracer provider builder. For ADOT Java agent usage, prefer {@link #ExecutionOtelPlugin()}
      * with the plugin jar configured through {@code OTEL_JAVAAGENT_EXTENSIONS}.
      *
-     * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+     * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
      */
     public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
         this(tracerProviderBuilder, OtelPluginConfig.defaults());
@@ -138,14 +138,13 @@ public ExecutionOtelPlugin() {
      *     OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
      * }
    * - * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) + * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped) * @param config the plugin configuration */ public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) { - this.idGenerator = new DeterministicIdGenerator(); + this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder); - this.sdkTracerProvider = - tracerProviderBuilder.setIdGenerator(idGenerator).build(); + this.sdkTracerProvider = tracerProviderBuilder.build(); this.tracer = sdkTracerProvider.get(config.instrumentationName()); this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); @@ -156,10 +155,8 @@ public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelP /** * Creates a Workflow-rooted OTel plugin from configuration alone (no caller-supplied tracer provider builder). * - *

    The provider is taken from {@link OtelPluginConfig#providerSource()}: {@link ProviderSource#GLOBAL} uses the - * ADOT/global provider, otherwise the default {@link ProviderSource#AUTO_OTLP} builds a plugin-owned OTLP/HTTP - * provider (matching the JavaScript and Python SDK plugins). {@link ProviderSource#EXPLICIT} is rejected here — - * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for that. + *

    The config-only constructor uses the ADOT/global provider. {@link ProviderSource#EXPLICIT} is rejected here; + * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for an application-owned provider. * * @param config the plugin configuration * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT} @@ -187,18 +184,10 @@ public ProviderSource providerSource() { public void onInvocationStart(InvocationInfo info) { this.durableExecutionArn = info.durableExecutionArn(); - // Set execution ARN for deterministic span/trace ID generation - idGenerator.setDurableExecutionArn(info.durableExecutionArn()); - - // Extract trace context from the environment (X-Ray header), falling back to the ambient OTel span. - var extractedContext = contextExtractor.extract(); - if (extractedContext == null) { - extractedContext = extractCurrentSpanContext(); - } - if (extractedContext != null) { - idGenerator.setExtractedTraceId(extractedContext.traceId()); - } else { - idGenerator.setExtractedTraceId(null); + // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context. + var invocationParent = extractCurrentSpanContext(); + if (invocationParent == null) { + invocationParent = contextExtractor.extract(); } // Workflow root span — deterministic span ID from the ARN, no parent. Recreated every invocation with the @@ -209,14 +198,16 @@ public void onInvocationStart(InvocationInfo info) { .setNoParent() .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn()) .setStartTimestamp(info.executionStartTime() != null ? info.executionStartTime() : Instant.now()); - idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId()); - workflowSpan = workflowSpanBuilder.startSpan(); + workflowTraceId = + idGenerator.generateTraceIdForExecution(info.durableExecutionArn(), info.executionStartTime()); + var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn()); + workflowSpan = idGenerator.startSpan(workflowSpanBuilder, workflowTraceId, workflowSpanId); Context parentContext; - if (extractedContext != null && extractedContext.parentSpanId() != null) { + if (invocationParent != null && invocationParent.parentSpanId() != null) { var parentSpanContext = SpanContext.createFromRemoteParent( - extractedContext.traceId(), - extractedContext.parentSpanId(), + invocationParent.traceId(), + invocationParent.parentSpanId(), TraceFlags.getSampled(), TraceState.getDefault()); parentContext = Context.root().with(Span.wrap(parentSpanContext)); @@ -239,8 +230,9 @@ public void onInvocationStart(InvocationInfo info) { // Inject MDC on the handler thread so handler-level logs (between steps) have trace context. if (enableMdc) { - var traceId = idGenerator.generateTraceId(); - MDC.put(MdcSpanEnricher.MDC_TRACE_ID, traceId); + MDC.put( + MdcSpanEnricher.MDC_TRACE_ID, + invocationSpan.getSpanContext().getTraceId()); } } @@ -316,10 +308,6 @@ public void onOperationStart(OperationInfo info) { var parentContext = resolveParentContext(info.parentId()); - // Always use a deterministic span ID keyed by operation ID (regardless of replay) so a suspended-then-resumed - // operation stitches into a single logical span across invocations. - idGenerator.setNextSpanOperationId(info.id()); - var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name())) .setParent(parentContext) .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn) @@ -337,7 +325,8 @@ public void onOperationStart(OperationInfo info) { spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType()); } - var span = spanBuilder.startSpan(); + var operationSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id()); + var span = idGenerator.startSpan(spanBuilder, null, operationSpanId); // Store the open span — will be ended in onOperationEnd or onInvocationEnd operationSpans.put(info.id(), span); @@ -376,7 +365,6 @@ public void onOperationEnd(OperationEndInfo info) { // now, using its deterministic span ID (stable across the execution), plus a link to the invocation // that completed it. operationContexts.remove(info.id()); - idGenerator.setNextSpanOperationId(info.id()); var parentContext = resolveParentContext(info.parentId()); @@ -397,7 +385,8 @@ public void onOperationEnd(OperationEndInfo info) { spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType()); } - var continuationSpan = spanBuilder.startSpan(); + var operationSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id()); + var continuationSpan = idGenerator.startSpan(spanBuilder, null, operationSpanId); if (info.status() != null) { continuationSpan.setAttribute(DURABLE_OPERATION_STATUS, info.status()); @@ -556,10 +545,9 @@ private Context resolveParentContext(String parentId) { return Context.current().with(Span.wrap(parentSpanContext)); } // Parent operation from a prior invocation — create a non-recording placeholder with its deterministic ID. - var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(parentId); - var traceId = idGenerator.generateTraceId(); + var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, parentId); var placeholderContext = SpanContext.create( - traceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault()); + workflowTraceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault()); return Context.current().with(Span.wrap(placeholderContext)); } // No parent operation — hang off the Workflow root span. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java index 6aaacd2e9..d381732cb 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java @@ -5,9 +5,8 @@ /** * Trace context extracted from the Lambda runtime environment. * - *

    Contains the trace ID (always present) and an optional parent span ID. When the durable execution backend - * propagates the same X-Ray Root across all invocations, the trace ID will be consistent, enabling spans from different - * invocations to be stitched into a single trace. + *

    Contains the trace ID (always present) and an optional parent span ID used to parent an Invocation span to ambient + * Lambda/X-Ray context. * * @param traceId 32-character lowercase hex trace ID (OTel format, no dashes) * @param parentSpanId 16-character lowercase hex parent span ID (may be null if no parent available) diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 36e741f37..cd69225ef 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -50,22 +50,16 @@ * *

      *
    • When using the ADOT Java agent ({@link #InvocationOtelPlugin()}), the Workflow span appears as a separate root - * segment in the X-Ray trace because it uses {@code setNoParent()} with a deterministic span ID. This is expected - * — it serves as a correlation anchor across invocations. The Invocation span and its children nest under the - * ADOT agent's Lambda segment as subsegments. + * trace because it uses {@code setNoParent()} with deterministic trace and span IDs. It serves as a correlation + * anchor across invocations. The Invocation span and its children nest under the ADOT agent's Lambda segment as + * subsegments. *
    • When using a custom {@link io.opentelemetry.sdk.trace.SdkTracerProviderBuilder} (no ADOT agent), the Workflow - * span is similarly unparented but all spans share the same trace ID. Operation and attempt spans link to the - * Workflow span for execution-level correlation. + * span is similarly unparented. Invocation roots receive provider-generated trace IDs, and operation and attempt + * spans link to the Workflow span for execution-level correlation. *
    * - *

    Trace ID resolution: - * - *

      - *
    1. Uses the X-Ray trace ID from {@code _X_AMZN_TRACE_ID} when available. The durable execution backend propagates - * the same Root to all invocations of the same execution, naturally unifying the trace. - *
    2. Falls back to a deterministic trace ID derived from the execution ARN (for local tests or non-Lambda - * environments). - *
    + *

    The Workflow trace ID is derived from the execution start time and ARN, and is independent of the ambient + * Lambda/X-Ray trace. Invocation spans inherit the active ambient context, or extracted upstream context as a fallback. * *

    Requires the ADOT Lambda Layer for trace export. Configure with: * @@ -82,8 +76,8 @@ * operation, attempt) do not appear as nested subsegments of the Lambda platform segment. This is a known limitation of * the OTLP-to-X-Ray conversion: the ADOT collector cannot attach OTLP-exported spans as subsegments of the Lambda * service's native X-Ray segment because that segment is created outside the OTLP pipeline. Use the "Group by nodes" - * view to see the full span hierarchy correctly — it stitches all spans together by trace ID and parent-child - * relationships regardless of segment boundaries. + * view to inspect parent-child relationships within the ambient Invocation trace and the links to the independent + * Workflow trace. * *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple * threads. @@ -133,7 +127,7 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin { * SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter))); * } * - * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) + * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped) */ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { this(tracerProviderBuilder, OtelPluginConfig.defaults()); @@ -162,14 +156,13 @@ public InvocationOtelPlugin() { * OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build()); * } * - * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden) + * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped) * @param config the plugin configuration */ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) { - this.idGenerator = new DeterministicIdGenerator(); + this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder); - this.sdkTracerProvider = - tracerProviderBuilder.setIdGenerator(idGenerator).build(); + this.sdkTracerProvider = tracerProviderBuilder.build(); this.tracer = sdkTracerProvider.get(config.instrumentationName()); this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); @@ -180,10 +173,8 @@ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, Otel /** * Creates an OTel plugin from configuration alone (no caller-supplied tracer provider builder). * - *

    The provider is taken from {@link OtelPluginConfig#providerSource()}: {@link ProviderSource#GLOBAL} uses the - * ADOT/global provider, otherwise the default {@link ProviderSource#AUTO_OTLP} builds a plugin-owned OTLP/HTTP - * provider (matching the JavaScript and Python SDK plugins). {@link ProviderSource#EXPLICIT} is rejected here — - * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for that. + *

    The config-only constructor uses the ADOT/global provider. {@link ProviderSource#EXPLICIT} is rejected here; + * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for an application-owned provider. * * @param config the plugin configuration * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT} @@ -211,22 +202,11 @@ public ProviderSource providerSource() { public void onInvocationStart(InvocationInfo info) { this.durableExecutionArn = info.durableExecutionArn(); - // Set execution ARN for deterministic span ID generation - idGenerator.setDurableExecutionArn(info.durableExecutionArn()); - - // Extract trace context from environment (X-Ray header) - var extractedContext = contextExtractor.extract(); - if (extractedContext == null) { - extractedContext = extractCurrentSpanContext(); - } - - if (extractedContext != null) { - // Use the X-Ray trace ID — backend propagates same Root across all invocations - idGenerator.setExtractedTraceId(extractedContext.traceId()); - } else { - idGenerator.setExtractedTraceId(null); + // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context. + var invocationParent = extractCurrentSpanContext(); + if (invocationParent == null) { + invocationParent = contextExtractor.extract(); } - // If no extracted context, idGenerator falls back to ARN-derived trace ID // Workflow root span — one logical span per durable execution, created unconditionally (independent of the // X-Ray parent below). Deterministic span ID from the ARN so it is the same across invocations; exported once, @@ -237,17 +217,19 @@ public void onInvocationStart(InvocationInfo info) { .setNoParent() .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn()) .setStartTimestamp(info.executionStartTime() != null ? info.executionStartTime() : Instant.now()); - idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId()); - workflowSpan = workflowSpanBuilder.startSpan(); + var workflowTraceId = + idGenerator.generateTraceIdForExecution(info.durableExecutionArn(), info.executionStartTime()); + var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn()); + workflowSpan = idGenerator.startSpan(workflowSpanBuilder, workflowTraceId, workflowSpanId); // Determine parent context for the invocation span. Context parentContext; - if (extractedContext != null && extractedContext.parentSpanId() != null) { + if (invocationParent != null && invocationParent.parentSpanId() != null) { // Reconstruct a remote parent from the extracted trace context (X-Ray header or current span). // This connects plugin spans to the Lambda service's X-Ray segments. var parentSpanContext = SpanContext.createFromRemoteParent( - extractedContext.traceId(), - extractedContext.parentSpanId(), + invocationParent.traceId(), + invocationParent.parentSpanId(), TraceFlags.getSampled(), TraceState.getDefault()); parentContext = Context.root().with(Span.wrap(parentSpanContext)); @@ -271,8 +253,9 @@ public void onInvocationStart(InvocationInfo info) { // Inject MDC on the handler thread so handler-level logs (between steps) have trace context. // This runs on the same thread as context.getLogger() calls in the handler. if (enableMdc) { - var traceId = idGenerator.generateTraceId(); - MDC.put(MdcSpanEnricher.MDC_TRACE_ID, traceId); + MDC.put( + MdcSpanEnricher.MDC_TRACE_ID, + invocationSpan.getSpanContext().getTraceId()); } } @@ -363,19 +346,6 @@ public void onOperationStart(OperationInfo info) { .setAttribute(DURABLE_OPERATION_TYPE, info.type()) .setAttribute(DURABLE_OPERATION_STATUS, info.status() != null ? info.status() : "STARTED"); - if (info.isReplay()) { - // Operation was already started in a prior invocation — use a random span ID - // and add a Link to the deterministic span from the original invocation for correlation. - var deterministicSpanId = idGenerator.generateSpanIdForOperation(info.id()); - var traceId = idGenerator.generateTraceId(); - var linkedSpanContext = - SpanContext.create(traceId, deterministicSpanId, TraceFlags.getSampled(), TraceState.getDefault()); - spanBuilder.addLink(linkedSpanContext); - } else { - // First execution — use deterministic span ID so continuations can link back - idGenerator.setNextSpanOperationId(info.id()); - } - // Link to the Workflow span for execution-level correlation (operation stays parented to the invocation span). addWorkflowLink(spanBuilder); @@ -386,7 +356,10 @@ public void onOperationStart(OperationInfo info) { spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType()); } - var span = spanBuilder.startSpan(); + var span = info.isReplay() + ? spanBuilder.startSpan() + : idGenerator.startSpan( + spanBuilder, null, idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id())); // Store the open span — will be ended in onOperationEnd or onInvocationEnd operationSpans.put(info.id(), span); @@ -421,18 +394,10 @@ public void onOperationEnd(OperationEndInfo info) { } span.end(); } else { - // Operation was started in a prior invocation — create a continuation span with Link - // to the deterministic span ID from the original invocation. - var deterministicSpanId = idGenerator.generateSpanIdForOperation(info.id()); - var traceId = idGenerator.generateTraceId(); - var linkedSpanContext = - SpanContext.create(traceId, deterministicSpanId, TraceFlags.getSampled(), TraceState.getDefault()); - var parentContext = resolveParentContext(info.parentId()); var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name())) .setParent(parentContext) - .addLink(linkedSpanContext) .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn) .setAttribute(DURABLE_OPERATION_ID, info.id()) .setAttribute(DURABLE_OPERATION_TYPE, info.type()); @@ -599,12 +564,6 @@ private Context resolveParentContext(String parentId) { if (parentSpanContext != null) { return Context.current().with(Span.wrap(parentSpanContext)); } - // Parent operation from a prior invocation — create non-recording placeholder - var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(parentId); - var traceId = idGenerator.generateTraceId(); - var placeholderContext = SpanContext.create( - traceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault()); - return Context.current().with(Span.wrap(placeholderContext)); } // Fall back to invocation span as parent if (invocationSpan != null) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java index 33fd6ea66..74c7fc1d3 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java @@ -5,14 +5,20 @@ import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; -/** Installs the durable-execution ID generator when the OpenTelemetry Java agent auto-configures the SDK. */ +/** Wraps the Java agent's configured ID generator with scoped durable-execution overrides. */ public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider { - private static final DeterministicIdGenerator ID_GENERATOR = new DeterministicIdGenerator(); - @Override public void customize(AutoConfigurationCustomizer autoConfiguration) { OtelPluginAutoConfigurationState.markInstalled(); - autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR)); + autoConfiguration.addTracerProviderCustomizer((builder, config) -> { + DeterministicIdGenerator.installOn(builder); + return builder; + }); + } + + @Override + public int order() { + return Integer.MAX_VALUE; } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java index 3cf021ad8..b6dd40b82 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java @@ -2,8 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.otel; -import java.util.Map; - /** * Immutable configuration for {@link InvocationOtelPlugin} and {@link ExecutionOtelPlugin}. * @@ -39,8 +37,6 @@ public final class OtelPluginConfig { private final String workflowSpanName; private final String instrumentationName; private final ProviderSource providerSource; - private final String otlpEndpoint; - private final Map otlpHeaders; private OtelPluginConfig(Builder builder) { this.contextExtractor = @@ -51,8 +47,6 @@ private OtelPluginConfig(Builder builder) { this.instrumentationName = builder.instrumentationName != null ? builder.instrumentationName : DEFAULT_INSTRUMENTATION_NAME; this.providerSource = builder.providerSource != null ? builder.providerSource : ProviderSource.GLOBAL; - this.otlpEndpoint = builder.otlpEndpoint; - this.otlpHeaders = builder.otlpHeaders != null ? Map.copyOf(builder.otlpHeaders) : Map.of(); } /** Returns a new builder with all fields defaulted. */ @@ -87,8 +81,7 @@ public String instrumentationName() { /** * The tracer-provider source to use when no {@code SdkTracerProviderBuilder} is supplied (the config-only - * constructors). {@link ProviderSource#GLOBAL} (the default) uses the globally configured (ADOT) provider; - * {@link ProviderSource#AUTO_OTLP} makes the plugin build and own an OTLP/HTTP provider. + * constructors). {@link ProviderSource#GLOBAL} (the default) uses the globally configured ADOT provider. * *

    {@link ProviderSource#EXPLICIT} is not valid here — it is implied by using a {@code (SdkTracerProviderBuilder, * OtelPluginConfig)} constructor and is rejected by the config-only constructors. @@ -97,16 +90,6 @@ public ProviderSource providerSource() { return providerSource; } - /** OTLP/HTTP endpoint for the auto-configured provider, or {@code null} to use the OTel default / env var. */ - public String otlpEndpoint() { - return otlpEndpoint; - } - - /** Extra headers sent by the auto-configured OTLP exporter (never {@code null}). */ - public Map otlpHeaders() { - return otlpHeaders; - } - /** Builder for {@link OtelPluginConfig}. */ public static final class Builder { @@ -115,8 +98,6 @@ public static final class Builder { private String workflowSpanName; private String instrumentationName; private ProviderSource providerSource = ProviderSource.GLOBAL; - private String otlpEndpoint; - private Map otlpHeaders; private Builder() {} @@ -166,15 +147,14 @@ public Builder instrumentationName(String instrumentationName) { } /** - * Sets the tracer-provider source used when no {@code SdkTracerProviderBuilder} is supplied. Defaults to - * {@link ProviderSource#GLOBAL} (the globally configured ADOT provider); pass {@link ProviderSource#AUTO_OTLP} - * to make the plugin build and own an OTLP/HTTP provider. A {@code null} falls back to + * Sets the tracer-provider source used when no {@code SdkTracerProviderBuilder} is supplied. The config-only + * constructors accept {@link ProviderSource#GLOBAL}; a {@code null} also falls back to * {@link ProviderSource#GLOBAL}. * *

    {@link ProviderSource#EXPLICIT} is not accepted through the config-only constructors — supply a * {@code SdkTracerProviderBuilder} via the two-arg constructor instead. * - * @param providerSource the provider source, {@link ProviderSource#GLOBAL} or {@link ProviderSource#AUTO_OTLP} + * @param providerSource the provider source * @return this builder */ public Builder providerSource(ProviderSource providerSource) { @@ -182,29 +162,6 @@ public Builder providerSource(ProviderSource providerSource) { return this; } - /** - * Sets the OTLP/HTTP endpoint for the auto-configured provider. When null, the OTel default (or - * {@code OTEL_EXPORTER_OTLP_ENDPOINT}) is used. - * - * @param otlpEndpoint the OTLP/HTTP traces endpoint - * @return this builder - */ - public Builder otlpEndpoint(String otlpEndpoint) { - this.otlpEndpoint = otlpEndpoint; - return this; - } - - /** - * Sets extra headers for the auto-configured OTLP exporter (e.g. auth headers for a third-party endpoint). - * - * @param otlpHeaders header name/value pairs; null is treated as empty - * @return this builder - */ - public Builder otlpHeaders(Map otlpHeaders) { - this.otlpHeaders = otlpHeaders; - return this; - } - /** Builds an immutable {@link OtelPluginConfig}. */ public OtelPluginConfig build() { return new OtelPluginConfig(this); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 24626157c..765975499 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -3,16 +3,10 @@ package software.amazon.lambda.durable.otel; import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; -import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; -import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; -import io.opentelemetry.sdk.trace.samplers.Sampler; -import io.opentelemetry.semconv.ServiceAttributes; import java.nio.file.Files; import java.nio.file.Path; import org.slf4j.Logger; @@ -47,40 +41,6 @@ static DeterministicIdGenerator createDefaultIdGenerator() { return new DeterministicIdGenerator(); } - /** - * Builds a plugin-owned {@link SdkTracerProvider} that exports over OTLP/HTTP (the {@link ProviderSource#AUTO_OTLP} - * default). Mirrors the auto-configured provider in the JavaScript and Python SDK plugins: an OTLP/HTTP exporter, a - * batch span processor, an env-driven sampler, Lambda resource attributes, and the deterministic ID generator. - * - * @param config the plugin configuration (endpoint + headers) - * @param idGenerator the deterministic ID generator to install - * @param additionalResource extra resource attributes to merge (e.g. ExecutionOtelPlugin's service.name), or null - */ - static SdkTracerProvider buildAutoOtlpProvider( - OtelPluginConfig config, DeterministicIdGenerator idGenerator, Resource additionalResource) { - var exporterBuilder = OtlpHttpSpanExporter.builder(); - var endpoint = resolveOtlpEndpoint(config); - if (endpoint != null) { - exporterBuilder.setEndpoint(endpoint); - } - for (var header : config.otlpHeaders().entrySet()) { - exporterBuilder.addHeader(header.getKey(), header.getValue()); - } - - var resource = buildLambdaResource(); - if (additionalResource != null) { - resource = resource.merge(additionalResource); - } - - return SdkTracerProvider.builder() - .setIdGenerator(idGenerator) - .setSampler(resolveSampler()) - .setResource(resource) - .addSpanProcessor( - BatchSpanProcessor.builder(exporterBuilder.build()).build()) - .build(); - } - /** * The tracer provider, tracer, and ID generator resolved for a config-only plugin constructor, plus the * {@link ProviderSource} that produced them. @@ -99,8 +59,6 @@ record ProviderSetup( *

      *
    • {@link ProviderSource#GLOBAL} — binds to the ADOT/global provider (not plugin-owned); the deterministic ID * generator is created for the application-side state bridge. - *
    • {@link ProviderSource#AUTO_OTLP} — builds a plugin-owned OTLP/HTTP provider (see - * {@link #buildAutoOtlpProvider}). *
    • {@link ProviderSource#EXPLICIT} — rejected: an explicit provider requires the * {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructor. *
    @@ -121,15 +79,6 @@ yield new ProviderSetup( tracerProvider.get(config.instrumentationName()), idGenerator); } - case AUTO_OTLP -> { - var idGenerator = new DeterministicIdGenerator(); - var sdkTracerProvider = buildAutoOtlpProvider(config, idGenerator, null); - yield new ProviderSetup( - ProviderSource.AUTO_OTLP, - sdkTracerProvider, - sdkTracerProvider.get(config.instrumentationName()), - idGenerator); - } case EXPLICIT -> throw new IllegalArgumentException( "OtelPluginConfig.providerSource(EXPLICIT) requires a caller-supplied SdkTracerProviderBuilder; " @@ -137,58 +86,6 @@ yield new ProviderSetup( }; } - /** Resolves the OTLP/HTTP traces endpoint (config -> env -> exporter default), appending the signal path. */ - private static String resolveOtlpEndpoint(OtelPluginConfig config) { - if (config.otlpEndpoint() != null && !config.otlpEndpoint().isBlank()) { - return config.otlpEndpoint(); - } - var envEndpoint = System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); - if (envEndpoint != null && !envEndpoint.isBlank()) { - var base = envEndpoint.endsWith("/") ? envEndpoint.substring(0, envEndpoint.length() - 1) : envEndpoint; - return base.endsWith("/v1/traces") ? base : base + "/v1/traces"; - } - // null -> the OTLP/HTTP exporter's own default (http://localhost:4318/v1/traces) - return null; - } - - /** Builds the sampler from {@code OTEL_DURABLE_SAMPLING_RATIO}, falling back to always-on. */ - private static Sampler resolveSampler() { - var raw = System.getenv("OTEL_DURABLE_SAMPLING_RATIO"); - if (raw != null) { - try { - var ratio = Double.parseDouble(raw); - if (ratio >= 0.0 && ratio <= 1.0) { - return Sampler.traceIdRatioBased(ratio); - } - } catch (NumberFormatException ignored) { - // fall through to always-on - } - } - return Sampler.alwaysOn(); - } - - /** Builds Lambda resource attributes from AWS_* env vars, merged onto the default resource. */ - private static Resource buildLambdaResource() { - var functionName = System.getenv("AWS_LAMBDA_FUNCTION_NAME"); - if (functionName == null || functionName.isBlank()) { - return Resource.getDefault(); - } - var attributes = Attributes.builder() - .put(ServiceAttributes.SERVICE_NAME, functionName) - .put("faas.name", functionName) - .put("cloud.provider", "aws") - .put("cloud.platform", "aws_lambda"); - var region = System.getenv("AWS_REGION"); - if (region != null && !region.isBlank()) { - attributes.put("cloud.region", region); - } - var version = System.getenv("AWS_LAMBDA_FUNCTION_VERSION"); - if (version != null && !version.isBlank()) { - attributes.put("faas.version", version); - } - return Resource.getDefault().merge(Resource.create(attributes.build())); - } - /** Extracts trace context from the current OTel span (fallback when X-Ray header is unavailable). */ static ExtractedContext extractCurrentSpanContext() { var spanContext = Span.current().getSpanContext(); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java index 5081c45c5..14c5e2d2e 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java @@ -3,18 +3,15 @@ package software.amazon.lambda.durable.otel; /** - * Which of the three resolution tiers produced a plugin's tracer provider. + * Which resolution path produced a plugin's tracer provider. * *

    Mirrors the {@code ProviderSource} used by the JavaScript and Python SDK OTel plugins for cross-SDK parity: * *

      *
    • {@link #EXPLICIT} — the caller supplied a {@link io.opentelemetry.sdk.trace.SdkTracerProviderBuilder} (the * {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructors); the plugin builds and owns that provider. - *
    • {@link #GLOBAL} — {@code GlobalOpenTelemetry} / the ADOT Java agent (the no-arg constructor, or an - * {@link OtelPluginConfig} with its {@code providerSource} left at the default {@code GLOBAL}); the plugin does - * not own the provider. - *
    • {@link #AUTO_OTLP} — opt-in via {@code providerSource(AUTO_OTLP)} on a config-only constructor: the plugin - * builds and owns an OTLP/HTTP provider. + *
    • {@link #GLOBAL} — {@code GlobalOpenTelemetry} / the ADOT Java agent (the no-arg or config-only constructors); + * the plugin does not own the provider. *
    * *

    This is the single knob that selects a plugin's tracer provider. {@link OtelPluginConfig#providerSource()} carries @@ -25,7 +22,5 @@ public enum ProviderSource { /** Caller-supplied {@code SdkTracerProviderBuilder}; plugin-owned. */ EXPLICIT, /** Globally configured provider (ADOT Java agent); not plugin-owned. */ - GLOBAL, - /** Auto-configured OTLP/HTTP provider; plugin-owned. */ - AUTO_OTLP + GLOBAL } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java index 8a7fb367c..109f633de 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java @@ -9,9 +9,9 @@ /** * Extracts OTel trace context from the AWS X-Ray {@code _X_AMZN_TRACE_ID} environment variable. * - *

    The durable execution backend propagates the same Root trace ID to every invocation of the same execution, so all - * invocations share one trace. This extractor parses that header and returns the trace ID in OTel format (32 hex chars) - * along with the parent span ID (16 hex chars). + *

    This extractor parses the Lambda/X-Ray header and returns the trace ID in OTel format (32 hex chars) along with + * the parent span ID (16 hex chars). Plugins use it as a fallback parent for Invocation spans; the deterministic + * Workflow trace is derived separately from the durable execution ARN. * *

    X-Ray header format: {@code Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1} * diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java index 2b9389fc7..caf09cc60 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java @@ -4,12 +4,20 @@ import static org.junit.jupiter.api.Assertions.*; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.sdk.trace.IdGenerator; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import java.time.Instant; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executors; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class DeterministicIdGeneratorTest { + private static final Instant EXECUTION_START_TIME = Instant.parse("2026-08-15T00:00:00Z"); + private DeterministicIdGenerator generator; @BeforeEach @@ -34,6 +42,122 @@ void generateTraceId_withoutArn_returnsRandom() { assertNotEquals(id1, id2); } + @Test + void scopedIds_delegateOutsideScope() { + var providerGenerator = new DeterministicIdGenerator(); + try (var provider = + SdkTracerProvider.builder().setIdGenerator(providerGenerator).build()) { + var pluginTracer = provider.get("durable-plugin"); + var unrelatedTracer = provider.get("unrelated-library"); + + var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan(); + var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME); + var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1"); + var workflow = generator.startSpan( + pluginTracer.spanBuilder("Workflow").setNoParent(), workflowTraceId, workflowSpanId); + var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan(); + var after = unrelatedTracer.spanBuilder("after").setNoParent().startSpan(); + + assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId()); + assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId()); + assertAllFreshRoots( + workflow.getSpanContext(), + before.getSpanContext(), + during.getSpanContext(), + after.getSpanContext()); + + before.end(); + workflow.end(); + during.end(); + after.end(); + } + } + + @Test + void scopedIds_bridgeAcrossGeneratorInstances() { + var agentGenerator = new DeterministicIdGenerator(); + try (var provider = + SdkTracerProvider.builder().setIdGenerator(agentGenerator).build()) { + var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME); + var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1"); + var workflow = generator.startSpan( + provider.get("durable-plugin").spanBuilder("Workflow").setNoParent(), + workflowTraceId, + workflowSpanId); + + assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId()); + assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId()); + assertNotEquals(workflowTraceId, agentGenerator.generateTraceId()); + workflow.end(); + } + } + + @Test + void concurrentScopedIds_doNotOverwriteEachOther() throws Exception { + var agentGenerator = new DeterministicIdGenerator(); + var executor = Executors.newFixedThreadPool(2); + try (var provider = + SdkTracerProvider.builder().setIdGenerator(agentGenerator).build()) { + var tracer = provider.get("durable-plugin"); + var barrier = new CyclicBarrier(2); + var traceId1 = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME); + var traceId2 = generator.generateTraceIdForExecution("arn:exec2", EXECUTION_START_TIME); + var spanId1 = generator.generateWorkflowSpanId("arn:exec1"); + var spanId2 = generator.generateWorkflowSpanId("arn:exec2"); + + var first = executor.submit(() -> { + barrier.await(); + return scopedSpanContext(tracer, new DeterministicIdGenerator(), traceId1, spanId1); + }); + var second = executor.submit(() -> { + barrier.await(); + return scopedSpanContext(tracer, new DeterministicIdGenerator(), traceId2, spanId2); + }); + + assertEquals(traceId1, first.get().getTraceId()); + assertEquals(spanId1, first.get().getSpanId()); + assertEquals(traceId2, second.get().getTraceId()); + assertEquals(spanId2, second.get().getSpanId()); + } finally { + executor.shutdownNow(); + } + } + + @Test + void installOn_preservesConfiguredFallbackGenerator() { + var fallbackTraceId = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + var fallbackSpanId = "cccccccccccccccc"; + var builder = SdkTracerProvider.builder().setIdGenerator(fixedIds(fallbackTraceId, fallbackSpanId)); + var installedGenerator = DeterministicIdGenerator.installOn(builder); + + try (var provider = builder.build()) { + var unrelated = + provider.get("unrelated").spanBuilder("root").setNoParent().startSpan(); + var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME); + var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1"); + var workflow = generator.startSpan( + provider.get("durable").spanBuilder("Workflow").setNoParent(), workflowTraceId, workflowSpanId); + + assertEquals(fallbackTraceId, unrelated.getSpanContext().getTraceId()); + assertEquals(fallbackSpanId, unrelated.getSpanContext().getSpanId()); + assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId()); + assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId()); + assertSame(installedGenerator, DeterministicIdGenerator.installOn(builder)); + + unrelated.end(); + workflow.end(); + } + } + + @Test + void executionTraceId_usesStartTimestampAndArn() { + var traceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME); + + assertEquals("6a7fac00", traceId.substring(0, 8)); + assertEquals(traceId, generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME)); + assertNotEquals(traceId, generator.generateTraceIdForExecution("arn:exec2", EXECUTION_START_TIME)); + } + @Test void generateTraceId_withArn_returnsDeterministic() { generator.setDurableExecutionArn("arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1"); @@ -128,27 +252,17 @@ void generateSpanIdForOperation_isDeterministic() { } @Test - void generatedIds_areSharedAcrossGeneratorInstances() { + void persistentIds_areIsolatedAcrossGeneratorInstances() { var pluginGenerator = new DeterministicIdGenerator(); - var agentGenerator = new DeterministicIdGenerator(); + var fallbackTraceId = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + var fallbackSpanId = "cccccccccccccccc"; + var agentGenerator = new DeterministicIdGenerator(fixedIds(fallbackTraceId, fallbackSpanId)); pluginGenerator.setDurableExecutionArn("arn:exec1"); pluginGenerator.setNextSpanOperationId("op-1"); - assertEquals(pluginGenerator.generateTraceId(), agentGenerator.generateTraceId()); - assertEquals(pluginGenerator.generateSpanIdForOperation("op-1"), agentGenerator.generateSpanId()); - } - - @Test - void rawSpanId_isSharedAcrossGeneratorInstances() { - var pluginGenerator = new DeterministicIdGenerator(); - var agentGenerator = new DeterministicIdGenerator(); - - pluginGenerator.setDurableExecutionArn("arn:exec1"); - var workflowSpanId = pluginGenerator.generateWorkflowSpanId(); - pluginGenerator.setNextSpanId(workflowSpanId); - - assertEquals(workflowSpanId, agentGenerator.generateSpanId()); + assertEquals(fallbackTraceId, agentGenerator.generateTraceId()); + assertEquals(fallbackSpanId, agentGenerator.generateSpanId()); } @Test @@ -285,4 +399,40 @@ void setExtractedTraceId_validXrayFormat_32HexChars() { assertEquals(xrayTraceId, result); assertTrue(result.matches("[0-9a-f]{32}")); } + + private static SpanContext scopedSpanContext( + io.opentelemetry.api.trace.Tracer tracer, + DeterministicIdGenerator pluginGenerator, + String traceId, + String spanId) { + var span = pluginGenerator.startSpan(tracer.spanBuilder("Workflow").setNoParent(), traceId, spanId); + var spanContext = span.getSpanContext(); + span.end(); + return spanContext; + } + + private static void assertAllFreshRoots(SpanContext workflow, SpanContext... unrelated) { + for (var spanContext : unrelated) { + assertNotEquals(workflow.getTraceId(), spanContext.getTraceId()); + } + for (var left = 0; left < unrelated.length; left++) { + for (var right = left + 1; right < unrelated.length; right++) { + assertNotEquals(unrelated[left].getTraceId(), unrelated[right].getTraceId()); + } + } + } + + private static IdGenerator fixedIds(String traceId, String spanId) { + return new IdGenerator() { + @Override + public String generateSpanId() { + return spanId; + } + + @Override + public String generateTraceId() { + return traceId; + } + }; + } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index eb9401722..c6d7cb6b3 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -93,14 +93,6 @@ void configOnlyConstructor_defaultsToGlobalProvider() { assertEquals(ProviderSource.GLOBAL, plugin.providerSource()); } - @Test - void configWithAutoOtlp_buildsPluginOwnedProvider() { - var plugin = new ExecutionOtelPlugin(OtelPluginConfig.builder() - .providerSource(ProviderSource.AUTO_OTLP) - .build()); - assertEquals(ProviderSource.AUTO_OTLP, plugin.providerSource()); - } - @Test void builderConstructor_isExplicitSource() { var plugin = new ExecutionOtelPlugin(SdkTracerProvider.builder(), OtelPluginConfig.defaults()); @@ -108,14 +100,8 @@ void builderConstructor_isExplicitSource() { } @Test - void configProviderSource_defaultsToGlobalAndHonorsAutoOtlp() { + void configProviderSource_defaultsToGlobal() { assertEquals(ProviderSource.GLOBAL, OtelPluginConfig.defaults().providerSource()); - assertEquals( - ProviderSource.AUTO_OTLP, - OtelPluginConfig.builder() - .providerSource(ProviderSource.AUTO_OTLP) - .build() - .providerSource()); } @Test @@ -250,7 +236,8 @@ void workflowAndInvocationSpans_areIndependentRoots_withoutAmbientContext() { assertFalse(workflowSpan.getParentSpanContext().isValid(), "Workflow span must be a root"); assertFalse(invocationSpan.getParentSpanContext().isValid(), "Invocation span must be a root"); - assertEquals(workflowSpan.getTraceId(), invocationSpan.getTraceId()); + assertNotEquals( + workflowSpan.getTraceId(), invocationSpan.getTraceId(), "Independent roots must not share a trace ID"); assertEquals(SpanKind.INTERNAL, invocationSpan.getKind()); } @@ -756,8 +743,9 @@ void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() { // ─── Cross-invocation stitching ────────────────────────────────────── @Test - void allSpansShareTraceId_acrossInvocations() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + void workflowTraceIsStableAndInvocationRootsAreFresh_acrossInvocations() { + var executionStartTime = Instant.parse("2026-08-15T00:00:00Z"); + plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, executionStartTime)); plugin.onOperationStart( new OperationInfo("op-1", "step-1", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -774,16 +762,21 @@ void allSpansShareTraceId_acrossInvocations() { null, null)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); - var firstTraceId = spanExporter.getFinishedSpanItems().get(0).getTraceId(); + var firstSpans = spanExporter.getFinishedSpanItems(); + var workflowTraceId = spanByName(firstSpans, "step-1").getTraceId(); + var firstInvocationTraceId = spanByName(firstSpans, "Invocation").getTraceId(); spanExporter.reset(); - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now())); + plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, executionStartTime)); plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); var secondSpans = spanExporter.getFinishedSpanItems(); + var workflowSpan = spanByName(secondSpans, "Workflow"); + var secondInvocationSpan = spanByName(secondSpans, "Invocation"); - assertTrue( - secondSpans.stream().allMatch(s -> s.getTraceId().equals(firstTraceId)), - "All spans of one execution must share the same trace ID"); + assertEquals(workflowTraceId, workflowSpan.getTraceId()); + assertNotEquals(workflowTraceId, firstInvocationTraceId); + assertNotEquals(workflowTraceId, secondInvocationSpan.getTraceId()); + assertNotEquals(firstInvocationTraceId, secondInvocationSpan.getTraceId()); } @Test @@ -865,13 +858,14 @@ void sampling_disabled_producesNoSpans() { // ─── X-Ray trace ID ────────────────────────────────────────────────── @Test - void xrayExtraction_allSpansShareExtractedTraceId() { + void xrayExtraction_keepsWorkflowTraceIndependent() { var xrayTraceId = "aabbccddee112233445566778899aabb"; + var parentSpanId = "53995c3f42cd8ad8"; var exporter = InMemorySpanExporter.create(); var xrayPlugin = new ExecutionOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() - .contextExtractor(() -> new ExtractedContext(xrayTraceId, null)) + .contextExtractor(() -> new ExtractedContext(xrayTraceId, parentSpanId)) .enableMdc(false) .workflowSpanName("Workflow") .build()); @@ -895,9 +889,13 @@ void xrayExtraction_allSpansShareExtractedTraceId() { var spans = exporter.getFinishedSpanItems(); assertTrue(spans.size() >= 3, "Workflow + invocation + operation spans expected"); - assertTrue( - spans.stream().allMatch(s -> s.getTraceId().equals(xrayTraceId)), - "All spans must share the extracted X-Ray trace ID"); + var workflowSpan = spanByName(spans, "Workflow"); + var invocationSpan = spanByName(spans, "Invocation"); + var operationSpan = spanByName(spans, "step-a"); + assertEquals(xrayTraceId, invocationSpan.getTraceId()); + assertEquals(parentSpanId, invocationSpan.getParentSpanId()); + assertNotEquals(xrayTraceId, workflowSpan.getTraceId()); + assertEquals(workflowSpan.getTraceId(), operationSpan.getTraceId()); } @Test diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java index 2266b5a94..abf9a66bd 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java @@ -71,17 +71,29 @@ void simpleStep_producesInvocationAndOperationAndAttemptSpans() { var spans = spanExporter.getFinishedSpanItems(); - // Should have: invocation + operation (backfilled) + attempt = 3 - assertTrue(spans.size() >= 3, "Expected at least 3 spans, got " + spans.size()); + // Should have: Workflow + invocation + operation + attempt = 4 + assertTrue(spans.size() >= 4, "Expected at least 4 spans, got " + spans.size()); // Verify span names + assertSpanExists(spans, "Workflow"); assertSpanExists(spans, "Invocation"); assertSpanExists(spans, "greet"); assertSpanExists(spans, "greet attempt 1"); - // All spans share the same trace ID - var traceId = spans.get(0).getTraceId(); - assertTrue(spans.stream().allMatch(s -> s.getTraceId().equals(traceId))); + var workflowTraceId = spans.stream() + .filter(span -> span.getName().equals("Workflow")) + .findFirst() + .orElseThrow() + .getTraceId(); + var invocationTraceId = spans.stream() + .filter(span -> span.getName().equals("Invocation")) + .findFirst() + .orElseThrow() + .getTraceId(); + assertNotEquals(workflowTraceId, invocationTraceId); + assertTrue(spans.stream() + .filter(span -> !span.getName().equals("Workflow")) + .allMatch(span -> span.getTraceId().equals(invocationTraceId))); } @Test diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index 859efc5bd..79d9756db 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -24,6 +24,7 @@ import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.IdGenerator; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; @@ -178,7 +179,7 @@ public ContextPropagators getPropagators() { } @Test - void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator() { + void autoConfigurationCustomizerProvider_appliesOnlyScopedDeterministicIds() { OtelPluginAutoConfigurationState.resetInstalledForTest(); var exporter = InMemorySpanExporter.create(); var autoConfiguration = mock(AutoConfigurationCustomizer.class); @@ -192,26 +193,46 @@ void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator( verify(autoConfiguration).addTracerProviderCustomizer(customizer.capture()); var pluginGenerator = new DeterministicIdGenerator(); - pluginGenerator.setDurableExecutionArn("arn:spi"); - pluginGenerator.setNextSpanOperationId("op-spi"); @SuppressWarnings("unchecked") var tracerProviderCustomizer = (BiFunction) customizer.getValue(); - var tracerProvider = tracerProviderCustomizer - .apply(SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), null) - .build(); - - var span = tracerProvider.get("test").spanBuilder("step").startSpan(); + var fallbackTraceId = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + var fallbackSpanId = "cccccccccccccccc"; + var builder = SdkTracerProvider.builder() + .setIdGenerator(fixedIds(fallbackTraceId, fallbackSpanId)) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var tracerProvider = tracerProviderCustomizer.apply(builder, null).build(); + + var traceId = pluginGenerator.generateTraceIdForExecution("arn:spi", Instant.parse("2026-08-15T00:00:00Z")); + var spanId = pluginGenerator.generateSpanIdForOperation("arn:spi", "op-spi"); + var span = pluginGenerator.startSpan( + tracerProvider.get("test").spanBuilder("step").setNoParent(), traceId, spanId); + var unrelated = tracerProvider + .get("unrelated") + .spanBuilder("root") + .setNoParent() + .startSpan(); span.end(); + unrelated.end(); tracerProvider.forceFlush().join(5, TimeUnit.SECONDS); var spans = exporter.getFinishedSpanItems(); - assertEquals(1, spans.size()); - assertEquals( - pluginGenerator.generateSpanIdForOperation("op-spi"), - spans.get(0).getSpanId()); + var pluginSpan = spans.stream() + .filter(item -> item.getName().equals("step")) + .findFirst() + .orElseThrow(); + var unrelatedSpan = spans.stream() + .filter(item -> item.getName().equals("root")) + .findFirst() + .orElseThrow(); + assertEquals(2, spans.size()); + assertEquals(spanId, pluginSpan.getSpanId()); + assertEquals(traceId, pluginSpan.getTraceId()); + assertEquals(fallbackTraceId, unrelatedSpan.getTraceId()); + assertEquals(fallbackSpanId, unrelatedSpan.getSpanId()); + assertEquals(Integer.MAX_VALUE, new OtelPluginAutoConfigurationCustomizerProvider().order()); } @Test @@ -306,14 +327,6 @@ void configOnlyConstructor_defaultsToGlobalProvider() { assertEquals(ProviderSource.GLOBAL, plugin.providerSource()); } - @Test - void configWithAutoOtlp_buildsPluginOwnedProvider() { - var plugin = new InvocationOtelPlugin(OtelPluginConfig.builder() - .providerSource(ProviderSource.AUTO_OTLP) - .build()); - assertEquals(ProviderSource.AUTO_OTLP, plugin.providerSource()); - } - @Test void builderConstructor_isExplicitSource() { var plugin = new InvocationOtelPlugin(SdkTracerProvider.builder(), OtelPluginConfig.defaults()); @@ -321,14 +334,62 @@ void builderConstructor_isExplicitSource() { } @Test - void configProviderSource_defaultsToGlobalAndHonorsAutoOtlp() { + void explicitProvider_unrelatedRootSpansKeepFreshTraceIds() { + var provider = sdkTracerProvider(plugin); + var unrelatedTracer = provider.get("unrelated-library"); + var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan(); + + plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan(); + plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + var after = unrelatedTracer.spanBuilder("after").setNoParent().startSpan(); + + assertFreshTraceIds( + spanByName("Workflow").getSpanContext(), + before.getSpanContext(), + during.getSpanContext(), + after.getSpanContext()); + before.end(); + during.end(); + after.end(); + } + + @Test + void globalProvider_unrelatedRootSpansKeepFreshTraceIds() { + OtelPluginAutoConfigurationState.markInstalled(); + GlobalOpenTelemetry.resetForTest(); + var exporter = InMemorySpanExporter.create(); + var provider = SdkTracerProvider.builder() + .setIdGenerator(new DeterministicIdGenerator()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + var javaAgentTracerProvider = new FakeJavaAgentTracerProvider(provider); + GlobalOpenTelemetry.set(openTelemetry(javaAgentTracerProvider)); + var unrelatedTracer = provider.get("unrelated-library"); + var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan(); + + var globalPlugin = new InvocationOtelPlugin(); + globalPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan(); + globalPlugin.onInvocationEnd( + new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); + var after = unrelatedTracer.spanBuilder("after").setNoParent().startSpan(); + + var workflow = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("Workflow")) + .findFirst() + .orElseThrow(); + assertFreshTraceIds( + workflow.getSpanContext(), before.getSpanContext(), during.getSpanContext(), after.getSpanContext()); + before.end(); + during.end(); + after.end(); + provider.close(); + } + + @Test + void configProviderSource_defaultsToGlobal() { assertEquals(ProviderSource.GLOBAL, OtelPluginConfig.defaults().providerSource()); - assertEquals( - ProviderSource.AUTO_OTLP, - OtelPluginConfig.builder() - .providerSource(ProviderSource.AUTO_OTLP) - .build() - .providerSource()); } @Test @@ -780,28 +841,34 @@ void fullLifecycle_producesCorrectSpanHierarchy() { // 2 attempt spans + 2 operation spans + 1 invocation span + 1 Workflow span = 6 assertEquals(6, spans.size()); - // All spans should share the same trace ID - var traceId = spans.get(0).getTraceId(); - assertTrue(spans.stream().allMatch(s -> s.getTraceId().equals(traceId))); + var workflowTraceId = spanByName("Workflow").getTraceId(); + var invocationTraceId = spanByName("Invocation").getTraceId(); + assertNotEquals(workflowTraceId, invocationTraceId); + assertTrue(spans.stream() + .filter(span -> !span.getName().equals("Workflow")) + .allMatch(span -> span.getTraceId().equals(invocationTraceId))); } @Test - void deterministicIds_sameExecutionProducesSameTraceId() { + void invocationRoots_sameExecutionReceiveFreshTraceIds() { var arn = "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1"; plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null)); - var firstTraceId = spanExporter.getFinishedSpanItems().get(0).getTraceId(); + var firstTraceId = spanByName("Invocation").getTraceId(); spanExporter.reset(); // Second invocation of same execution plugin.onInvocationStart(new InvocationInfo("req-2", arn, false, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null)); - var secondTraceId = spanExporter.getFinishedSpanItems().get(0).getTraceId(); + var secondTraceId = spanByName("Invocation").getTraceId(); + var workflowTraceId = spanByName("Workflow").getTraceId(); - assertEquals(firstTraceId, secondTraceId, "Same execution ARN should produce same trace ID"); + assertNotEquals(firstTraceId, secondTraceId); + assertNotEquals(firstTraceId, workflowTraceId); + assertNotEquals(secondTraceId, workflowTraceId); } @Test @@ -913,7 +980,7 @@ void sampling_disabled_producesNoSpans() { // ─── X-Ray trace ID extraction integration tests ───────────────────── @Test - void xrayExtraction_usesExtractedTraceId_overArnDerived() { + void xrayExtraction_withoutParentDoesNotForceTraceId() { var xrayTraceId = "5759e988bd862e3fe1be46a994272793"; var extractedContext = new ExtractedContext(xrayTraceId, null); @@ -930,13 +997,19 @@ void xrayExtraction_usesExtractedTraceId_overArnDerived() { var spans = spanExporter.getFinishedSpanItems(); assertEquals(2, spans.size()); // invocation + Workflow - assertEquals(xrayTraceId, spans.get(0).getTraceId(), "Span should use the extracted X-Ray trace ID"); + var invocationSpan = spanByName("Invocation"); + var workflowSpan = spanByName("Workflow"); + assertFalse(invocationSpan.getParentSpanContext().isValid()); + assertNotEquals(xrayTraceId, invocationSpan.getTraceId()); + assertNotEquals(xrayTraceId, workflowSpan.getTraceId()); + assertNotEquals(invocationSpan.getTraceId(), workflowSpan.getTraceId()); } @Test - void xrayExtraction_allSpansShareExtractedTraceId() { + void xrayExtraction_invocationTreeUsesExtractedTraceId_workflowRemainsIndependent() { var xrayTraceId = "aabbccddee112233445566778899aabb"; - var extractedContext = new ExtractedContext(xrayTraceId, null); + var parentSpanId = "53995c3f42cd8ad8"; + var extractedContext = new ExtractedContext(xrayTraceId, parentSpanId); spanExporter = InMemorySpanExporter.create(); var xrayPlugin = new InvocationOtelPlugin( @@ -969,10 +1042,13 @@ void xrayExtraction_allSpansShareExtractedTraceId() { xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertTrue(spans.size() >= 2, "Should have invocation + operation + attempt spans"); + var workflowTraceId = spanByName("Workflow").getTraceId(); + assertNotEquals(xrayTraceId, workflowTraceId); assertTrue( - spans.stream().allMatch(s -> s.getTraceId().equals(xrayTraceId)), - "All spans must share the extracted X-Ray trace ID"); + spans.stream() + .filter(span -> !span.getName().equals("Workflow")) + .allMatch(span -> span.getTraceId().equals(xrayTraceId)), + "The invocation tree should inherit the extracted X-Ray trace ID"); } @Test @@ -995,12 +1071,14 @@ void xrayExtraction_withParentSpanId_invocationSpanHasCorrectParent() { var spans = spanExporter.getFinishedSpanItems(); assertEquals(2, spans.size()); // invocation + Workflow - var invocationSpan = spans.get(0); + var invocationSpan = spanByName("Invocation"); + var workflowSpan = spanByName("Workflow"); assertEquals(xrayTraceId, invocationSpan.getTraceId()); assertEquals( parentSpanId, invocationSpan.getParentSpanId(), "Invocation span should be parented to X-Ray Parent span"); + assertNotEquals(xrayTraceId, workflowSpan.getTraceId()); } @Test @@ -1022,17 +1100,9 @@ void xrayExtraction_withoutParentSpanId_invocationSpanIsRoot() { var spans = spanExporter.getFinishedSpanItems(); assertEquals(2, spans.size()); // invocation + Workflow - var invocationSpan = spans.get(0); - assertEquals(xrayTraceId, invocationSpan.getTraceId()); - // Parent span ID should be empty/invalid when no parent provided - assertFalse( - io.opentelemetry.api.trace.SpanContext.create( - xrayTraceId, - invocationSpan.getParentSpanId(), - io.opentelemetry.api.trace.TraceFlags.getSampled(), - io.opentelemetry.api.trace.TraceState.getDefault()) - .isRemote(), - "Without X-Ray parent, invocation span should not have a remote parent"); + var invocationSpan = spanByName("Invocation"); + assertFalse(invocationSpan.getParentSpanContext().isValid()); + assertNotEquals(xrayTraceId, invocationSpan.getTraceId()); } @Test @@ -1090,14 +1160,17 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() { var spans = spanExporter.getFinishedSpanItems(); assertTrue(spans.size() >= 4, "Should have spans from both invocations"); - // All spans share the same X-Ray trace ID — unified trace + var workflowTraceId = spanByName("Workflow").getTraceId(); + assertNotEquals(xrayTraceId, workflowTraceId); assertTrue( - spans.stream().allMatch(s -> s.getTraceId().equals(xrayTraceId)), - "Both invocations should produce spans with the same X-Ray trace ID"); + spans.stream() + .filter(span -> !span.getName().equals("Workflow")) + .allMatch(span -> span.getTraceId().equals(xrayTraceId)), + "Both invocation trees should inherit the X-Ray trace ID"); } @Test - void xrayExtraction_nullExtractor_fallsBackToArnDerived() { + void xrayExtraction_nullExtractor_usesIndependentValidRootIds() { spanExporter = InMemorySpanExporter.create(); var noXrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), @@ -1113,10 +1186,11 @@ void xrayExtraction_nullExtractor_fallsBackToArnDerived() { var spans = spanExporter.getFinishedSpanItems(); assertEquals(2, spans.size()); // invocation + Workflow - var traceId = spans.get(0).getTraceId(); - assertNotNull(traceId); - assertEquals(32, traceId.length()); - assertTrue(traceId.matches("[0-9a-f]{32}"), "ARN-derived trace ID should be valid hex"); + var invocationTraceId = spanByName("Invocation").getTraceId(); + var workflowTraceId = spanByName("Workflow").getTraceId(); + assertTrue(invocationTraceId.matches("[0-9a-f]{32}")); + assertTrue(workflowTraceId.matches("[0-9a-f]{32}")); + assertNotEquals(invocationTraceId, workflowTraceId); } @Test @@ -1130,7 +1204,7 @@ void xrayExtraction_extractedTraceIdMatchesXrayConversion() { assertEquals(expectedOtelTraceId, convertedId); // Now feed it through the plugin - var extractedContext = new ExtractedContext(convertedId, null); + var extractedContext = new ExtractedContext(convertedId, "53995c3f42cd8ad8"); spanExporter = InMemorySpanExporter.create(); var xrayPlugin = new InvocationOtelPlugin( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), @@ -1142,8 +1216,8 @@ void xrayExtraction_extractedTraceIdMatchesXrayConversion() { xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); - var spans = spanExporter.getFinishedSpanItems(); - assertEquals(expectedOtelTraceId, spans.get(0).getTraceId()); + assertEquals(expectedOtelTraceId, spanByName("Invocation").getTraceId()); + assertNotEquals(expectedOtelTraceId, spanByName("Workflow").getTraceId()); } // ─── Cross-invocation continuation span tests ──────────────────────── @@ -1395,8 +1469,18 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null)); // Invocation 1 should have: step op + step attempt + wait (PENDING) + invocation = 4 - assertEquals(4, spanExporter.getFinishedSpanItems().size()); - var inv1TraceId = spanExporter.getFinishedSpanItems().get(0).getTraceId(); + var inv1Spans = spanExporter.getFinishedSpanItems(); + assertEquals(4, inv1Spans.size()); + var inv1TraceId = inv1Spans.stream() + .filter(span -> span.getName().equals("Invocation")) + .findFirst() + .orElseThrow() + .getTraceId(); + var originalWaitSpanId = inv1Spans.stream() + .filter(span -> span.getName().equals("pause")) + .findFirst() + .orElseThrow() + .getSpanId(); spanExporter.reset(); @@ -1440,16 +1524,29 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { // wait continuation + step-B op + step-B attempt + invocation + Workflow = 5 assertEquals(5, inv2Spans.size()); - // Same trace ID across invocations - var inv2TraceId = inv2Spans.get(0).getTraceId(); - assertEquals(inv1TraceId, inv2TraceId); + var inv2TraceId = inv2Spans.stream() + .filter(span -> span.getName().equals("Invocation")) + .findFirst() + .orElseThrow() + .getTraceId(); + var workflowSpan = inv2Spans.stream() + .filter(span -> span.getName().equals("Workflow")) + .findFirst() + .orElseThrow(); + assertNotEquals(inv1TraceId, inv2TraceId); + assertNotEquals(inv1TraceId, workflowSpan.getTraceId()); + assertNotEquals(inv2TraceId, workflowSpan.getTraceId()); - // Wait continuation should have a Link var waitContinuation = inv2Spans.stream() .filter(s -> s.getName().contains("pause")) .findFirst() .orElseThrow(); - assertFalse(waitContinuation.getLinks().isEmpty()); + assertTrue(waitContinuation.getLinks().stream() + .anyMatch(link -> link.getSpanContext().getSpanId().equals(workflowSpan.getSpanId()))); + assertTrue( + waitContinuation.getLinks().stream() + .noneMatch(link -> link.getSpanContext().getSpanId().equals(originalWaitSpanId)), + "Continuation spans must not fabricate a link to an uncheckpointed prior span context"); } // ─── Cross-invocation step retry scenario ──────────────────────────── @@ -1550,18 +1647,32 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { inv2OperationSpan.getSpanId(), "Continuation operation span must have a different span ID from the original"); - // The continuation operation span should have a Link to the original for correlation - assertFalse( - inv2OperationSpan.getLinks().isEmpty(), - "Continuation operation span should have a Link to the original"); + var inv1InvocationTraceId = inv1Spans.stream() + .filter(span -> span.getName().equals("Invocation")) + .findFirst() + .orElseThrow() + .getTraceId(); + var inv2InvocationTraceId = inv2Spans.stream() + .filter(span -> span.getName().equals("Invocation")) + .findFirst() + .orElseThrow() + .getTraceId(); + var workflowSpan = inv2Spans.stream() + .filter(span -> span.getName().equals("Workflow")) + .findFirst() + .orElseThrow(); - // All spans share the same trace ID - var allSpans = new java.util.ArrayList<>(inv1Spans); - allSpans.addAll(inv2Spans); - var traceId = allSpans.get(0).getTraceId(); + assertNotEquals(inv1InvocationTraceId, inv2InvocationTraceId); + assertTrue(inv1Spans.stream().allMatch(span -> span.getTraceId().equals(inv1InvocationTraceId))); + assertTrue(inv2Spans.stream() + .filter(span -> !span.getName().equals("Workflow")) + .allMatch(span -> span.getTraceId().equals(inv2InvocationTraceId))); + assertTrue(inv2OperationSpan.getLinks().stream() + .anyMatch(link -> link.getSpanContext().getSpanId().equals(workflowSpan.getSpanId()))); assertTrue( - allSpans.stream().allMatch(s -> s.getTraceId().equals(traceId)), - "All spans across invocations should share the same trace ID"); + inv2OperationSpan.getLinks().stream() + .noneMatch(link -> link.getSpanContext().getSpanId().equals(inv1OperationSpan.getSpanId())), + "Replay spans must not fabricate a link to an uncheckpointed prior span context"); } // ─── Workflow span + links ─────────────────────────────────────────── @@ -1726,4 +1837,53 @@ private static boolean hasLinkTo(io.opentelemetry.sdk.trace.data.SpanData span, return span.getLinks().stream() .anyMatch(l -> l.getSpanContext().getSpanId().equals(spanId)); } + + private static SdkTracerProvider sdkTracerProvider(InvocationOtelPlugin plugin) { + try { + var field = InvocationOtelPlugin.class.getDeclaredField("sdkTracerProvider"); + field.setAccessible(true); + return (SdkTracerProvider) field.get(plugin); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } + + private static OpenTelemetry openTelemetry(io.opentelemetry.api.trace.TracerProvider tracerProvider) { + return new OpenTelemetry() { + @Override + public io.opentelemetry.api.trace.TracerProvider getTracerProvider() { + return tracerProvider; + } + + @Override + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }; + } + + private static void assertFreshTraceIds(SpanContext workflow, SpanContext... unrelated) { + for (var spanContext : unrelated) { + assertNotEquals(workflow.getTraceId(), spanContext.getTraceId()); + } + for (var left = 0; left < unrelated.length; left++) { + for (var right = left + 1; right < unrelated.length; right++) { + assertNotEquals(unrelated[left].getTraceId(), unrelated[right].getTraceId()); + } + } + } + + private static IdGenerator fixedIds(String traceId, String spanId) { + return new IdGenerator() { + @Override + public String generateSpanId() { + return spanId; + } + + @Override + public String generateTraceId() { + return traceId; + } + }; + } } From 681751f0d4a34ab071aae02a349cb9404922abf0 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Sat, 15 Aug 2026 18:35:16 +0000 Subject: [PATCH 2/3] refactor(otel): remove provider source --- .../durable/otel/ExecutionOtelPlugin.java | 15 ++---- .../durable/otel/InvocationOtelPlugin.java | 15 ++---- .../lambda/durable/otel/OtelPluginConfig.java | 34 +------------ .../durable/otel/OtelPluginSupport.java | 49 +++++-------------- .../lambda/durable/otel/ProviderSource.java | 26 ---------- .../durable/otel/ExecutionOtelPluginTest.java | 32 ------------ .../otel/InvocationOtelPluginTest.java | 32 ------------ 7 files changed, 20 insertions(+), 183 deletions(-) delete mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 57f65d1f9..a3b845311 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -84,7 +84,6 @@ public class ExecutionOtelPlugin implements DurableExecutionPlugin { private final ContextExtractor contextExtractor; private final boolean enableMdc; private final String workflowSpanName; - private final ProviderSource providerSource; // Per-invocation state private volatile Span workflowSpan; @@ -149,35 +148,27 @@ public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelP this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - this.providerSource = ProviderSource.EXPLICIT; } /** * Creates a Workflow-rooted OTel plugin from configuration alone (no caller-supplied tracer provider builder). * - *

    The config-only constructor uses the ADOT/global provider. {@link ProviderSource#EXPLICIT} is rejected here; - * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for an application-owned provider. + *

    The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the + * two-arg constructor for an application-owned provider. * * @param config the plugin configuration - * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT} */ public ExecutionOtelPlugin(OtelPluginConfig config) { this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - var setup = OtelPluginSupport.resolveConfiguredProvider(config, "ExecutionOtelPlugin"); - this.providerSource = setup.source(); + var setup = OtelPluginSupport.resolveGlobalProvider(config, "ExecutionOtelPlugin"); this.idGenerator = setup.idGenerator(); this.sdkTracerProvider = setup.sdkTracerProvider(); this.tracer = setup.tracer(); } - /** The tier that produced this plugin's tracer provider. */ - public ProviderSource providerSource() { - return providerSource; - } - // ─── Invocation hooks ──────────────────────────────────────────────── @Override diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index cd69225ef..17486d5e6 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -92,7 +92,6 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin { private final ContextExtractor contextExtractor; private final boolean enableMdc; private final String workflowSpanName; - private final ProviderSource providerSource; // Per-invocation state private volatile Span workflowSpan; @@ -167,35 +166,27 @@ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, Otel this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - this.providerSource = ProviderSource.EXPLICIT; } /** * Creates an OTel plugin from configuration alone (no caller-supplied tracer provider builder). * - *

    The config-only constructor uses the ADOT/global provider. {@link ProviderSource#EXPLICIT} is rejected here; - * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for an application-owned provider. + *

    The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the + * two-arg constructor for an application-owned provider. * * @param config the plugin configuration - * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT} */ public InvocationOtelPlugin(OtelPluginConfig config) { this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - var setup = OtelPluginSupport.resolveConfiguredProvider(config, "InvocationOtelPlugin"); - this.providerSource = setup.source(); + var setup = OtelPluginSupport.resolveGlobalProvider(config, "InvocationOtelPlugin"); this.idGenerator = setup.idGenerator(); this.sdkTracerProvider = setup.sdkTracerProvider(); this.tracer = setup.tracer(); } - /** The tier that produced this plugin's tracer provider. */ - public ProviderSource providerSource() { - return providerSource; - } - // ─── Invocation hooks ──────────────────────────────────────────────── @Override diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java index b6dd40b82..ed4ac9be5 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java @@ -24,8 +24,8 @@ * } * *

    Defaults: {@code contextExtractor = new XRayContextExtractor()}, {@code enableMdc = true}, {@code workflowSpanName - * = "Workflow"}, {@code instrumentationName = "aws-durable-execution-sdk-java"}, {@code providerSource = - * ProviderSource.GLOBAL}. A {@code null} passed to any builder setter falls back to the corresponding default. + * = "Workflow"}, {@code instrumentationName = "aws-durable-execution-sdk-java"}. A {@code null} passed to any builder + * setter falls back to the corresponding default. */ public final class OtelPluginConfig { @@ -36,7 +36,6 @@ public final class OtelPluginConfig { private final boolean enableMdc; private final String workflowSpanName; private final String instrumentationName; - private final ProviderSource providerSource; private OtelPluginConfig(Builder builder) { this.contextExtractor = @@ -46,7 +45,6 @@ private OtelPluginConfig(Builder builder) { builder.workflowSpanName != null ? builder.workflowSpanName : DEFAULT_WORKFLOW_SPAN_NAME; this.instrumentationName = builder.instrumentationName != null ? builder.instrumentationName : DEFAULT_INSTRUMENTATION_NAME; - this.providerSource = builder.providerSource != null ? builder.providerSource : ProviderSource.GLOBAL; } /** Returns a new builder with all fields defaulted. */ @@ -79,17 +77,6 @@ public String instrumentationName() { return instrumentationName; } - /** - * The tracer-provider source to use when no {@code SdkTracerProviderBuilder} is supplied (the config-only - * constructors). {@link ProviderSource#GLOBAL} (the default) uses the globally configured ADOT provider. - * - *

    {@link ProviderSource#EXPLICIT} is not valid here — it is implied by using a {@code (SdkTracerProviderBuilder, - * OtelPluginConfig)} constructor and is rejected by the config-only constructors. - */ - public ProviderSource providerSource() { - return providerSource; - } - /** Builder for {@link OtelPluginConfig}. */ public static final class Builder { @@ -97,7 +84,6 @@ public static final class Builder { private boolean enableMdc = true; private String workflowSpanName; private String instrumentationName; - private ProviderSource providerSource = ProviderSource.GLOBAL; private Builder() {} @@ -146,22 +132,6 @@ public Builder instrumentationName(String instrumentationName) { return this; } - /** - * Sets the tracer-provider source used when no {@code SdkTracerProviderBuilder} is supplied. The config-only - * constructors accept {@link ProviderSource#GLOBAL}; a {@code null} also falls back to - * {@link ProviderSource#GLOBAL}. - * - *

    {@link ProviderSource#EXPLICIT} is not accepted through the config-only constructors — supply a - * {@code SdkTracerProviderBuilder} via the two-arg constructor instead. - * - * @param providerSource the provider source - * @return this builder - */ - public Builder providerSource(ProviderSource providerSource) { - this.providerSource = providerSource != null ? providerSource : ProviderSource.GLOBAL; - return this; - } - /** Builds an immutable {@link OtelPluginConfig}. */ public OtelPluginConfig build() { return new OtelPluginConfig(this); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 765975499..f495a7137 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -41,49 +41,24 @@ static DeterministicIdGenerator createDefaultIdGenerator() { return new DeterministicIdGenerator(); } - /** - * The tracer provider, tracer, and ID generator resolved for a config-only plugin constructor, plus the - * {@link ProviderSource} that produced them. - */ - record ProviderSetup( - ProviderSource source, - SdkTracerProvider sdkTracerProvider, - Tracer tracer, - DeterministicIdGenerator idGenerator) {} + /** The tracer provider, tracer, and ID generator resolved for a config-only plugin constructor. */ + record ProviderSetup(SdkTracerProvider sdkTracerProvider, Tracer tracer, DeterministicIdGenerator idGenerator) {} /** - * Resolves the tracer provider for the config-only plugin constructors from - * {@link OtelPluginConfig#providerSource()}, centralizing the {@link ProviderSource} branching shared by - * {@link InvocationOtelPlugin} and {@link ExecutionOtelPlugin}: - * - *

      - *
    • {@link ProviderSource#GLOBAL} — binds to the ADOT/global provider (not plugin-owned); the deterministic ID - * generator is created for the application-side state bridge. - *
    • {@link ProviderSource#EXPLICIT} — rejected: an explicit provider requires the - * {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructor. - *
    + * Resolves the ADOT/global tracer provider for config-only plugin constructors. Explicit providers are supplied + * through the {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructors instead. * * @param config the plugin configuration * @param pluginName the plugin name used in diagnostics/flush logging - * @return the resolved provider, tracer, ID generator, and source - * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT} + * @return the resolved provider, tracer, and ID generator */ - static ProviderSetup resolveConfiguredProvider(OtelPluginConfig config, String pluginName) { - return switch (config.providerSource()) { - case GLOBAL -> { - var idGenerator = createDefaultIdGenerator(); - var tracerProvider = getDefaultTracerProvider(pluginName); - yield new ProviderSetup( - ProviderSource.GLOBAL, - getSdkTracerProviderForFlush(tracerProvider, pluginName), - tracerProvider.get(config.instrumentationName()), - idGenerator); - } - case EXPLICIT -> - throw new IllegalArgumentException( - "OtelPluginConfig.providerSource(EXPLICIT) requires a caller-supplied SdkTracerProviderBuilder; " - + "use the (SdkTracerProviderBuilder, OtelPluginConfig) constructor."); - }; + static ProviderSetup resolveGlobalProvider(OtelPluginConfig config, String pluginName) { + var idGenerator = createDefaultIdGenerator(); + var tracerProvider = getDefaultTracerProvider(pluginName); + return new ProviderSetup( + getSdkTracerProviderForFlush(tracerProvider, pluginName), + tracerProvider.get(config.instrumentationName()), + idGenerator); } /** Extracts trace context from the current OTel span (fallback when X-Ray header is unavailable). */ diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java deleted file mode 100644 index 14c5e2d2e..000000000 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.otel; - -/** - * Which resolution path produced a plugin's tracer provider. - * - *

    Mirrors the {@code ProviderSource} used by the JavaScript and Python SDK OTel plugins for cross-SDK parity: - * - *

      - *
    • {@link #EXPLICIT} — the caller supplied a {@link io.opentelemetry.sdk.trace.SdkTracerProviderBuilder} (the - * {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructors); the plugin builds and owns that provider. - *
    • {@link #GLOBAL} — {@code GlobalOpenTelemetry} / the ADOT Java agent (the no-arg or config-only constructors); - * the plugin does not own the provider. - *
    - * - *

    This is the single knob that selects a plugin's tracer provider. {@link OtelPluginConfig#providerSource()} carries - * it for the config-only constructors; the {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructors always - * report {@link #EXPLICIT}. - */ -public enum ProviderSource { - /** Caller-supplied {@code SdkTracerProviderBuilder}; plugin-owned. */ - EXPLICIT, - /** Globally configured provider (ADOT Java agent); not plugin-owned. */ - GLOBAL -} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index c6d7cb6b3..fb417f7ed 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -81,38 +81,6 @@ void customInstrumentationName_isUsedForTracerScope() { } } - @Test - void configOnlyConstructor_defaultsToGlobalProvider() { - OtelPluginAutoConfigurationState.markInstalled(); - GlobalOpenTelemetry.resetForTest(); - OpenTelemetrySdk.builder() - .setTracerProvider(SdkTracerProvider.builder().build()) - .buildAndRegisterGlobal(); - - var plugin = new ExecutionOtelPlugin(OtelPluginConfig.defaults()); - assertEquals(ProviderSource.GLOBAL, plugin.providerSource()); - } - - @Test - void builderConstructor_isExplicitSource() { - var plugin = new ExecutionOtelPlugin(SdkTracerProvider.builder(), OtelPluginConfig.defaults()); - assertEquals(ProviderSource.EXPLICIT, plugin.providerSource()); - } - - @Test - void configProviderSource_defaultsToGlobal() { - assertEquals(ProviderSource.GLOBAL, OtelPluginConfig.defaults().providerSource()); - } - - @Test - void configOnlyConstructor_rejectsExplicitProviderSource() { - var config = OtelPluginConfig.builder() - .providerSource(ProviderSource.EXPLICIT) - .build(); - var error = assertThrows(IllegalArgumentException.class, () -> new ExecutionOtelPlugin(config)); - assertTrue(error.getMessage().contains("SdkTracerProviderBuilder")); - } - @Test void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstalled() { GlobalOpenTelemetry.resetForTest(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index 79d9756db..283c7e2b3 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -315,24 +315,6 @@ void customInstrumentationName_isUsedForTracerScope() { } } - @Test - void configOnlyConstructor_defaultsToGlobalProvider() { - OtelPluginAutoConfigurationState.markInstalled(); - GlobalOpenTelemetry.resetForTest(); - OpenTelemetrySdk.builder() - .setTracerProvider(SdkTracerProvider.builder().build()) - .buildAndRegisterGlobal(); - - var plugin = new InvocationOtelPlugin(OtelPluginConfig.defaults()); - assertEquals(ProviderSource.GLOBAL, plugin.providerSource()); - } - - @Test - void builderConstructor_isExplicitSource() { - var plugin = new InvocationOtelPlugin(SdkTracerProvider.builder(), OtelPluginConfig.defaults()); - assertEquals(ProviderSource.EXPLICIT, plugin.providerSource()); - } - @Test void explicitProvider_unrelatedRootSpansKeepFreshTraceIds() { var provider = sdkTracerProvider(plugin); @@ -387,20 +369,6 @@ void globalProvider_unrelatedRootSpansKeepFreshTraceIds() { provider.close(); } - @Test - void configProviderSource_defaultsToGlobal() { - assertEquals(ProviderSource.GLOBAL, OtelPluginConfig.defaults().providerSource()); - } - - @Test - void configOnlyConstructor_rejectsExplicitProviderSource() { - var config = OtelPluginConfig.builder() - .providerSource(ProviderSource.EXPLICIT) - .build(); - var error = assertThrows(IllegalArgumentException.class, () -> new InvocationOtelPlugin(config)); - assertTrue(error.getMessage().contains("SdkTracerProviderBuilder")); - } - @Test void invocationSpan_hasInternalKind() { plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); From 6efa92c042b8abdc8476b47c3afe481ab1490598 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Sat, 15 Aug 2026 21:36:19 +0000 Subject: [PATCH 3/3] fix(otel): late-bind global tracer providers --- otel-plugin/README.md | 9 ++- .../durable/otel/ExecutionOtelPlugin.java | 56 ++++++++++--- .../durable/otel/InvocationOtelPlugin.java | 59 +++++++++++--- .../durable/otel/OtelPluginSupport.java | 80 +++++++++---------- .../durable/otel/ExecutionOtelPluginTest.java | 62 ++++++++++++-- .../InvocationOtelPluginIntegrationTest.java | 9 ++- .../otel/InvocationOtelPluginTest.java | 66 +++++++++++---- 7 files changed, 252 insertions(+), 89 deletions(-) diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c14577d64..dcf5d1b4d 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -10,7 +10,7 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo - **Span-per-Operation**: Each durable operation (step, wait, map, etc.) gets its own span with accurate timing - **Attempt Spans**: Each user function execution (step attempt, child context run) gets a span, including retries - **Log Correlation**: Injects `trace_id`, `span_id`, and `traceSampled` into SLF4J MDC for end-to-end observability -- **ADOT Java Agent Integration**: `new InvocationOtelPlugin()` uses the ADOT Java agent's global provider with no handler-side OpenTelemetry initialization +- **ADOT Java Agent Integration**: `new InvocationOtelPlugin()` late-binds the ADOT Java agent's global provider with no handler-side OpenTelemetry initialization - **Lambda Layer Discovery**: `DURABLE_EXECUTION_PLUGINS` loads either OTel plugin from a JAR under a layer's `java/lib` directory ## Installation @@ -50,7 +50,7 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The `new InvocationOtelPlugin()` constructor uses the global provider initialized by the ADOT Java agent with deterministic span ID generation installed through the plugin's `AutoConfigurationCustomizerProvider` SPI. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The `new InvocationOtelPlugin()` constructor resolves the global provider initialized by the ADOT Java agent at invocation start, with deterministic span ID generation installed through the plugin's `AutoConfigurationCustomizerProvider` SPI. If the provider is not ready, the plugin emits no telemetry for that invocation and retries provider resolution on the next invocation. The layer ARN follows the format: @@ -293,8 +293,9 @@ new ExecutionOtelPlugin( | `instrumentationName(...)` | Instrumentation scope name registered with the tracer | `"aws-durable-execution-sdk-java"` | > The `tracerProviderBuilder` argument is not used by the no-arg `new InvocationOtelPlugin()` / -> `new ExecutionOtelPlugin()` constructors; those use the ADOT Java agent's global provider. A `null` passed to any -> `OtelPluginConfig` builder setter falls back to that option's default. +> `new ExecutionOtelPlugin()` constructors; those resolve the ADOT Java agent's global provider at invocation start. +> If it is not ready, all telemetry is disabled for that invocation and resolution is retried on the next invocation. +> A `null` passed to any `OtelPluginConfig` builder setter falls back to that option's default. ## Known Limitations diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index a3b845311..1ee243b8a 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -58,6 +58,9 @@ * *

    The Workflow trace ID is derived from the execution start time and ARN, and is independent of the ambient * Lambda/X-Ray trace. Invocation spans inherit the active ambient context, or extracted upstream context as a fallback. + * When using {@link #ExecutionOtelPlugin()}, the plugin resolves the global provider at invocation start. If the + * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider + * resolution is retried on the next invocation. * *

    Status mapping (parity with the Python/JS references): * @@ -78,14 +81,16 @@ public class ExecutionOtelPlugin implements DurableExecutionPlugin { private static final Logger logger = LoggerFactory.getLogger(ExecutionOtelPlugin.class); - private final SdkTracerProvider sdkTracerProvider; - private final Tracer tracer; + private volatile SdkTracerProvider sdkTracerProvider; + private volatile Tracer tracer; private final DeterministicIdGenerator idGenerator; private final ContextExtractor contextExtractor; private final boolean enableMdc; private final String workflowSpanName; + private final String instrumentationName; // Per-invocation state + private volatile boolean tracingEnabled; private volatile Span workflowSpan; private volatile Span invocationSpan; private volatile String durableExecutionArn; @@ -117,8 +122,8 @@ public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { /** * Creates a Workflow-rooted OTel plugin with default settings: X-Ray context extraction and MDC enabled. * - *

    Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by - * {@code OtelPluginAutoConfigurationCustomizerProvider}. + *

    Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet, + * telemetry is disabled for that invocation and resolution is retried on the next invocation. */ public ExecutionOtelPlugin() { this(OtelPluginConfig.defaults()); @@ -148,6 +153,7 @@ public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelP this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); + this.instrumentationName = config.instrumentationName(); } /** @@ -162,17 +168,19 @@ public ExecutionOtelPlugin(OtelPluginConfig config) { this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - - var setup = OtelPluginSupport.resolveGlobalProvider(config, "ExecutionOtelPlugin"); - this.idGenerator = setup.idGenerator(); - this.sdkTracerProvider = setup.sdkTracerProvider(); - this.tracer = setup.tracer(); + this.instrumentationName = config.instrumentationName(); + this.idGenerator = OtelPluginSupport.createDefaultIdGenerator(); } // ─── Invocation hooks ──────────────────────────────────────────────── @Override public void onInvocationStart(InvocationInfo info) { + tracingEnabled = false; + if (!bindTracer()) { + return; + } + this.durableExecutionArn = info.durableExecutionArn(); // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context. @@ -225,10 +233,16 @@ public void onInvocationStart(InvocationInfo info) { MdcSpanEnricher.MDC_TRACE_ID, invocationSpan.getSpanContext().getTraceId()); } + tracingEnabled = true; } @Override public void onInvocationEnd(InvocationEndInfo info) { + if (!tracingEnabled) { + return; + } + tracingEnabled = false; + // Clear invocation-level MDC if (enableMdc) { MdcSpanEnricher.clear(); @@ -295,6 +309,7 @@ public void onInvocationEnd(InvocationEndInfo info) { @Override public void onOperationStart(OperationInfo info) { + if (!tracingEnabled) return; if (info.id() == null) return; var parentContext = resolveParentContext(info.parentId()); @@ -326,6 +341,7 @@ public void onOperationStart(OperationInfo info) { @Override public void onOperationEnd(OperationEndInfo info) { + if (!tracingEnabled) return; if (info.id() == null) return; var span = operationSpans.remove(info.id()); @@ -404,6 +420,8 @@ public void onOperationEnd(OperationEndInfo info) { @Override public void onUserFunctionStart(UserFunctionStartInfo info) { + if (!tracingEnabled) return; + // Skip attempt spans for CONTEXT operations — they are a scoping construct, not a retriable unit of work. Still // make the operation span current so auto-instrumented calls become children. if ("CONTEXT".equals(info.type())) { @@ -459,6 +477,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { + if (!tracingEnabled) return; + var key = attemptKey(info.id(), info.attempt()); // Close scope first (must happen on same thread as makeCurrent) @@ -490,6 +510,24 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { // ─── Helpers ───────────────────────────────────────────────────────── + private boolean bindTracer() { + if (tracer != null) { + return true; + } + synchronized (this) { + if (tracer != null) { + return true; + } + var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "ExecutionOtelPlugin"); + if (setup == null) { + return false; + } + sdkTracerProvider = setup.sdkTracerProvider(); + tracer = setup.tracer(); + return true; + } + } + private void applyInvocationStatus(Span span, InvocationEndInfo info) { // Invocation span status mapping: // SUCCEEDED, PENDING -> OK diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 17486d5e6..b51a4ce85 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -68,9 +68,9 @@ *

  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) * * - *

    When using {@link #InvocationOtelPlugin()}, the plugin requires - * {@code OtelPluginAutoConfigurationCustomizerProvider} to have been installed by the OpenTelemetry Java agent and uses - * the global provider directly. + *

    When using {@link #InvocationOtelPlugin()}, the plugin resolves the global provider at invocation start. If the + * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider + * resolution is retried on the next invocation. * *

    X-Ray console limitation: In the X-Ray "Segments Timeline" ungrouped view, the plugin's spans (Invocation, * operation, attempt) do not appear as nested subsegments of the Lambda platform segment. This is a known limitation of @@ -86,14 +86,16 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin { private static final Logger logger = LoggerFactory.getLogger(InvocationOtelPlugin.class); - private final SdkTracerProvider sdkTracerProvider; - private final Tracer tracer; + private volatile SdkTracerProvider sdkTracerProvider; + private volatile Tracer tracer; private final DeterministicIdGenerator idGenerator; private final ContextExtractor contextExtractor; private final boolean enableMdc; private final String workflowSpanName; + private final String instrumentationName; // Per-invocation state + private volatile boolean tracingEnabled; private volatile Span workflowSpan; private volatile Span invocationSpan; private volatile String durableExecutionArn; @@ -135,8 +137,8 @@ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { /** * Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled. * - *

    Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by - * {@code OtelPluginAutoConfigurationCustomizerProvider}. + *

    Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet, + * telemetry is disabled for that invocation and resolution is retried on the next invocation. */ public InvocationOtelPlugin() { this(OtelPluginConfig.defaults()); @@ -166,6 +168,7 @@ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, Otel this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); + this.instrumentationName = config.instrumentationName(); } /** @@ -180,17 +183,19 @@ public InvocationOtelPlugin(OtelPluginConfig config) { this.contextExtractor = config.contextExtractor(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - - var setup = OtelPluginSupport.resolveGlobalProvider(config, "InvocationOtelPlugin"); - this.idGenerator = setup.idGenerator(); - this.sdkTracerProvider = setup.sdkTracerProvider(); - this.tracer = setup.tracer(); + this.instrumentationName = config.instrumentationName(); + this.idGenerator = OtelPluginSupport.createDefaultIdGenerator(); } // ─── Invocation hooks ──────────────────────────────────────────────── @Override public void onInvocationStart(InvocationInfo info) { + tracingEnabled = false; + if (!bindTracer()) { + return; + } + this.durableExecutionArn = info.durableExecutionArn(); // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context. @@ -248,10 +253,16 @@ public void onInvocationStart(InvocationInfo info) { MdcSpanEnricher.MDC_TRACE_ID, invocationSpan.getSpanContext().getTraceId()); } + tracingEnabled = true; } @Override public void onInvocationEnd(InvocationEndInfo info) { + if (!tracingEnabled) { + return; + } + tracingEnabled = false; + // Clear invocation-level MDC (set in onInvocationStart on the handler thread) if (enableMdc) { MdcSpanEnricher.clear(); @@ -326,6 +337,7 @@ public void onInvocationEnd(InvocationEndInfo info) { @Override public void onOperationStart(OperationInfo info) { + if (!tracingEnabled) return; if (info.id() == null) return; var parentContext = resolveParentContext(info.parentId()); @@ -360,6 +372,7 @@ public void onOperationStart(OperationInfo info) { @Override public void onOperationEnd(OperationEndInfo info) { + if (!tracingEnabled) return; if (info.id() == null) return; var span = operationSpans.remove(info.id()); @@ -427,6 +440,8 @@ public void onOperationEnd(OperationEndInfo info) { @Override public void onUserFunctionStart(UserFunctionStartInfo info) { + if (!tracingEnabled) return; + // Skip attempt spans for CONTEXT operations — they are a scoping construct, not a // retriable unit of work, so attempt number/outcome attributes don't apply. // The operation span itself provides parent context for auto-instrumented calls. @@ -485,6 +500,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { + if (!tracingEnabled) return; + var key = attemptKey(info.id(), info.attempt()); // Close scope first (must happen on same thread as makeCurrent) @@ -525,6 +542,24 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { // ─── Helpers ───────────────────────────────────────────────────────── + private boolean bindTracer() { + if (tracer != null) { + return true; + } + synchronized (this) { + if (tracer != null) { + return true; + } + var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "InvocationOtelPlugin"); + if (setup == null) { + return false; + } + sdkTracerProvider = setup.sdkTracerProvider(); + tracer = setup.tracer(); + return true; + } + } + private void endOpenSpansChildFirst() { // Attempt spans are children of operation spans. for (var scope : attemptScopes.values()) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index f495a7137..cac13d10f 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -19,46 +19,56 @@ final class OtelPluginSupport { private OtelPluginSupport() {} - /** Gets the global TracerProvider after validating the SPI was installed. */ - static TracerProvider getDefaultTracerProvider(String pluginName) { - validateAutoConfigurationCustomizerProviderInstalled(pluginName); - - var globalTracerProvider = GlobalOpenTelemetry.getTracerProvider(); - if (globalTracerProvider == TracerProvider.noop()) { - throw new IllegalStateException(pluginName + "() requires GlobalOpenTelemetry to be initialized by " - + "OtelPluginAutoConfigurationCustomizerProvider through the OpenTelemetry Java agent."); - } - logger.info( - "{} initialized from existing GlobalOpenTelemetry tracer provider {}; assuming " - + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider", - pluginName, - globalTracerProvider.getClass().getName()); - return globalTracerProvider; - } - /** Creates a new DeterministicIdGenerator for the application-side state bridge. */ static DeterministicIdGenerator createDefaultIdGenerator() { return new DeterministicIdGenerator(); } - /** The tracer provider, tracer, and ID generator resolved for a config-only plugin constructor. */ - record ProviderSetup(SdkTracerProvider sdkTracerProvider, Tracer tracer, DeterministicIdGenerator idGenerator) {} + /** The tracer provider and tracer resolved from the global OpenTelemetry instance. */ + record ProviderSetup(SdkTracerProvider sdkTracerProvider, Tracer tracer) {} /** - * Resolves the ADOT/global tracer provider for config-only plugin constructors. Explicit providers are supplied - * through the {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructors instead. + * Tries to resolve the ADOT/global tracer provider without installing OpenTelemetry's no-op global. This is called + * at invocation start so a plugin constructed before the Java agent finishes initialization can bind later. * - * @param config the plugin configuration + * @param instrumentationName the instrumentation scope name * @param pluginName the plugin name used in diagnostics/flush logging - * @return the resolved provider, tracer, and ID generator + * @return the resolved provider and tracer, or {@code null} when telemetry must be disabled for this invocation */ - static ProviderSetup resolveGlobalProvider(OtelPluginConfig config, String pluginName) { - var idGenerator = createDefaultIdGenerator(); - var tracerProvider = getDefaultTracerProvider(pluginName); + static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String pluginName) { + if (!OtelPluginAutoConfigurationState.isInstalled()) { + logger.warn( + "{} telemetry is disabled for this invocation because " + + "OtelPluginAutoConfigurationCustomizerProvider is not installed yet. Provider resolution " + + "will be retried on the next invocation. {}", + pluginName, + javaAgentExtensionsDiagnostic()); + return null; + } + if (!GlobalOpenTelemetry.isSet()) { + logger.warn( + "{} telemetry is disabled for this invocation because GlobalOpenTelemetry is not initialized yet. " + + "Provider resolution will be retried on the next invocation.", + pluginName); + return null; + } + + var tracerProvider = GlobalOpenTelemetry.getOrNoop().getTracerProvider(); + if (tracerProvider == TracerProvider.noop()) { + logger.warn( + "{} telemetry is disabled for this invocation because GlobalOpenTelemetry contains a no-op tracer " + + "provider. Provider resolution will be retried on the next invocation.", + pluginName); + return null; + } + + logger.info( + "{} initialized from existing GlobalOpenTelemetry tracer provider {}; assuming " + + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider", + pluginName, + tracerProvider.getClass().getName()); return new ProviderSetup( - getSdkTracerProviderForFlush(tracerProvider, pluginName), - tracerProvider.get(config.instrumentationName()), - idGenerator); + getSdkTracerProviderForFlush(tracerProvider, pluginName), tracerProvider.get(instrumentationName)); } /** Extracts trace context from the current OTel span (fallback when X-Ray header is unavailable). */ @@ -84,18 +94,6 @@ static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvi return null; } - private static void validateAutoConfigurationCustomizerProviderInstalled(String pluginName) { - if (OtelPluginAutoConfigurationState.isInstalled()) { - return; - } - throw new IllegalStateException( - pluginName + "() requires OtelPluginAutoConfigurationCustomizerProvider to be installed by the " - + "OpenTelemetry Java agent. Package this plugin jar as an agent extension and set " - + "OTEL_JAVAAGENT_EXTENSIONS or -Dotel.javaagent.extensions to that jar before constructing " - + pluginName + "(). " - + javaAgentExtensionsDiagnostic()); - } - private static String javaAgentExtensionsDiagnostic() { var propertyValue = System.getProperty("otel.javaagent.extensions"); var environmentValue = System.getenv("OTEL_JAVAAGENT_EXTENSIONS"); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index fb417f7ed..0c2a1d3f1 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -82,23 +82,75 @@ void customInstrumentationName_isUsedForTracerScope() { } @Test - void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstalled() { + void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { GlobalOpenTelemetry.resetForTest(); - var error = assertThrows(IllegalStateException.class, ExecutionOtelPlugin::new); - assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider")); + OtelPluginAutoConfigurationState.markInstalled(); + + var defaultPlugin = new ExecutionOtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now())); + defaultPlugin.onOperationStart(new OperationInfo( + "op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-disabled", + "disabled-step", + "STEP", + "Step", + null, + Instant.now(), + Instant.now(), + "SUCCEEDED", + null, + false, + null, + null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-disabled", "arn:disabled", true, InvocationStatus.SUCCEEDED, null)); + + assertFalse(GlobalOpenTelemetry.isSet(), "An unavailable provider must not install the no-op global"); + + var globalExporter = InMemorySpanExporter.create(); + var globalTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); + + defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now())); + defaultPlugin.onOperationStart(new OperationInfo( + "op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-enabled", + "enabled-step", + "STEP", + "Step", + null, + Instant.now(), + Instant.now(), + "SUCCEEDED", + null, + false, + null, + null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null)); + + var spans = globalExporter.getFinishedSpanItems(); + assertEquals(3, spans.size()); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step"))); + assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step"))); } @Test void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { + var defaultPlugin = new ExecutionOtelPlugin(); + assertFalse(GlobalOpenTelemetry.isSet()); + OtelPluginAutoConfigurationState.markInstalled(); - GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var globalTracerProvider = SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - var defaultPlugin = new ExecutionOtelPlugin(); defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); defaultPlugin.onOperationStart( new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false)); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java index abf9a66bd..274944c36 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java @@ -544,17 +544,18 @@ void waitForCondition_producesSpansWithAttempts() { } @Test - void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { + void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() { + var defaultPlugin = new InvocationOtelPlugin(); + assertFalse(GlobalOpenTelemetry.isSet()); + OtelPluginAutoConfigurationState.markInstalled(); - GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var globalTracerProvider = SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - var defaultConfig = - DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); + var defaultConfig = DurableConfig.builder().withPlugins(defaultPlugin).build(); var runner = LocalDurableTestRunner.create( String.class, (input, ctx) -> ctx.step("global-step", String.class, stepCtx -> "Hello " + input), diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index 283c7e2b3..be68c9716 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -66,37 +66,75 @@ void tearDown() { } @Test - void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstalled() { + void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.markInstalled(); - var error = assertThrows(IllegalStateException.class, InvocationOtelPlugin::new); + var defaultPlugin = new InvocationOtelPlugin(); + defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now())); + defaultPlugin.onOperationStart(new OperationInfo( + "op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-disabled", + "disabled-step", + "STEP", + "Step", + null, + Instant.now(), + Instant.now(), + "SUCCEEDED", + null, + false, + null, + null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-disabled", "arn:disabled", true, InvocationStatus.SUCCEEDED, null)); - assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider")); - assertTrue(error.getMessage().contains("OTEL_JAVAAGENT_EXTENSIONS")); - } + assertFalse(GlobalOpenTelemetry.isSet(), "An unavailable provider must not install the no-op global"); - @Test - void defaultConstructor_throwsWhenGlobalOpenTelemetryIsNotInitializedBySpi() { - OtelPluginAutoConfigurationState.markInstalled(); - GlobalOpenTelemetry.resetForTest(); + var globalExporter = InMemorySpanExporter.create(); + var globalTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) + .build(); + OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - var error = assertThrows(IllegalStateException.class, InvocationOtelPlugin::new); + defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now())); + defaultPlugin.onOperationStart(new OperationInfo( + "op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); + defaultPlugin.onOperationEnd(new OperationEndInfo( + "op-enabled", + "enabled-step", + "STEP", + "Step", + null, + Instant.now(), + Instant.now(), + "SUCCEEDED", + null, + false, + null, + null)); + defaultPlugin.onInvocationEnd( + new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null)); - assertTrue(error.getMessage().contains("GlobalOpenTelemetry")); - assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider")); + var spans = globalExporter.getFinishedSpanItems(); + assertEquals(3, spans.size()); + assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step"))); + assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step"))); } @Test void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { + var defaultPlugin = new InvocationOtelPlugin(); + assertFalse(GlobalOpenTelemetry.isSet()); + OtelPluginAutoConfigurationState.markInstalled(); - GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); var globalTracerProvider = SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(globalExporter)) .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - var defaultPlugin = new InvocationOtelPlugin(); defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); defaultPlugin.onOperationStart( new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false));