This handler configures the OTel plugin with: * *
Expected trace structure: * *
- * durable.invocation - * ├── durable.step:create-greeting [attempt 1] - * ├── durable.step:create-greeting (operation, backfilled) - * ├── durable.step:transform [attempt 1] - * └── durable.step:transform (operation, backfilled) + * Workflow + * + * Invocation + * ├── create-greeting + * │ └── create-greeting attempt 1 + * └── transform + * └── transform attempt 1 **/ public class OtelExample extends DurableHandler
{@link ExecutionOtelPlugin#ExecutionOtelPlugin()} uses the global provider initialized by the ADOT Java agent. The
- * ExecutionOtelPlugin renders the Workflow span as the trace root with operations as siblings of the invocation span.
+ * ExecutionOtelPlugin renders the Workflow span as the durable trace root with operations beneath it. Operations link
+ * to the Invocation span in the ambient Lambda trace.
*/
@ExampleTemplate(tracing = true, javaAgent = true)
public class OtelXRayExecutionStepExample extends DurableHandler Expected trace structure in X-Ray (all under one trace ID — backend propagates same Root):
+ * Expected trace structure in X-Ray:
*
* After invoking the function, the test queries the X-Ray API to verify:
*
* Enable with: {@code -Dtest.cloud.enabled=true}
@@ -112,7 +113,7 @@ private static String arn(String functionName) {
// ─── Test: Simple Steps (Single Invocation) ──────────────────────────
@Test
- void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
+ void simpleSteps_producesCorrelatedWorkflowAndInvocationTracesInXRay() throws Exception {
var startTime = Instant.now();
// 1. Invoke the function (use unique input to avoid stale executions)
@@ -128,11 +129,12 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
Thread.sleep(XRAY_INGESTION_DELAY.toMillis());
// 3. Query X-Ray for the trace, retrying until durable spans appear
- var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "create-greeting");
+ var invocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "create-greeting");
+ var workflowTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "Workflow");
// 5. Verify span structure
var segmentDocuments =
- durableTrace.segments().stream().map(Segment::document).toList();
+ invocationTrace.segments().stream().map(Segment::document).toList();
var allSegmentText = String.join("\n", segmentDocuments);
// Verify expected span names appear in the trace
@@ -142,19 +144,20 @@ void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
assertTrue(allSegmentText.contains("create-greeting"), "Expected create-greeting span in trace");
assertTrue(allSegmentText.contains("transform"), "Expected transform span in trace");
- // Verify all segments share the same trace ID (single unified trace)
- var uniqueTraceIds =
- durableTrace.segments().stream().map(seg -> durableTrace.id()).collect(Collectors.toSet());
- assertEquals(1, uniqueTraceIds.size(), "All segments should belong to a single trace");
+ assertNotEquals(
+ workflowTrace.id(),
+ invocationTrace.id(),
+ "Workflow and Invocation spans must not create disconnected roots in one trace");
System.out.println("✅ Simple steps test passed — "
- + durableTrace.segments().size() + " segments in trace " + durableTrace.id());
+ + invocationTrace.segments().size() + " invocation segments correlated with Workflow trace "
+ + workflowTrace.id());
}
// ─── Test: Wait + Resume (Multi-Invocation) ─────────────────────────
@Test
- void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
+ void waitAndResume_preservesWorkflowCorrelationAcrossInvocations() throws Exception {
var startTime = Instant.now();
// 1. Invoke the function — will suspend on wait, then resume automatically
@@ -172,14 +175,22 @@ void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
Thread.sleep(XRAY_INGESTION_DELAY.plus(Duration.ofSeconds(5)).toMillis());
// 3. Query X-Ray for the trace, retrying until durable spans appear
- var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "before-wait");
+ var firstInvocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "before-wait");
+ var secondInvocationTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "after-wait");
+ var workflowTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "Workflow");
// 4. Verify multi-invocation trace structure
- var segmentDocuments =
- durableTrace.segments().stream().map(Segment::document).toList();
+ var segmentDocuments = new ArrayList Trace ID resolution order:
+ * The durable plugins use short-lived ID overrides around their own {@link SpanBuilder#startSpan()} calls. Outside
+ * those scopes, generation delegates to the fallback generator so unrelated instrumentation keeps normal root trace ID
+ * generation. Scoped values are also bridged through thread-keyed system properties because the application plugin and
+ * Java-agent extension may load this class in different class loaders.
*
- * Span IDs for operations are deterministic (derived from execution ARN + operation ID), ensuring the same operation
- * produces the same span across invocations. When no pending operation ID is set, falls back to random generation.
+ * The existing setter methods remain available for callers that use this class directly. Plugin code does not use
+ * that persistent mode.
*/
public class DeterministicIdGenerator implements IdGenerator {
- private static final IdGenerator RANDOM = IdGenerator.random();
private static final String PROPERTY_PREFIX = "software.amazon.lambda.durable.otel.";
- private static final String EXTRACTED_TRACE_ID_PROPERTY = PROPERTY_PREFIX + "extractedTraceId";
- private static final String DURABLE_EXECUTION_ARN_PROPERTY = PROPERTY_PREFIX + "durableExecutionArn";
- private static final String PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingSpanOperationId.";
- private static final String PENDING_RAW_SPAN_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingRawSpanId.";
+ private static final String SCOPED_TRACE_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "scopedTraceId.";
+ private static final String SCOPED_SPAN_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "scopedSpanId.";
- private final AtomicReference Trace ID resolution matches {@link InvocationOtelPlugin}: the X-Ray trace ID from {@code _X_AMZN_TRACE_ID} when
- * available (the backend propagates the same Root to all invocations, unifying the trace), else a deterministic trace
- * ID derived from the execution ARN.
+ * The Workflow trace ID is derived from the execution start time and ARN, and is independent of the ambient
+ * Lambda/X-Ray trace. Invocation spans inherit the active ambient context, or extracted upstream context as a fallback.
+ * When using {@link #ExecutionOtelPlugin()}, the plugin resolves the global provider at invocation start. If the
+ * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider
+ * resolution is retried on the next invocation.
*
* Status mapping (parity with the Python/JS references):
*
@@ -79,18 +81,20 @@ public class ExecutionOtelPlugin implements DurableExecutionPlugin {
private static final Logger logger = LoggerFactory.getLogger(ExecutionOtelPlugin.class);
- private final SdkTracerProvider sdkTracerProvider;
- private final Tracer tracer;
+ private volatile SdkTracerProvider sdkTracerProvider;
+ private volatile Tracer tracer;
private final DeterministicIdGenerator idGenerator;
private final ContextExtractor contextExtractor;
private final boolean enableMdc;
private final String workflowSpanName;
- private final ProviderSource providerSource;
+ private final String instrumentationName;
// Per-invocation state
+ private volatile boolean tracingEnabled;
private volatile Span workflowSpan;
private volatile Span invocationSpan;
private volatile String durableExecutionArn;
+ private volatile String workflowTraceId;
// Thread-safe storage for operation spans (keyed by operationId) — open spans that need ending
private final ConcurrentHashMap Uses the provided tracer provider builder. For ADOT Java agent usage, prefer {@link #ExecutionOtelPlugin()}
* with the plugin jar configured through {@code OTEL_JAVAAGENT_EXTENSIONS}.
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
*/
public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
this(tracerProviderBuilder, OtelPluginConfig.defaults());
@@ -118,8 +122,8 @@ public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
/**
* Creates a Workflow-rooted OTel plugin with default settings: X-Ray context extraction and MDC enabled.
*
- * Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by
- * {@code OtelPluginAutoConfigurationCustomizerProvider}.
+ * Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet,
+ * telemetry is disabled for that invocation and resolution is retried on the next invocation.
*/
public ExecutionOtelPlugin() {
this(OtelPluginConfig.defaults());
@@ -138,67 +142,51 @@ public ExecutionOtelPlugin() {
* OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
* }
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
* @param config the plugin configuration
*/
public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
- this.idGenerator = new DeterministicIdGenerator();
+ this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder);
- this.sdkTracerProvider =
- tracerProviderBuilder.setIdGenerator(idGenerator).build();
+ this.sdkTracerProvider = tracerProviderBuilder.build();
this.tracer = sdkTracerProvider.get(config.instrumentationName());
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
- this.providerSource = ProviderSource.EXPLICIT;
+ this.instrumentationName = config.instrumentationName();
}
/**
* Creates a Workflow-rooted OTel plugin from configuration alone (no caller-supplied tracer provider builder).
*
- * The provider is taken from {@link OtelPluginConfig#providerSource()}: {@link ProviderSource#GLOBAL} uses the
- * ADOT/global provider, otherwise the default {@link ProviderSource#AUTO_OTLP} builds a plugin-owned OTLP/HTTP
- * provider (matching the JavaScript and Python SDK plugins). {@link ProviderSource#EXPLICIT} is rejected here —
- * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for that.
+ * The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the
+ * two-arg constructor for an application-owned provider.
*
* @param config the plugin configuration
- * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT}
*/
public ExecutionOtelPlugin(OtelPluginConfig config) {
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
-
- var setup = OtelPluginSupport.resolveConfiguredProvider(config, "ExecutionOtelPlugin");
- this.providerSource = setup.source();
- this.idGenerator = setup.idGenerator();
- this.sdkTracerProvider = setup.sdkTracerProvider();
- this.tracer = setup.tracer();
- }
-
- /** The tier that produced this plugin's tracer provider. */
- public ProviderSource providerSource() {
- return providerSource;
+ this.instrumentationName = config.instrumentationName();
+ this.idGenerator = OtelPluginSupport.createDefaultIdGenerator();
}
// ─── Invocation hooks ────────────────────────────────────────────────
@Override
public void onInvocationStart(InvocationInfo info) {
- this.durableExecutionArn = info.durableExecutionArn();
+ tracingEnabled = false;
+ if (!bindTracer()) {
+ return;
+ }
- // Set execution ARN for deterministic span/trace ID generation
- idGenerator.setDurableExecutionArn(info.durableExecutionArn());
+ this.durableExecutionArn = info.durableExecutionArn();
- // Extract trace context from the environment (X-Ray header), falling back to the ambient OTel span.
- var extractedContext = contextExtractor.extract();
- if (extractedContext == null) {
- extractedContext = extractCurrentSpanContext();
- }
- if (extractedContext != null) {
- idGenerator.setExtractedTraceId(extractedContext.traceId());
- } else {
- idGenerator.setExtractedTraceId(null);
+ // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context.
+ var invocationParent = extractCurrentSpanContext();
+ if (invocationParent == null) {
+ invocationParent = contextExtractor.extract();
}
// Workflow root span — deterministic span ID from the ARN, no parent. Recreated every invocation with the
@@ -209,14 +197,16 @@ public void onInvocationStart(InvocationInfo info) {
.setNoParent()
.setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn())
.setStartTimestamp(info.executionStartTime() != null ? info.executionStartTime() : Instant.now());
- idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId());
- workflowSpan = workflowSpanBuilder.startSpan();
+ workflowTraceId =
+ idGenerator.generateTraceIdForExecution(info.durableExecutionArn(), info.executionStartTime());
+ var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn());
+ workflowSpan = idGenerator.startSpan(workflowSpanBuilder, workflowTraceId, workflowSpanId);
Context parentContext;
- if (extractedContext != null && extractedContext.parentSpanId() != null) {
+ if (invocationParent != null && invocationParent.parentSpanId() != null) {
var parentSpanContext = SpanContext.createFromRemoteParent(
- extractedContext.traceId(),
- extractedContext.parentSpanId(),
+ invocationParent.traceId(),
+ invocationParent.parentSpanId(),
TraceFlags.getSampled(),
TraceState.getDefault());
parentContext = Context.root().with(Span.wrap(parentSpanContext));
@@ -239,13 +229,20 @@ public void onInvocationStart(InvocationInfo info) {
// Inject MDC on the handler thread so handler-level logs (between steps) have trace context.
if (enableMdc) {
- var traceId = idGenerator.generateTraceId();
- MDC.put(MdcSpanEnricher.MDC_TRACE_ID, traceId);
+ MDC.put(
+ MdcSpanEnricher.MDC_TRACE_ID,
+ invocationSpan.getSpanContext().getTraceId());
}
+ tracingEnabled = true;
}
@Override
public void onInvocationEnd(InvocationEndInfo info) {
+ if (!tracingEnabled) {
+ return;
+ }
+ tracingEnabled = false;
+
// Clear invocation-level MDC
if (enableMdc) {
MdcSpanEnricher.clear();
@@ -312,14 +309,11 @@ public void onInvocationEnd(InvocationEndInfo info) {
@Override
public void onOperationStart(OperationInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var parentContext = resolveParentContext(info.parentId());
- // Always use a deterministic span ID keyed by operation ID (regardless of replay) so a suspended-then-resumed
- // operation stitches into a single logical span across invocations.
- idGenerator.setNextSpanOperationId(info.id());
-
var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name()))
.setParent(parentContext)
.setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn)
@@ -337,7 +331,8 @@ public void onOperationStart(OperationInfo info) {
spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType());
}
- var span = spanBuilder.startSpan();
+ var operationSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id());
+ var span = idGenerator.startSpan(spanBuilder, null, operationSpanId);
// Store the open span — will be ended in onOperationEnd or onInvocationEnd
operationSpans.put(info.id(), span);
@@ -346,6 +341,7 @@ public void onOperationStart(OperationInfo info) {
@Override
public void onOperationEnd(OperationEndInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var span = operationSpans.remove(info.id());
@@ -376,7 +372,6 @@ public void onOperationEnd(OperationEndInfo info) {
// now, using its deterministic span ID (stable across the execution), plus a link to the invocation
// that completed it.
operationContexts.remove(info.id());
- idGenerator.setNextSpanOperationId(info.id());
var parentContext = resolveParentContext(info.parentId());
@@ -397,7 +392,8 @@ public void onOperationEnd(OperationEndInfo info) {
spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType());
}
- var continuationSpan = spanBuilder.startSpan();
+ var operationSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id());
+ var continuationSpan = idGenerator.startSpan(spanBuilder, null, operationSpanId);
if (info.status() != null) {
continuationSpan.setAttribute(DURABLE_OPERATION_STATUS, info.status());
@@ -424,6 +420,8 @@ public void onOperationEnd(OperationEndInfo info) {
@Override
public void onUserFunctionStart(UserFunctionStartInfo info) {
+ if (!tracingEnabled) return;
+
// Skip attempt spans for CONTEXT operations — they are a scoping construct, not a retriable unit of work. Still
// make the operation span current so auto-instrumented calls become children.
if ("CONTEXT".equals(info.type())) {
@@ -479,6 +477,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if (!tracingEnabled) return;
+
var key = attemptKey(info.id(), info.attempt());
// Close scope first (must happen on same thread as makeCurrent)
@@ -510,6 +510,24 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
// ─── Helpers ─────────────────────────────────────────────────────────
+ private boolean bindTracer() {
+ if (tracer != null) {
+ return true;
+ }
+ synchronized (this) {
+ if (tracer != null) {
+ return true;
+ }
+ var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "ExecutionOtelPlugin");
+ if (setup == null) {
+ return false;
+ }
+ sdkTracerProvider = setup.sdkTracerProvider();
+ tracer = setup.tracer();
+ return true;
+ }
+ }
+
private void applyInvocationStatus(Span span, InvocationEndInfo info) {
// Invocation span status mapping:
// SUCCEEDED, PENDING -> OK
@@ -556,10 +574,9 @@ private Context resolveParentContext(String parentId) {
return Context.current().with(Span.wrap(parentSpanContext));
}
// Parent operation from a prior invocation — create a non-recording placeholder with its deterministic ID.
- var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(parentId);
- var traceId = idGenerator.generateTraceId();
+ var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, parentId);
var placeholderContext = SpanContext.create(
- traceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault());
+ workflowTraceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault());
return Context.current().with(Span.wrap(placeholderContext));
}
// No parent operation — hang off the Workflow root span.
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java
index 6aaacd2e9..d381732cb 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java
@@ -5,9 +5,8 @@
/**
* Trace context extracted from the Lambda runtime environment.
*
- * Contains the trace ID (always present) and an optional parent span ID. When the durable execution backend
- * propagates the same X-Ray Root across all invocations, the trace ID will be consistent, enabling spans from different
- * invocations to be stitched into a single trace.
+ * Contains the trace ID (always present) and an optional parent span ID used to parent an Invocation span to ambient
+ * Lambda/X-Ray context.
*
* @param traceId 32-character lowercase hex trace ID (OTel format, no dashes)
* @param parentSpanId 16-character lowercase hex parent span ID (may be null if no parent available)
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
index 36e741f37..b51a4ce85 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
@@ -50,22 +50,16 @@
*
* Trace ID resolution:
- *
- * The Workflow trace ID is derived from the execution start time and ARN, and is independent of the ambient
+ * Lambda/X-Ray trace. Invocation spans inherit the active ambient context, or extracted upstream context as a fallback.
*
* Requires the ADOT Lambda Layer for trace export. Configure with:
*
@@ -74,16 +68,16 @@
* When using {@link #InvocationOtelPlugin()}, the plugin requires
- * {@code OtelPluginAutoConfigurationCustomizerProvider} to have been installed by the OpenTelemetry Java agent and uses
- * the global provider directly.
+ * When using {@link #InvocationOtelPlugin()}, the plugin resolves the global provider at invocation start. If the
+ * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider
+ * resolution is retried on the next invocation.
*
* X-Ray console limitation: In the X-Ray "Segments Timeline" ungrouped view, the plugin's spans (Invocation,
* operation, attempt) do not appear as nested subsegments of the Lambda platform segment. This is a known limitation of
* the OTLP-to-X-Ray conversion: the ADOT collector cannot attach OTLP-exported spans as subsegments of the Lambda
* service's native X-Ray segment because that segment is created outside the OTLP pipeline. Use the "Group by nodes"
- * view to see the full span hierarchy correctly — it stitches all spans together by trace ID and parent-child
- * relationships regardless of segment boundaries.
+ * view to inspect parent-child relationships within the ambient Invocation trace and the links to the independent
+ * Workflow trace.
*
* Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple
* threads.
@@ -92,15 +86,16 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin {
private static final Logger logger = LoggerFactory.getLogger(InvocationOtelPlugin.class);
- private final SdkTracerProvider sdkTracerProvider;
- private final Tracer tracer;
+ private volatile SdkTracerProvider sdkTracerProvider;
+ private volatile Tracer tracer;
private final DeterministicIdGenerator idGenerator;
private final ContextExtractor contextExtractor;
private final boolean enableMdc;
private final String workflowSpanName;
- private final ProviderSource providerSource;
+ private final String instrumentationName;
// Per-invocation state
+ private volatile boolean tracingEnabled;
private volatile Span workflowSpan;
private volatile Span invocationSpan;
private volatile String durableExecutionArn;
@@ -133,7 +128,7 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin {
* SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
* }
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
*/
public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
this(tracerProviderBuilder, OtelPluginConfig.defaults());
@@ -142,8 +137,8 @@ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
/**
* Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled.
*
- * Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by
- * {@code OtelPluginAutoConfigurationCustomizerProvider}.
+ * Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet,
+ * telemetry is disabled for that invocation and resolution is retried on the next invocation.
*/
public InvocationOtelPlugin() {
this(OtelPluginConfig.defaults());
@@ -162,71 +157,52 @@ public InvocationOtelPlugin() {
* OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
* }
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
* @param config the plugin configuration
*/
public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
- this.idGenerator = new DeterministicIdGenerator();
+ this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder);
- this.sdkTracerProvider =
- tracerProviderBuilder.setIdGenerator(idGenerator).build();
+ this.sdkTracerProvider = tracerProviderBuilder.build();
this.tracer = sdkTracerProvider.get(config.instrumentationName());
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
- this.providerSource = ProviderSource.EXPLICIT;
+ this.instrumentationName = config.instrumentationName();
}
/**
* Creates an OTel plugin from configuration alone (no caller-supplied tracer provider builder).
*
- * The provider is taken from {@link OtelPluginConfig#providerSource()}: {@link ProviderSource#GLOBAL} uses the
- * ADOT/global provider, otherwise the default {@link ProviderSource#AUTO_OTLP} builds a plugin-owned OTLP/HTTP
- * provider (matching the JavaScript and Python SDK plugins). {@link ProviderSource#EXPLICIT} is rejected here —
- * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for that.
+ * The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the
+ * two-arg constructor for an application-owned provider.
*
* @param config the plugin configuration
- * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT}
*/
public InvocationOtelPlugin(OtelPluginConfig config) {
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
-
- var setup = OtelPluginSupport.resolveConfiguredProvider(config, "InvocationOtelPlugin");
- this.providerSource = setup.source();
- this.idGenerator = setup.idGenerator();
- this.sdkTracerProvider = setup.sdkTracerProvider();
- this.tracer = setup.tracer();
- }
-
- /** The tier that produced this plugin's tracer provider. */
- public ProviderSource providerSource() {
- return providerSource;
+ this.instrumentationName = config.instrumentationName();
+ this.idGenerator = OtelPluginSupport.createDefaultIdGenerator();
}
// ─── Invocation hooks ────────────────────────────────────────────────
@Override
public void onInvocationStart(InvocationInfo info) {
- this.durableExecutionArn = info.durableExecutionArn();
-
- // Set execution ARN for deterministic span ID generation
- idGenerator.setDurableExecutionArn(info.durableExecutionArn());
-
- // Extract trace context from environment (X-Ray header)
- var extractedContext = contextExtractor.extract();
- if (extractedContext == null) {
- extractedContext = extractCurrentSpanContext();
+ tracingEnabled = false;
+ if (!bindTracer()) {
+ return;
}
- if (extractedContext != null) {
- // Use the X-Ray trace ID — backend propagates same Root across all invocations
- idGenerator.setExtractedTraceId(extractedContext.traceId());
- } else {
- idGenerator.setExtractedTraceId(null);
+ this.durableExecutionArn = info.durableExecutionArn();
+
+ // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context.
+ var invocationParent = extractCurrentSpanContext();
+ if (invocationParent == null) {
+ invocationParent = contextExtractor.extract();
}
- // If no extracted context, idGenerator falls back to ARN-derived trace ID
// Workflow root span — one logical span per durable execution, created unconditionally (independent of the
// X-Ray parent below). Deterministic span ID from the ARN so it is the same across invocations; exported once,
@@ -237,17 +213,19 @@ public void onInvocationStart(InvocationInfo info) {
.setNoParent()
.setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn())
.setStartTimestamp(info.executionStartTime() != null ? info.executionStartTime() : Instant.now());
- idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId());
- workflowSpan = workflowSpanBuilder.startSpan();
+ var workflowTraceId =
+ idGenerator.generateTraceIdForExecution(info.durableExecutionArn(), info.executionStartTime());
+ var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn());
+ workflowSpan = idGenerator.startSpan(workflowSpanBuilder, workflowTraceId, workflowSpanId);
// Determine parent context for the invocation span.
Context parentContext;
- if (extractedContext != null && extractedContext.parentSpanId() != null) {
+ if (invocationParent != null && invocationParent.parentSpanId() != null) {
// Reconstruct a remote parent from the extracted trace context (X-Ray header or current span).
// This connects plugin spans to the Lambda service's X-Ray segments.
var parentSpanContext = SpanContext.createFromRemoteParent(
- extractedContext.traceId(),
- extractedContext.parentSpanId(),
+ invocationParent.traceId(),
+ invocationParent.parentSpanId(),
TraceFlags.getSampled(),
TraceState.getDefault());
parentContext = Context.root().with(Span.wrap(parentSpanContext));
@@ -271,13 +249,20 @@ public void onInvocationStart(InvocationInfo info) {
// Inject MDC on the handler thread so handler-level logs (between steps) have trace context.
// This runs on the same thread as context.getLogger() calls in the handler.
if (enableMdc) {
- var traceId = idGenerator.generateTraceId();
- MDC.put(MdcSpanEnricher.MDC_TRACE_ID, traceId);
+ MDC.put(
+ MdcSpanEnricher.MDC_TRACE_ID,
+ invocationSpan.getSpanContext().getTraceId());
}
+ tracingEnabled = true;
}
@Override
public void onInvocationEnd(InvocationEndInfo info) {
+ if (!tracingEnabled) {
+ return;
+ }
+ tracingEnabled = false;
+
// Clear invocation-level MDC (set in onInvocationStart on the handler thread)
if (enableMdc) {
MdcSpanEnricher.clear();
@@ -352,6 +337,7 @@ public void onInvocationEnd(InvocationEndInfo info) {
@Override
public void onOperationStart(OperationInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var parentContext = resolveParentContext(info.parentId());
@@ -363,19 +349,6 @@ public void onOperationStart(OperationInfo info) {
.setAttribute(DURABLE_OPERATION_TYPE, info.type())
.setAttribute(DURABLE_OPERATION_STATUS, info.status() != null ? info.status() : "STARTED");
- if (info.isReplay()) {
- // Operation was already started in a prior invocation — use a random span ID
- // and add a Link to the deterministic span from the original invocation for correlation.
- var deterministicSpanId = idGenerator.generateSpanIdForOperation(info.id());
- var traceId = idGenerator.generateTraceId();
- var linkedSpanContext =
- SpanContext.create(traceId, deterministicSpanId, TraceFlags.getSampled(), TraceState.getDefault());
- spanBuilder.addLink(linkedSpanContext);
- } else {
- // First execution — use deterministic span ID so continuations can link back
- idGenerator.setNextSpanOperationId(info.id());
- }
-
// Link to the Workflow span for execution-level correlation (operation stays parented to the invocation span).
addWorkflowLink(spanBuilder);
@@ -386,7 +359,10 @@ public void onOperationStart(OperationInfo info) {
spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType());
}
- var span = spanBuilder.startSpan();
+ var span = info.isReplay()
+ ? spanBuilder.startSpan()
+ : idGenerator.startSpan(
+ spanBuilder, null, idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id()));
// Store the open span — will be ended in onOperationEnd or onInvocationEnd
operationSpans.put(info.id(), span);
@@ -396,6 +372,7 @@ public void onOperationStart(OperationInfo info) {
@Override
public void onOperationEnd(OperationEndInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var span = operationSpans.remove(info.id());
@@ -421,18 +398,10 @@ public void onOperationEnd(OperationEndInfo info) {
}
span.end();
} else {
- // Operation was started in a prior invocation — create a continuation span with Link
- // to the deterministic span ID from the original invocation.
- var deterministicSpanId = idGenerator.generateSpanIdForOperation(info.id());
- var traceId = idGenerator.generateTraceId();
- var linkedSpanContext =
- SpanContext.create(traceId, deterministicSpanId, TraceFlags.getSampled(), TraceState.getDefault());
-
var parentContext = resolveParentContext(info.parentId());
var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name()))
.setParent(parentContext)
- .addLink(linkedSpanContext)
.setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn)
.setAttribute(DURABLE_OPERATION_ID, info.id())
.setAttribute(DURABLE_OPERATION_TYPE, info.type());
@@ -471,6 +440,8 @@ public void onOperationEnd(OperationEndInfo info) {
@Override
public void onUserFunctionStart(UserFunctionStartInfo info) {
+ if (!tracingEnabled) return;
+
// Skip attempt spans for CONTEXT operations — they are a scoping construct, not a
// retriable unit of work, so attempt number/outcome attributes don't apply.
// The operation span itself provides parent context for auto-instrumented calls.
@@ -529,6 +500,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if (!tracingEnabled) return;
+
var key = attemptKey(info.id(), info.attempt());
// Close scope first (must happen on same thread as makeCurrent)
@@ -569,6 +542,24 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
// ─── Helpers ─────────────────────────────────────────────────────────
+ private boolean bindTracer() {
+ if (tracer != null) {
+ return true;
+ }
+ synchronized (this) {
+ if (tracer != null) {
+ return true;
+ }
+ var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "InvocationOtelPlugin");
+ if (setup == null) {
+ return false;
+ }
+ sdkTracerProvider = setup.sdkTracerProvider();
+ tracer = setup.tracer();
+ return true;
+ }
+ }
+
private void endOpenSpansChildFirst() {
// Attempt spans are children of operation spans.
for (var scope : attemptScopes.values()) {
@@ -599,12 +590,6 @@ private Context resolveParentContext(String parentId) {
if (parentSpanContext != null) {
return Context.current().with(Span.wrap(parentSpanContext));
}
- // Parent operation from a prior invocation — create non-recording placeholder
- var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(parentId);
- var traceId = idGenerator.generateTraceId();
- var placeholderContext = SpanContext.create(
- traceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault());
- return Context.current().with(Span.wrap(placeholderContext));
}
// Fall back to invocation span as parent
if (invocationSpan != null) {
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java
index 33fd6ea66..74c7fc1d3 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java
@@ -5,14 +5,20 @@
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer;
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
-/** Installs the durable-execution ID generator when the OpenTelemetry Java agent auto-configures the SDK. */
+/** Wraps the Java agent's configured ID generator with scoped durable-execution overrides. */
public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider {
- private static final DeterministicIdGenerator ID_GENERATOR = new DeterministicIdGenerator();
-
@Override
public void customize(AutoConfigurationCustomizer autoConfiguration) {
OtelPluginAutoConfigurationState.markInstalled();
- autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR));
+ autoConfiguration.addTracerProviderCustomizer((builder, config) -> {
+ DeterministicIdGenerator.installOn(builder);
+ return builder;
+ });
+ }
+
+ @Override
+ public int order() {
+ return Integer.MAX_VALUE;
}
}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
index 3cf021ad8..ed4ac9be5 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
@@ -2,8 +2,6 @@
// SPDX-License-Identifier: Apache-2.0
package software.amazon.lambda.durable.otel;
-import java.util.Map;
-
/**
* Immutable configuration for {@link InvocationOtelPlugin} and {@link ExecutionOtelPlugin}.
*
@@ -26,8 +24,8 @@
* }
*
* Defaults: {@code contextExtractor = new XRayContextExtractor()}, {@code enableMdc = true}, {@code workflowSpanName
- * = "Workflow"}, {@code instrumentationName = "aws-durable-execution-sdk-java"}, {@code providerSource =
- * ProviderSource.GLOBAL}. A {@code null} passed to any builder setter falls back to the corresponding default.
+ * = "Workflow"}, {@code instrumentationName = "aws-durable-execution-sdk-java"}. A {@code null} passed to any builder
+ * setter falls back to the corresponding default.
*/
public final class OtelPluginConfig {
@@ -38,9 +36,6 @@ public final class OtelPluginConfig {
private final boolean enableMdc;
private final String workflowSpanName;
private final String instrumentationName;
- private final ProviderSource providerSource;
- private final String otlpEndpoint;
- private final Map {@link ProviderSource#EXPLICIT} is not valid here — it is implied by using a {@code (SdkTracerProviderBuilder,
- * OtelPluginConfig)} constructor and is rejected by the config-only constructors.
- */
- public ProviderSource providerSource() {
- return providerSource;
- }
-
- /** OTLP/HTTP endpoint for the auto-configured provider, or {@code null} to use the OTel default / env var. */
- public String otlpEndpoint() {
- return otlpEndpoint;
- }
-
- /** Extra headers sent by the auto-configured OTLP exporter (never {@code null}). */
- public Map {@link ProviderSource#EXPLICIT} is not accepted through the config-only constructors — supply a
- * {@code SdkTracerProviderBuilder} via the two-arg constructor instead.
- *
- * @param providerSource the provider source, {@link ProviderSource#GLOBAL} or {@link ProviderSource#AUTO_OTLP}
- * @return this builder
- */
- public Builder providerSource(ProviderSource providerSource) {
- this.providerSource = providerSource != null ? providerSource : ProviderSource.GLOBAL;
- return this;
- }
-
- /**
- * Sets the OTLP/HTTP endpoint for the auto-configured provider. When null, the OTel default (or
- * {@code OTEL_EXPORTER_OTLP_ENDPOINT}) is used.
- *
- * @param otlpEndpoint the OTLP/HTTP traces endpoint
- * @return this builder
- */
- public Builder otlpEndpoint(String otlpEndpoint) {
- this.otlpEndpoint = otlpEndpoint;
- return this;
- }
-
- /**
- * Sets extra headers for the auto-configured OTLP exporter (e.g. auth headers for a third-party endpoint).
- *
- * @param otlpHeaders header name/value pairs; null is treated as empty
- * @return this builder
- */
- public Builder otlpHeaders(Map Mirrors the {@code ProviderSource} used by the JavaScript and Python SDK OTel plugins for cross-SDK parity:
- *
- * This is the single knob that selects a plugin's tracer provider. {@link OtelPluginConfig#providerSource()} carries
- * it for the config-only constructors; the {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructors always
- * report {@link #EXPLICIT}.
- */
-public enum ProviderSource {
- /** Caller-supplied {@code SdkTracerProviderBuilder}; plugin-owned. */
- EXPLICIT,
- /** Globally configured provider (ADOT Java agent); not plugin-owned. */
- GLOBAL,
- /** Auto-configured OTLP/HTTP provider; plugin-owned. */
- AUTO_OTLP
-}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
index 8a7fb367c..109f633de 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
@@ -9,9 +9,9 @@
/**
* Extracts OTel trace context from the AWS X-Ray {@code _X_AMZN_TRACE_ID} environment variable.
*
- * The durable execution backend propagates the same Root trace ID to every invocation of the same execution, so all
- * invocations share one trace. This extractor parses that header and returns the trace ID in OTel format (32 hex chars)
- * along with the parent span ID (16 hex chars).
+ * This extractor parses the Lambda/X-Ray header and returns the trace ID in OTel format (32 hex chars) along with
+ * the parent span ID (16 hex chars). Plugins use it as a fallback parent for Invocation spans; the deterministic
+ * Workflow trace is derived separately from the durable execution ARN.
*
* X-Ray header format: {@code Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1}
*
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java
index 2b9389fc7..caf09cc60 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java
@@ -4,12 +4,20 @@
import static org.junit.jupiter.api.Assertions.*;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.sdk.trace.IdGenerator;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import java.time.Instant;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executors;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
class DeterministicIdGeneratorTest {
+ private static final Instant EXECUTION_START_TIME = Instant.parse("2026-08-15T00:00:00Z");
+
private DeterministicIdGenerator generator;
@BeforeEach
@@ -34,6 +42,122 @@ void generateTraceId_withoutArn_returnsRandom() {
assertNotEquals(id1, id2);
}
+ @Test
+ void scopedIds_delegateOutsideScope() {
+ var providerGenerator = new DeterministicIdGenerator();
+ try (var provider =
+ SdkTracerProvider.builder().setIdGenerator(providerGenerator).build()) {
+ var pluginTracer = provider.get("durable-plugin");
+ var unrelatedTracer = provider.get("unrelated-library");
+
+ var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan();
+ var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME);
+ var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1");
+ var workflow = generator.startSpan(
+ pluginTracer.spanBuilder("Workflow").setNoParent(), workflowTraceId, workflowSpanId);
+ var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan();
+ var after = unrelatedTracer.spanBuilder("after").setNoParent().startSpan();
+
+ assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId());
+ assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId());
+ assertAllFreshRoots(
+ workflow.getSpanContext(),
+ before.getSpanContext(),
+ during.getSpanContext(),
+ after.getSpanContext());
+
+ before.end();
+ workflow.end();
+ during.end();
+ after.end();
+ }
+ }
+
+ @Test
+ void scopedIds_bridgeAcrossGeneratorInstances() {
+ var agentGenerator = new DeterministicIdGenerator();
+ try (var provider =
+ SdkTracerProvider.builder().setIdGenerator(agentGenerator).build()) {
+ var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME);
+ var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1");
+ var workflow = generator.startSpan(
+ provider.get("durable-plugin").spanBuilder("Workflow").setNoParent(),
+ workflowTraceId,
+ workflowSpanId);
+
+ assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId());
+ assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId());
+ assertNotEquals(workflowTraceId, agentGenerator.generateTraceId());
+ workflow.end();
+ }
+ }
+
+ @Test
+ void concurrentScopedIds_doNotOverwriteEachOther() throws Exception {
+ var agentGenerator = new DeterministicIdGenerator();
+ var executor = Executors.newFixedThreadPool(2);
+ try (var provider =
+ SdkTracerProvider.builder().setIdGenerator(agentGenerator).build()) {
+ var tracer = provider.get("durable-plugin");
+ var barrier = new CyclicBarrier(2);
+ var traceId1 = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME);
+ var traceId2 = generator.generateTraceIdForExecution("arn:exec2", EXECUTION_START_TIME);
+ var spanId1 = generator.generateWorkflowSpanId("arn:exec1");
+ var spanId2 = generator.generateWorkflowSpanId("arn:exec2");
+
+ var first = executor.submit(() -> {
+ barrier.await();
+ return scopedSpanContext(tracer, new DeterministicIdGenerator(), traceId1, spanId1);
+ });
+ var second = executor.submit(() -> {
+ barrier.await();
+ return scopedSpanContext(tracer, new DeterministicIdGenerator(), traceId2, spanId2);
+ });
+
+ assertEquals(traceId1, first.get().getTraceId());
+ assertEquals(spanId1, first.get().getSpanId());
+ assertEquals(traceId2, second.get().getTraceId());
+ assertEquals(spanId2, second.get().getSpanId());
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void installOn_preservesConfiguredFallbackGenerator() {
+ var fallbackTraceId = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+ var fallbackSpanId = "cccccccccccccccc";
+ var builder = SdkTracerProvider.builder().setIdGenerator(fixedIds(fallbackTraceId, fallbackSpanId));
+ var installedGenerator = DeterministicIdGenerator.installOn(builder);
+
+ try (var provider = builder.build()) {
+ var unrelated =
+ provider.get("unrelated").spanBuilder("root").setNoParent().startSpan();
+ var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME);
+ var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1");
+ var workflow = generator.startSpan(
+ provider.get("durable").spanBuilder("Workflow").setNoParent(), workflowTraceId, workflowSpanId);
+
+ assertEquals(fallbackTraceId, unrelated.getSpanContext().getTraceId());
+ assertEquals(fallbackSpanId, unrelated.getSpanContext().getSpanId());
+ assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId());
+ assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId());
+ assertSame(installedGenerator, DeterministicIdGenerator.installOn(builder));
+
+ unrelated.end();
+ workflow.end();
+ }
+ }
+
+ @Test
+ void executionTraceId_usesStartTimestampAndArn() {
+ var traceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME);
+
+ assertEquals("6a7fac00", traceId.substring(0, 8));
+ assertEquals(traceId, generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME));
+ assertNotEquals(traceId, generator.generateTraceIdForExecution("arn:exec2", EXECUTION_START_TIME));
+ }
+
@Test
void generateTraceId_withArn_returnsDeterministic() {
generator.setDurableExecutionArn("arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1");
@@ -128,27 +252,17 @@ void generateSpanIdForOperation_isDeterministic() {
}
@Test
- void generatedIds_areSharedAcrossGeneratorInstances() {
+ void persistentIds_areIsolatedAcrossGeneratorInstances() {
var pluginGenerator = new DeterministicIdGenerator();
- var agentGenerator = new DeterministicIdGenerator();
+ var fallbackTraceId = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+ var fallbackSpanId = "cccccccccccccccc";
+ var agentGenerator = new DeterministicIdGenerator(fixedIds(fallbackTraceId, fallbackSpanId));
pluginGenerator.setDurableExecutionArn("arn:exec1");
pluginGenerator.setNextSpanOperationId("op-1");
- assertEquals(pluginGenerator.generateTraceId(), agentGenerator.generateTraceId());
- assertEquals(pluginGenerator.generateSpanIdForOperation("op-1"), agentGenerator.generateSpanId());
- }
-
- @Test
- void rawSpanId_isSharedAcrossGeneratorInstances() {
- var pluginGenerator = new DeterministicIdGenerator();
- var agentGenerator = new DeterministicIdGenerator();
-
- pluginGenerator.setDurableExecutionArn("arn:exec1");
- var workflowSpanId = pluginGenerator.generateWorkflowSpanId();
- pluginGenerator.setNextSpanId(workflowSpanId);
-
- assertEquals(workflowSpanId, agentGenerator.generateSpanId());
+ assertEquals(fallbackTraceId, agentGenerator.generateTraceId());
+ assertEquals(fallbackSpanId, agentGenerator.generateSpanId());
}
@Test
@@ -285,4 +399,40 @@ void setExtractedTraceId_validXrayFormat_32HexChars() {
assertEquals(xrayTraceId, result);
assertTrue(result.matches("[0-9a-f]{32}"));
}
+
+ private static SpanContext scopedSpanContext(
+ io.opentelemetry.api.trace.Tracer tracer,
+ DeterministicIdGenerator pluginGenerator,
+ String traceId,
+ String spanId) {
+ var span = pluginGenerator.startSpan(tracer.spanBuilder("Workflow").setNoParent(), traceId, spanId);
+ var spanContext = span.getSpanContext();
+ span.end();
+ return spanContext;
+ }
+
+ private static void assertAllFreshRoots(SpanContext workflow, SpanContext... unrelated) {
+ for (var spanContext : unrelated) {
+ assertNotEquals(workflow.getTraceId(), spanContext.getTraceId());
+ }
+ for (var left = 0; left < unrelated.length; left++) {
+ for (var right = left + 1; right < unrelated.length; right++) {
+ assertNotEquals(unrelated[left].getTraceId(), unrelated[right].getTraceId());
+ }
+ }
+ }
+
+ private static IdGenerator fixedIds(String traceId, String spanId) {
+ return new IdGenerator() {
+ @Override
+ public String generateSpanId() {
+ return spanId;
+ }
+
+ @Override
+ public String generateTraceId() {
+ return traceId;
+ }
+ };
+ }
}
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
index eb9401722..0c2a1d3f1 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java
@@ -82,69 +82,75 @@ void customInstrumentationName_isUsedForTracerScope() {
}
@Test
- void configOnlyConstructor_defaultsToGlobalProvider() {
- OtelPluginAutoConfigurationState.markInstalled();
+ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
GlobalOpenTelemetry.resetForTest();
- OpenTelemetrySdk.builder()
- .setTracerProvider(SdkTracerProvider.builder().build())
- .buildAndRegisterGlobal();
-
- var plugin = new ExecutionOtelPlugin(OtelPluginConfig.defaults());
- assertEquals(ProviderSource.GLOBAL, plugin.providerSource());
- }
-
- @Test
- void configWithAutoOtlp_buildsPluginOwnedProvider() {
- var plugin = new ExecutionOtelPlugin(OtelPluginConfig.builder()
- .providerSource(ProviderSource.AUTO_OTLP)
- .build());
- assertEquals(ProviderSource.AUTO_OTLP, plugin.providerSource());
- }
+ OtelPluginAutoConfigurationState.markInstalled();
- @Test
- void builderConstructor_isExplicitSource() {
- var plugin = new ExecutionOtelPlugin(SdkTracerProvider.builder(), OtelPluginConfig.defaults());
- assertEquals(ProviderSource.EXPLICIT, plugin.providerSource());
- }
+ var defaultPlugin = new ExecutionOtelPlugin();
+ defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now()));
+ defaultPlugin.onOperationStart(new OperationInfo(
+ "op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
+ defaultPlugin.onOperationEnd(new OperationEndInfo(
+ "op-disabled",
+ "disabled-step",
+ "STEP",
+ "Step",
+ null,
+ Instant.now(),
+ Instant.now(),
+ "SUCCEEDED",
+ null,
+ false,
+ null,
+ null));
+ defaultPlugin.onInvocationEnd(
+ new InvocationEndInfo("req-disabled", "arn:disabled", true, InvocationStatus.SUCCEEDED, null));
- @Test
- void configProviderSource_defaultsToGlobalAndHonorsAutoOtlp() {
- assertEquals(ProviderSource.GLOBAL, OtelPluginConfig.defaults().providerSource());
- assertEquals(
- ProviderSource.AUTO_OTLP,
- OtelPluginConfig.builder()
- .providerSource(ProviderSource.AUTO_OTLP)
- .build()
- .providerSource());
- }
+ assertFalse(GlobalOpenTelemetry.isSet(), "An unavailable provider must not install the no-op global");
- @Test
- void configOnlyConstructor_rejectsExplicitProviderSource() {
- var config = OtelPluginConfig.builder()
- .providerSource(ProviderSource.EXPLICIT)
+ var globalExporter = InMemorySpanExporter.create();
+ var globalTracerProvider = SdkTracerProvider.builder()
+ .addSpanProcessor(SimpleSpanProcessor.create(globalExporter))
.build();
- var error = assertThrows(IllegalArgumentException.class, () -> new ExecutionOtelPlugin(config));
- assertTrue(error.getMessage().contains("SdkTracerProviderBuilder"));
- }
+ OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- @Test
- void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstalled() {
- GlobalOpenTelemetry.resetForTest();
- var error = assertThrows(IllegalStateException.class, ExecutionOtelPlugin::new);
- assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider"));
+ defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now()));
+ defaultPlugin.onOperationStart(new OperationInfo(
+ "op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
+ defaultPlugin.onOperationEnd(new OperationEndInfo(
+ "op-enabled",
+ "enabled-step",
+ "STEP",
+ "Step",
+ null,
+ Instant.now(),
+ Instant.now(),
+ "SUCCEEDED",
+ null,
+ false,
+ null,
+ null));
+ defaultPlugin.onInvocationEnd(
+ new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null));
+
+ var spans = globalExporter.getFinishedSpanItems();
+ assertEquals(3, spans.size());
+ assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step")));
+ assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step")));
}
@Test
void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
+ var defaultPlugin = new ExecutionOtelPlugin();
+ assertFalse(GlobalOpenTelemetry.isSet());
+
OtelPluginAutoConfigurationState.markInstalled();
- GlobalOpenTelemetry.resetForTest();
var globalExporter = InMemorySpanExporter.create();
var globalTracerProvider = SdkTracerProvider.builder()
.addSpanProcessor(SimpleSpanProcessor.create(globalExporter))
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- var defaultPlugin = new ExecutionOtelPlugin();
defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
defaultPlugin.onOperationStart(
new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false));
@@ -250,7 +256,8 @@ void workflowAndInvocationSpans_areIndependentRoots_withoutAmbientContext() {
assertFalse(workflowSpan.getParentSpanContext().isValid(), "Workflow span must be a root");
assertFalse(invocationSpan.getParentSpanContext().isValid(), "Invocation span must be a root");
- assertEquals(workflowSpan.getTraceId(), invocationSpan.getTraceId());
+ assertNotEquals(
+ workflowSpan.getTraceId(), invocationSpan.getTraceId(), "Independent roots must not share a trace ID");
assertEquals(SpanKind.INTERNAL, invocationSpan.getKind());
}
@@ -756,8 +763,9 @@ void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() {
// ─── Cross-invocation stitching ──────────────────────────────────────
@Test
- void allSpansShareTraceId_acrossInvocations() {
- plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now()));
+ void workflowTraceIsStableAndInvocationRootsAreFresh_acrossInvocations() {
+ var executionStartTime = Instant.parse("2026-08-15T00:00:00Z");
+ plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, executionStartTime));
plugin.onOperationStart(
new OperationInfo("op-1", "step-1", "STEP", "Step", null, Instant.now(), null, null, false));
plugin.onOperationEnd(new OperationEndInfo(
@@ -774,16 +782,21 @@ void allSpansShareTraceId_acrossInvocations() {
null,
null));
plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null));
- var firstTraceId = spanExporter.getFinishedSpanItems().get(0).getTraceId();
+ var firstSpans = spanExporter.getFinishedSpanItems();
+ var workflowTraceId = spanByName(firstSpans, "step-1").getTraceId();
+ var firstInvocationTraceId = spanByName(firstSpans, "Invocation").getTraceId();
spanExporter.reset();
- plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now()));
+ plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, executionStartTime));
plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null));
var secondSpans = spanExporter.getFinishedSpanItems();
+ var workflowSpan = spanByName(secondSpans, "Workflow");
+ var secondInvocationSpan = spanByName(secondSpans, "Invocation");
- assertTrue(
- secondSpans.stream().allMatch(s -> s.getTraceId().equals(firstTraceId)),
- "All spans of one execution must share the same trace ID");
+ assertEquals(workflowTraceId, workflowSpan.getTraceId());
+ assertNotEquals(workflowTraceId, firstInvocationTraceId);
+ assertNotEquals(workflowTraceId, secondInvocationSpan.getTraceId());
+ assertNotEquals(firstInvocationTraceId, secondInvocationSpan.getTraceId());
}
@Test
@@ -865,13 +878,14 @@ void sampling_disabled_producesNoSpans() {
// ─── X-Ray trace ID ──────────────────────────────────────────────────
@Test
- void xrayExtraction_allSpansShareExtractedTraceId() {
+ void xrayExtraction_keepsWorkflowTraceIndependent() {
var xrayTraceId = "aabbccddee112233445566778899aabb";
+ var parentSpanId = "53995c3f42cd8ad8";
var exporter = InMemorySpanExporter.create();
var xrayPlugin = new ExecutionOtelPlugin(
SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
OtelPluginConfig.builder()
- .contextExtractor(() -> new ExtractedContext(xrayTraceId, null))
+ .contextExtractor(() -> new ExtractedContext(xrayTraceId, parentSpanId))
.enableMdc(false)
.workflowSpanName("Workflow")
.build());
@@ -895,9 +909,13 @@ void xrayExtraction_allSpansShareExtractedTraceId() {
var spans = exporter.getFinishedSpanItems();
assertTrue(spans.size() >= 3, "Workflow + invocation + operation spans expected");
- assertTrue(
- spans.stream().allMatch(s -> s.getTraceId().equals(xrayTraceId)),
- "All spans must share the extracted X-Ray trace ID");
+ var workflowSpan = spanByName(spans, "Workflow");
+ var invocationSpan = spanByName(spans, "Invocation");
+ var operationSpan = spanByName(spans, "step-a");
+ assertEquals(xrayTraceId, invocationSpan.getTraceId());
+ assertEquals(parentSpanId, invocationSpan.getParentSpanId());
+ assertNotEquals(xrayTraceId, workflowSpan.getTraceId());
+ assertEquals(workflowSpan.getTraceId(), operationSpan.getTraceId());
}
@Test
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java
index 2266b5a94..274944c36 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java
@@ -71,17 +71,29 @@ void simpleStep_producesInvocationAndOperationAndAttemptSpans() {
var spans = spanExporter.getFinishedSpanItems();
- // Should have: invocation + operation (backfilled) + attempt = 3
- assertTrue(spans.size() >= 3, "Expected at least 3 spans, got " + spans.size());
+ // Should have: Workflow + invocation + operation + attempt = 4
+ assertTrue(spans.size() >= 4, "Expected at least 4 spans, got " + spans.size());
// Verify span names
+ assertSpanExists(spans, "Workflow");
assertSpanExists(spans, "Invocation");
assertSpanExists(spans, "greet");
assertSpanExists(spans, "greet attempt 1");
- // All spans share the same trace ID
- var traceId = spans.get(0).getTraceId();
- assertTrue(spans.stream().allMatch(s -> s.getTraceId().equals(traceId)));
+ var workflowTraceId = spans.stream()
+ .filter(span -> span.getName().equals("Workflow"))
+ .findFirst()
+ .orElseThrow()
+ .getTraceId();
+ var invocationTraceId = spans.stream()
+ .filter(span -> span.getName().equals("Invocation"))
+ .findFirst()
+ .orElseThrow()
+ .getTraceId();
+ assertNotEquals(workflowTraceId, invocationTraceId);
+ assertTrue(spans.stream()
+ .filter(span -> !span.getName().equals("Workflow"))
+ .allMatch(span -> span.getTraceId().equals(invocationTraceId)));
}
@Test
@@ -532,17 +544,18 @@ void waitForCondition_producesSpansWithAttempts() {
}
@Test
- void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
+ void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() {
+ var defaultPlugin = new InvocationOtelPlugin();
+ assertFalse(GlobalOpenTelemetry.isSet());
+
OtelPluginAutoConfigurationState.markInstalled();
- GlobalOpenTelemetry.resetForTest();
var globalExporter = InMemorySpanExporter.create();
var globalTracerProvider = SdkTracerProvider.builder()
.addSpanProcessor(SimpleSpanProcessor.create(globalExporter))
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- var defaultConfig =
- DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build();
+ var defaultConfig = DurableConfig.builder().withPlugins(defaultPlugin).build();
var runner = LocalDurableTestRunner.create(
String.class,
(input, ctx) -> ctx.step("global-step", String.class, stepCtx -> "Hello " + input),
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
index 859efc5bd..be68c9716 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java
@@ -24,6 +24,7 @@
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.IdGenerator;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
@@ -65,37 +66,75 @@ void tearDown() {
}
@Test
- void defaultConstructor_throwsWhenAutoConfigurationCustomizerProviderIsNotInstalled() {
+ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() {
GlobalOpenTelemetry.resetForTest();
+ OtelPluginAutoConfigurationState.markInstalled();
- var error = assertThrows(IllegalStateException.class, InvocationOtelPlugin::new);
+ var defaultPlugin = new InvocationOtelPlugin();
+ defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now()));
+ defaultPlugin.onOperationStart(new OperationInfo(
+ "op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
+ defaultPlugin.onOperationEnd(new OperationEndInfo(
+ "op-disabled",
+ "disabled-step",
+ "STEP",
+ "Step",
+ null,
+ Instant.now(),
+ Instant.now(),
+ "SUCCEEDED",
+ null,
+ false,
+ null,
+ null));
+ defaultPlugin.onInvocationEnd(
+ new InvocationEndInfo("req-disabled", "arn:disabled", true, InvocationStatus.SUCCEEDED, null));
- assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider"));
- assertTrue(error.getMessage().contains("OTEL_JAVAAGENT_EXTENSIONS"));
- }
+ assertFalse(GlobalOpenTelemetry.isSet(), "An unavailable provider must not install the no-op global");
- @Test
- void defaultConstructor_throwsWhenGlobalOpenTelemetryIsNotInitializedBySpi() {
- OtelPluginAutoConfigurationState.markInstalled();
- GlobalOpenTelemetry.resetForTest();
+ var globalExporter = InMemorySpanExporter.create();
+ var globalTracerProvider = SdkTracerProvider.builder()
+ .addSpanProcessor(SimpleSpanProcessor.create(globalExporter))
+ .build();
+ OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- var error = assertThrows(IllegalStateException.class, InvocationOtelPlugin::new);
+ defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now()));
+ defaultPlugin.onOperationStart(new OperationInfo(
+ "op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false));
+ defaultPlugin.onOperationEnd(new OperationEndInfo(
+ "op-enabled",
+ "enabled-step",
+ "STEP",
+ "Step",
+ null,
+ Instant.now(),
+ Instant.now(),
+ "SUCCEEDED",
+ null,
+ false,
+ null,
+ null));
+ defaultPlugin.onInvocationEnd(
+ new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null));
- assertTrue(error.getMessage().contains("GlobalOpenTelemetry"));
- assertTrue(error.getMessage().contains("OtelPluginAutoConfigurationCustomizerProvider"));
+ var spans = globalExporter.getFinishedSpanItems();
+ assertEquals(3, spans.size());
+ assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step")));
+ assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step")));
}
@Test
void defaultConstructor_usesGlobalSdkTracerProviderDirectly() {
+ var defaultPlugin = new InvocationOtelPlugin();
+ assertFalse(GlobalOpenTelemetry.isSet());
+
OtelPluginAutoConfigurationState.markInstalled();
- GlobalOpenTelemetry.resetForTest();
var globalExporter = InMemorySpanExporter.create();
var globalTracerProvider = SdkTracerProvider.builder()
.addSpanProcessor(SimpleSpanProcessor.create(globalExporter))
.build();
OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal();
- var defaultPlugin = new InvocationOtelPlugin();
defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now()));
defaultPlugin.onOperationStart(
new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false));
@@ -178,7 +217,7 @@ public ContextPropagators getPropagators() {
}
@Test
- void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator() {
+ void autoConfigurationCustomizerProvider_appliesOnlyScopedDeterministicIds() {
OtelPluginAutoConfigurationState.resetInstalledForTest();
var exporter = InMemorySpanExporter.create();
var autoConfiguration = mock(AutoConfigurationCustomizer.class);
@@ -192,26 +231,46 @@ void autoConfigurationCustomizerProvider_installsSharedDeterministicIdGenerator(
verify(autoConfiguration).addTracerProviderCustomizer(customizer.capture());
var pluginGenerator = new DeterministicIdGenerator();
- pluginGenerator.setDurableExecutionArn("arn:spi");
- pluginGenerator.setNextSpanOperationId("op-spi");
@SuppressWarnings("unchecked")
var tracerProviderCustomizer =
(BiFunction
- * Trace (single trace ID across both invocations)
- * ├── invocation (invocation 1)
+ * Workflow trace:
+ * Workflow
+ *
+ * Ambient invocation trace:
+ * ├── Invocation (invocation 1)
* │ ├── before-wait
* │ │ └── before-wait attempt 1
* │ └── pause (ended as PENDING)
- * └── invocation (invocation 2)
+ * └── Invocation (invocation 2)
* ├── pause (completed)
* └── after-wait
* └── after-wait attempt 1
diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java
index 8f87e71f5..023025a46 100644
--- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java
+++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedOtelIntegrationTest.java
@@ -7,8 +7,8 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.Duration;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -22,6 +22,7 @@
import software.amazon.awssdk.services.xray.model.GetTraceSummariesRequest;
import software.amazon.awssdk.services.xray.model.Segment;
import software.amazon.awssdk.services.xray.model.TimeRangeType;
+import software.amazon.awssdk.services.xray.model.Trace;
import software.amazon.awssdk.services.xray.model.TraceSummary;
import software.amazon.lambda.durable.examples.types.GreetingRequest;
import software.amazon.lambda.durable.model.ExecutionStatus;
@@ -41,10 +42,10 @@
*
- *
*
*
- *
- *
- *
*
*
- *
- *
+ *
- *
- *
- * @param config the plugin configuration
+ * @param instrumentationName the instrumentation scope name
* @param pluginName the plugin name used in diagnostics/flush logging
- * @return the resolved provider, tracer, ID generator, and source
- * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT}
+ * @return the resolved provider and tracer, or {@code null} when telemetry must be disabled for this invocation
*/
- static ProviderSetup resolveConfiguredProvider(OtelPluginConfig config, String pluginName) {
- return switch (config.providerSource()) {
- case GLOBAL -> {
- var idGenerator = createDefaultIdGenerator();
- var tracerProvider = getDefaultTracerProvider(pluginName);
- yield new ProviderSetup(
- ProviderSource.GLOBAL,
- getSdkTracerProviderForFlush(tracerProvider, pluginName),
- tracerProvider.get(config.instrumentationName()),
- idGenerator);
- }
- case AUTO_OTLP -> {
- var idGenerator = new DeterministicIdGenerator();
- var sdkTracerProvider = buildAutoOtlpProvider(config, idGenerator, null);
- yield new ProviderSetup(
- ProviderSource.AUTO_OTLP,
- sdkTracerProvider,
- sdkTracerProvider.get(config.instrumentationName()),
- idGenerator);
- }
- case EXPLICIT ->
- throw new IllegalArgumentException(
- "OtelPluginConfig.providerSource(EXPLICIT) requires a caller-supplied SdkTracerProviderBuilder; "
- + "use the (SdkTracerProviderBuilder, OtelPluginConfig) constructor.");
- };
- }
-
- /** Resolves the OTLP/HTTP traces endpoint (config -> env -> exporter default), appending the signal path. */
- private static String resolveOtlpEndpoint(OtelPluginConfig config) {
- if (config.otlpEndpoint() != null && !config.otlpEndpoint().isBlank()) {
- return config.otlpEndpoint();
+ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String pluginName) {
+ if (!OtelPluginAutoConfigurationState.isInstalled()) {
+ logger.warn(
+ "{} telemetry is disabled for this invocation because "
+ + "OtelPluginAutoConfigurationCustomizerProvider is not installed yet. Provider resolution "
+ + "will be retried on the next invocation. {}",
+ pluginName,
+ javaAgentExtensionsDiagnostic());
+ return null;
}
- var envEndpoint = System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT");
- if (envEndpoint != null && !envEndpoint.isBlank()) {
- var base = envEndpoint.endsWith("/") ? envEndpoint.substring(0, envEndpoint.length() - 1) : envEndpoint;
- return base.endsWith("/v1/traces") ? base : base + "/v1/traces";
+ if (!GlobalOpenTelemetry.isSet()) {
+ logger.warn(
+ "{} telemetry is disabled for this invocation because GlobalOpenTelemetry is not initialized yet. "
+ + "Provider resolution will be retried on the next invocation.",
+ pluginName);
+ return null;
}
- // null -> the OTLP/HTTP exporter's own default (http://localhost:4318/v1/traces)
- return null;
- }
- /** Builds the sampler from {@code OTEL_DURABLE_SAMPLING_RATIO}, falling back to always-on. */
- private static Sampler resolveSampler() {
- var raw = System.getenv("OTEL_DURABLE_SAMPLING_RATIO");
- if (raw != null) {
- try {
- var ratio = Double.parseDouble(raw);
- if (ratio >= 0.0 && ratio <= 1.0) {
- return Sampler.traceIdRatioBased(ratio);
- }
- } catch (NumberFormatException ignored) {
- // fall through to always-on
- }
+ var tracerProvider = GlobalOpenTelemetry.getOrNoop().getTracerProvider();
+ if (tracerProvider == TracerProvider.noop()) {
+ logger.warn(
+ "{} telemetry is disabled for this invocation because GlobalOpenTelemetry contains a no-op tracer "
+ + "provider. Provider resolution will be retried on the next invocation.",
+ pluginName);
+ return null;
}
- return Sampler.alwaysOn();
- }
- /** Builds Lambda resource attributes from AWS_* env vars, merged onto the default resource. */
- private static Resource buildLambdaResource() {
- var functionName = System.getenv("AWS_LAMBDA_FUNCTION_NAME");
- if (functionName == null || functionName.isBlank()) {
- return Resource.getDefault();
- }
- var attributes = Attributes.builder()
- .put(ServiceAttributes.SERVICE_NAME, functionName)
- .put("faas.name", functionName)
- .put("cloud.provider", "aws")
- .put("cloud.platform", "aws_lambda");
- var region = System.getenv("AWS_REGION");
- if (region != null && !region.isBlank()) {
- attributes.put("cloud.region", region);
- }
- var version = System.getenv("AWS_LAMBDA_FUNCTION_VERSION");
- if (version != null && !version.isBlank()) {
- attributes.put("faas.version", version);
- }
- return Resource.getDefault().merge(Resource.create(attributes.build()));
+ logger.info(
+ "{} initialized from existing GlobalOpenTelemetry tracer provider {}; assuming "
+ + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider",
+ pluginName,
+ tracerProvider.getClass().getName());
+ return new ProviderSetup(
+ getSdkTracerProviderForFlush(tracerProvider, pluginName), tracerProvider.get(instrumentationName));
}
/** Extracts trace context from the current OTel span (fallback when X-Ray header is unavailable). */
@@ -212,18 +94,6 @@ static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvi
return null;
}
- private static void validateAutoConfigurationCustomizerProviderInstalled(String pluginName) {
- if (OtelPluginAutoConfigurationState.isInstalled()) {
- return;
- }
- throw new IllegalStateException(
- pluginName + "() requires OtelPluginAutoConfigurationCustomizerProvider to be installed by the "
- + "OpenTelemetry Java agent. Package this plugin jar as an agent extension and set "
- + "OTEL_JAVAAGENT_EXTENSIONS or -Dotel.javaagent.extensions to that jar before constructing "
- + pluginName + "(). "
- + javaAgentExtensionsDiagnostic());
- }
-
private static String javaAgentExtensionsDiagnostic() {
var propertyValue = System.getProperty("otel.javaagent.extensions");
var environmentValue = System.getenv("OTEL_JAVAAGENT_EXTENSIONS");
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java
deleted file mode 100644
index 5081c45c5..000000000
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-/**
- * Which of the three resolution tiers produced a plugin's tracer provider.
- *
- *
- *
- *
- *