test: cover OTel replay and user callback context in Java

.github/scripts/verify_otel_api_compatibility.py

@@ -142,11 +142,14 @@ def run_matrix(args: argparse.Namespace) -> int:
             for view in ("otel-invocation", "otel-execution"):
                 name = f"{label}-api{version}-{view}"
                 negative = label == "old-old" and version == "1.49.0"
+                incompatible_core = label == "old-new"
                 case: dict[str, object] = {"name": name, "expected_negative_control": negative,
+                                          "expected_core_rejection": incompatible_core,
                                           "api": jar_facts(api), "context": jar_facts(context)}
                 command = [args.java, "-cp", os.pathsep.join(map(str, [classes, core, plugin, *dependencies])),
                            PROBE, str(core), str(plugin), str(api), str(context), view,
-                           str(negative).lower(), str(version == "1.66.0").lower()]
+                           str(negative).lower(), str(version == "1.66.0").lower(),
+                           str(incompatible_core).lower()]
                 try:
                     log = output / f"{name}.log"
                     execute(command, log, env=probe_environment(view), timeout=90)
@@ -155,6 +158,8 @@ def run_matrix(args: argparse.Namespace) -> int:
                         raise RuntimeError("Probe did not report successful completion")
                     if negative and "NEGATIVE_CONTROL_REPRODUCED" not in contents:
                         raise RuntimeError("Released negative control did not reproduce the reported failure")
+                    if incompatible_core and "CORE_LIFECYCLE_REJECTION_CONFIRMED" not in contents:
+                        raise RuntimeError("Old core must reject the new plugin before invocation startup")
                     case["passed"] = True
                 except RuntimeError as error:
                     failures += 1

.github/workflows/otel-conformance-tests.yml

@@ -49,6 +49,7 @@ on:
       conformance_test_ref:
         description: Optional conformance test commit SHA or branch name
         required: false
+        default: "75987d46a915bc37409eed3ea9c3617a924c9756"
         type: string
 
 permissions: {}
@@ -68,14 +69,14 @@ jobs:
       actions: write
       contents: read
       id-token: write
-    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@a66037abbbfa55fde97f714e30f0bc262edefd63
+    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@bdb4f1cd0f9252c1aaa978bb8b341b71f2b9d9dc
     with:
       runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }}
       language: java
       resource_prefix: j
       sdk_repository: aws/aws-durable-execution-sdk-java
       sdk_ref: ${{ github.event.pull_request.head.sha || github.sha }}
-      conformance_test_ref: ${{ inputs.conformance_test_ref || '02d6dca971a38c13d94d6233d12f687e55b2a572' }}
+      conformance_test_ref: ${{ inputs.conformance_test_ref || '75987d46a915bc37409eed3ea9c3617a924c9756' }}
       checkout_sdk: true
       # Build the handlers from this repo's checked-out module instead of the conformance repo's
       # bundled examples/java. Path is relative to the conformance workspace where the SDK is

conformance-tests-otel/README.md

@@ -0,0 +1,74 @@
+# Java OpenTelemetry conformance handlers
+
+The shared conformance repository owns the requirements and validators. This
+module supplies public-API handlers and SAM resources for both tracing views.
+Existing cases 1–20 and their resources are unchanged by the additions below.
+
+| Case | Scenario | Behavior |
+| --- | --- | --- |
+| 21 | `completed-step-replay` | Complete a step, suspend on a one-second durable wait, then complete another step. The first step body is skipped on replay. |
+| 22 | `user-function-context` | Probe handler entry/restoration/resume, step, child, concurrent parallel branches and map iterations, and their nested steps. |
+| 23 | `callback-function-context` | Probe step retry attempts, condition checks, callback submission, a wrapped retry helper's body and strategy, and a virtual child after the asynchronous work finishes. |
+| 24 | `invocation-retry-status` | Throw the public retryable execution exception after a checkpointed step, then recover on replay. |
+| 25 | `wait-for-callback-failure` | Reuse case 17’s public handler; the driver fails its callback without Error only after invocation completion. The shared validator checks the observed empty error payload and leaf status. |
+| 26 | `external-callback-completion-replay` | Observe a root callback completion, checkpoint its result, then revisit it through two later callback barriers without duplicate terminal export. |
+
+Cases 22 and 23 require a valid active `SpanContext` and create/end ordinary
+`conformance.<label>` user spans with `conformance.callback=<label>`. They do not
+set parents, attach contexts, copy the execution ARN onto user spans, or create
+SDK spans. The shared validator determines the canonical trace from SDK spans
+and checks the actual user-span parents.
+
+Root handler probes preserve a valid same-trace ambient Lambda parent. Otherwise
+the plugin activates its canonical Invocation/Workflow context on the handler
+thread and restores it in the same-thread end hook. The plugin requires the
+core's 2.2.2 lifecycle capability; older cores are rejected during configuration.
+Nested callbacks retain their named context or attempt parents. Execution-view
+context placeholders can be valid without recording, so probes do not require
+`isRecording()`.
+
+Case 23 uses the public step attempt number and checkpointed condition state;
+it has no mutable warm-container markers. Its helper and virtual child run
+after callback completion, so normal suspension/replay does not repeat those
+probes. Only the intentional helper failure is caught. The wrapped helper's
+strategy executes inside its established context lifecycle; this does not
+establish a parent contract for every other policy callback.
+
+Serialization, map item naming, completion policies, standalone step retry or
+condition wait strategies, and callbacks invoked before tracing setup are not
+claimed to share the stable user-function hook boundary. Plugin hooks,
+extractors, and samplers cannot be required to have made their future SDK span
+current. Arbitrary user-created threads are also outside these cases.
+
+Case 24 captures `isReplaying()` at handler entry before consuming the completed
+step. Java's public `UnrecoverableDurableExecutionException(ErrorObject, true)`
+requests invocation retry; the handler does not inject plugin hooks or
+checkpoint messages. Its expected invocation statuses are `RETRYING` then
+`SUCCEEDED`. Recovery may repeat telemetry delivery, so this case imposes no
+exact operation-export count.
+
+The module builds against the SDK selected by `JAVA_SDK_VERSION` or the
+`durable.sdk.version` Maven property. Run matched shared requirements when
+validating new cases; a workflow pinned to an older requirement set does not
+validate the newly added cases. The configured OTel conformance workflow owns
+deployment and backend validation.
+
+Case 26 creates `otel-external-target` directly on the root context and reads its
+stored result on every replay. The `otel-external-target-observed` step returns
+that result unchanged and runs only on the first completion. Two later
+`waitForCallback` barriers force two more handler replays. The shared driver sends
+each callback success only after its creating invocation has completed, producing
+four invocations and the final result `target/one/two`. Lossless S3 telemetry must
+show one terminal export for each callback; the first target completion precedes
+the observation-step attempt. This is SDK cloud behavior coverage, not a claim
+that a deployed Lambda exercises the JS local-runner wrapper.
+
+Cases 25–26 use matched shared requirements and workflow pin
+`bdb4f1cd0f9252c1aaa978bb8b341b71f2b9d9dc`. The shared cloud checks establish the
+service error-payload representation and callback phase gating; local runner
+checks do not establish those service facts.
+
+For case 25, Java reports no plugin error details for an entirely empty error
+container. The persisted container, callback failure, and caller error stay
+unchanged. Any present type, message, data, or stack-trace field remains an error
+detail, including an empty string or an explicitly supplied empty stack list.

conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel21CompletedStepReplay.java

@@ -0,0 +1,20 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.conformance.otel;
+
+import java.time.Duration;
+import java.util.Map;
+import software.amazon.lambda.durable.DurableContext;
+
+/** Completed-step replay scenario for OTel requirement 21 in both views. */
+public final class Otel21CompletedStepReplay extends OtelConformanceHandler<String> {
+
+    @Override
+    public String handleRequest(Map<String, Object> event, DurableContext context) {
+        requireScenario(event, "completed-step-replay");
+        var before = context.step("otel-before-wait", String.class, step -> "before");
+        context.wait("otel-replay-wait", Duration.ofSeconds(1));
+        var after = context.step("otel-after-wait", String.class, step -> "after");
+        return before + "-" + after;
+    }
+}

conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel22UserFunctionContext.java

@@ -0,0 +1,96 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.conformance.otel;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.trace.Span;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.config.MapConfig;
+import software.amazon.lambda.durable.config.ParallelConfig;
+
+/** Active user-function context scenario for OTel requirement 22 in both views. */
+public final class Otel22UserFunctionContext extends OtelConformanceHandler<String> {
+
+    @Override
+    public String handleRequest(Map<String, Object> event, DurableContext context) {
+        requireScenario(event, "user-function-context");
+        probe("handler");
+        context.step("otel-context-step", String.class, step -> {
+            probe("step");
+            return "step";
+        });
+        runChild(context);
+        runParallel(context);
+        runMap(context);
+        probe("handler-restored");
+        context.wait("otel-context-resume", Duration.ofSeconds(1));
+        probe("handler-after-resume");
+        return "context-complete";
+    }
+
+    private static void runChild(DurableContext context) {
+        context.runInChildContext("otel-context-child", String.class, child -> {
+            probe("child");
+            var result = child.step("otel-context-child-step", String.class, step -> {
+                probe("child-step");
+                return "child";
+            });
+            probe("child-restored");
+            return result;
+        });
+    }
+
+    private static void runMap(DurableContext context) {
+        context.map(
+                "otel-context-map",
+                List.of(0, 1),
+                Integer.class,
+                (item, index, iteration) -> {
+                    probe("map-" + index);
+                    return iteration.step("otel-context-map-step-" + index, Integer.class, step -> {
+                        probe("map-step-" + index);
+                        return item;
+                    });
+                },
+                MapConfig.builder()
+                        .maxConcurrency(2)
+                        .itemNamer((item, index) -> "otel-context-iteration-" + index)
+                        .build());
+    }
+
+    private static void runParallel(DurableContext context) {
+        var parallel = context.parallel(
+                "otel-context-parallel",
+                ParallelConfig.builder().maxConcurrency(2).build());
+        try (parallel) {
+            parallel.branch("otel-context-branch-a", String.class, branch -> {
+                probe("parallel-a");
+                return branch.step("otel-context-branch-step-a", String.class, step -> {
+                    probe("parallel-step-a");
+                    return "a";
+                });
+            });
+            parallel.branch("otel-context-branch-b", String.class, branch -> {
+                probe("parallel-b");
+                return branch.step("otel-context-branch-step-b", String.class, step -> {
+                    probe("parallel-step-b");
+                    return "b";
+                });
+            });
+        }
+    }
+
+    private static void probe(String label) {
+        if (!Span.current().getSpanContext().isValid()) {
+            throw new IllegalStateException("No active span context for " + label);
+        }
+        var span = GlobalOpenTelemetry.getTracer("durable-conformance")
+                .spanBuilder("conformance." + label)
+                .setAttribute("conformance.callback", label)
+                .startSpan();
+        span.end();
+    }
+}

conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel23CallbackFunctionContext.java

@@ -0,0 +1,109 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.conformance.otel;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.trace.Span;
+import java.time.Duration;
+import java.util.Map;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.config.RunInChildContextConfig;
+import software.amazon.lambda.durable.config.StepConfig;
+import software.amazon.lambda.durable.config.WaitForConditionConfig;
+import software.amazon.lambda.durable.config.WithRetryConfig;
+import software.amazon.lambda.durable.exception.DurableOperationException;
+import software.amazon.lambda.durable.model.WaitForConditionResult;
+import software.amazon.lambda.durable.retry.RetryDecision;
+import software.amazon.lambda.durable.retry.RetryStrategies;
+import software.amazon.lambda.durable.retry.WaitStrategies;
+
+/** Retry, polling, submitter, and helper callback contexts for OTel requirement 23. */
+public final class Otel23CallbackFunctionContext extends OtelConformanceHandler<String> {
+    private static final String HELPER_FAILURE = "intentional-helper-failure";
+
+    @Override
+    public String handleRequest(Map<String, Object> event, DurableContext context) {
+        requireScenario(event, "callback-function-context");
+        runRetryStep(context);
+        runCondition(context);
+        context.waitForCallback(
+                "otel-context-callback", String.class, (callbackId, step) -> probe("callback-submitter"));
+        runFailedHelper(context);
+        context.runInChildContext(
+                "otel-context-virtual",
+                String.class,
+                child -> {
+                    probe("virtual-child");
+                    return "virtual";
+                },
+                RunInChildContextConfig.builder().isVirtual(true).build());
+        return "callback-context-complete";
+    }
+
+    private static void runRetryStep(DurableContext context) {
+        context.step(
+                "otel-context-retry-step",
+                String.class,
+                step -> {
+                    var attempt = step.getAttempt();
+                    probe("retry-attempt-" + attempt);
+                    if (attempt == 1) throw new IllegalStateException("intentional-step-retry");
+                    return "retried";
+                },
+                StepConfig.builder()
+                        .retryStrategy(RetryStrategies.fixedDelay(2, Duration.ofSeconds(1)))
+                        .build());
+    }
+
+    private static void runCondition(DurableContext context) {
+        context.waitForCondition(
+                "otel-context-condition",
+                Integer.class,
+                (state, step) -> {
+                    var next = state + 1;
+                    probe("condition-check-" + next);
+                    return next == 2
+                            ? WaitForConditionResult.stopPolling(next)
+                            : WaitForConditionResult.continuePolling(next);
+                },
+                WaitForConditionConfig.<Integer>builder()
+                        .initialState(0)
+                        .waitStrategy(WaitStrategies.fixedDelay(2, Duration.ofSeconds(1)))
+                        .build());
+    }
+
+    private static void runFailedHelper(DurableContext context) {
+        try {
+            context.withRetry(
+                    "otel-context-with-retry",
+                    (attempt, child) -> {
+                        probe("with-retry-body");
+                        throw new IllegalStateException(HELPER_FAILURE);
+                    },
+                    WithRetryConfig.builder()
+                            .wrapInChildContext(true)
+                            .retryStrategy((error, attempt) -> {
+                                probe("with-retry-strategy");
+                                return RetryDecision.fail();
+                            })
+                            .build());
+        } catch (RuntimeException error) {
+            var message =
+                    error instanceof DurableOperationException operationError && operationError.getErrorObject() != null
+                            ? operationError.getErrorObject().errorMessage()
+                            : error.getMessage();
+            if (!HELPER_FAILURE.equals(message)) throw error;
+        }
+    }
+
+    private static void probe(String label) {
+        if (!Span.current().getSpanContext().isValid()) {
+            throw new IllegalStateException("No active span context for " + label);
+        }
+        var span = GlobalOpenTelemetry.getTracer("durable-conformance")
+                .spanBuilder("conformance." + label)
+                .setAttribute("conformance.callback", label)
+                .startSpan();
+        span.end();
+    }
+}

conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel24InvocationRetryStatus.java

@@ -0,0 +1,28 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.conformance.otel;
+
+import java.util.Map;
+import software.amazon.awssdk.services.lambda.model.ErrorObject;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
+
+/** Public invocation-retry signal and checkpoint recovery for OTel requirement 24. */
+public final class Otel24InvocationRetryStatus extends OtelConformanceHandler<String> {
+
+    @Override
+    public String handleRequest(Map<String, Object> event, DurableContext context) {
+        requireScenario(event, "invocation-retry-status");
+        var replayingAtEntry = context.isReplaying();
+        context.step("otel-before-invocation-retry", String.class, step -> "saved");
+        if (!replayingAtEntry) {
+            throw new UnrecoverableDurableExecutionException(
+                    ErrorObject.builder()
+                            .errorType("ConformanceInvocationRetry")
+                            .errorMessage("intentional-invocation-retry")
+                            .build(),
+                    true);
+        }
+        return "retry-complete";
+    }
+}