test: cover OTel replay and status mappings

.github/scripts/install_otel_test_wheels.py

@@ -16,6 +16,7 @@
 ROOT = Path(__file__).resolve().parents[2]
 CORE = "aws-durable-execution-sdk-python"
 OTEL = CORE + "-otel"
+TESTING = CORE + "-testing"
 
 
 def built_wheel(package: str) -> Path:
@@ -72,7 +73,9 @@ def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--legacy-plugin", action="store_true")
     args = parser.parse_args()
-    packages = [CORE] if args.legacy_plugin else [CORE, OTEL]
+    # This PR also repairs the local service simulator. Validate its actual
+    # built artifact with the SDK/plugin, retaining every test assertion.
+    packages = [CORE] if args.legacy_plugin else [CORE, OTEL, TESTING]
     wheels = [built_wheel(package) for package in packages]
     subprocess.run(
         [

.github/scripts/tests/test_opentelemetry_conformance_workflow.py

@@ -1,8 +1,12 @@
+import re
 from pathlib import Path
 
 import yaml
 
 
+SHARED_WORKFLOW_REF = "bdb4f1cd0f9252c1aaa978bb8b341b71f2b9d9dc"
+CONFORMANCE_TEST_REF = "75987d46a915bc37409eed3ea9c3617a924c9756"
+
 WORKFLOW_PATH = (
     Path(__file__).parents[2] / "workflows" / "opentelemetry-conformance-tests.yml"
 )
@@ -18,7 +22,10 @@ def test_opentelemetry_conformance_caller_uses_current_workflow_contract() -> No
         "uses: aws/aws-durable-execution-conformance-tests/.github/workflows/"
         "opentelemetry-orchestrator.yml@"
     )
-    assert orchestrator in workflow
+    pinned_ref = re.search(re.escape(orchestrator) + r"([0-9a-f]{40})", workflow)
+    assert pinned_ref is not None
+    assert pinned_ref.group(1) == SHARED_WORKFLOW_REF
+    assert f"default: {CONFORMANCE_TEST_REF}" in workflow
     assert "python-opentelemetry.yml@" not in workflow
     assert "\n      otlp_endpoint:" not in workflow
 
@@ -28,7 +35,7 @@ def test_opentelemetry_conformance_caller_uses_current_workflow_contract() -> No
         "resource_prefix: p",
         "sdk_repository: aws/aws-durable-execution-sdk-python",
         "sdk_ref: ${{ github.event.pull_request.head.sha || github.sha }}",
-        "conformance_test_ref: ${{ inputs.conformance_test_ref || 'main' }}",
+        f"conformance_test_ref: ${{{{ inputs.conformance_test_ref || '{CONFORMANCE_TEST_REF}' }}}}",
         "checkout_sdk: true",
         f"examples_dir: {EXAMPLES_DIR}",
         "adot_release_repository: aws-observability/aws-otel-python-instrumentation",

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
     branches: [ main ]
 
   pull_request:
-    branches: [ main ]
+    branches: ["main"]
 
 jobs:
   lint-commits:

.github/workflows/opentelemetry-conformance-tests.yml

@@ -45,7 +45,7 @@ on:
       conformance_test_ref:
         description: Conformance test commit SHA or branch name
         required: true
-        default: main
+        default: 75987d46a915bc37409eed3ea9c3617a924c9756
         type: string
 
 # Backend stacks are shared across PRs. Queue whole runs so reusable
@@ -66,14 +66,14 @@ jobs:
       actions: write
       contents: read
       id-token: write
-    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@f18bd0b5f28c5c90e288d0fb8bca08a849b51863
+    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@bdb4f1cd0f9252c1aaa978bb8b341b71f2b9d9dc
     with:
       language: python
       runs_on: codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }}
       resource_prefix: p
       sdk_repository: aws/aws-durable-execution-sdk-python
       sdk_ref: ${{ github.event.pull_request.head.sha || github.sha }}
-      conformance_test_ref: ${{ inputs.conformance_test_ref || 'main' }}
+      conformance_test_ref: ${{ inputs.conformance_test_ref || '75987d46a915bc37409eed3ea9c3617a924c9756' }}
       # Check the SDK out so the handlers and templates below are on disk. The handlers
       # themselves are installed from sdk_ref by src/requirements.txt during the SAM build.
       checkout_sdk: true

CONTRIBUTING.md

@@ -72,11 +72,11 @@ hatch run dev-examples:test    # run examples tests only
 
 ### Installed package compatibility testing
 
-Build the core and OTel distributions with `hatch build` in each package, then run
+Build the core, OTel and testing-library distributions with `hatch build` in each package, then run
 these commands from the repository root:
 
 ```bash
-hatch run test-wheel-otel:test        # full OTel suite on the two built wheels
+hatch run test-wheel-otel:test        # full OTel suite on the three built wheels
 hatch run test-wheel-otel-legacy:test # released OTel 1.0.0 with the built core
 hatch run test-pypi-examples:test     # examples against the published core
 ```

packages/aws-durable-execution-sdk-python-conformance-tests-otel/README.md

@@ -33,7 +33,7 @@ template-long-running.yaml  # otel-long-running suite
 tests/                      # contract tests for the templates and handlers
 ```
 
-The 20 invocation and 20 execution requirements reuse the same scenario
+The invocation and execution requirements reuse the same scenario
 handlers; the view is selected per function through the `OTEL_PLUGIN_MODE`
 environment variable, which `common.otel_plugin()` reads to pick
 `InvocationOtelPlugin` or `ExecutionOtelPlugin`. `template.yaml` deploys only the
@@ -63,6 +63,12 @@ view named by its `OtelSuite` parameter.
 | `otel-invocation-18` | `otel_18_chained_invoke_failure.handler` | Verifies failed chained-invoke telemetry. |
 | `otel-invocation-19` | `otel_19_execution_failure.handler` | Verifies telemetry for a direct handler failure. |
 | `otel-invocation-20` | `otel_20_virtual_context.handler` | Verifies a virtual child-context span without context checkpoints. |
+| `otel-invocation-21` | `otel_21_completed_step_replay.handler` | Replays a completed step after a successful wait/resume; each step body runs once. |
+| `otel-invocation-22` | `otel_22_user_function_context.handler` | Creates ordinary user spans under the active handler, attempt, child, branch, and iteration contexts. |
+| `otel-invocation-23` | `otel_23_callback_function_context.handler` | Verifies retry/check attempts, callback submitter, wrapped retry helper, and virtual-child callback parents. |
+| `otel-invocation-24` | `otel_24_invocation_retry_status.handler` | Raises a retryable invocation error after a completed step, then resumes with its saved result. |
+| `otel-invocation-25` | `otel_17_wait_for_callback_failure.handler` | Targets a failed callback without error details; service history must satisfy the explicit no-error-details precondition. |
+| `otel-invocation-26` | `otel_26_external_callback_completion_replay.handler` | Completes a root callback after suspension, saves its result in a step, and replays it through two callback barriers. |
 | `otel-execution-1` | `otel_1_success.handler` | Verifies the execution-view workflow, step, and attempt hierarchy. |
 | `otel-execution-2` | `otel_2_wait_resume.handler` | Verifies the execution view across a resumed invocation. |
 | `otel-execution-3` | `otel_3_retry.handler` | Verifies the execution view across retry attempts. |
@@ -83,6 +89,12 @@ view named by its `OtelSuite` parameter.
 | `otel-execution-18` | `otel_18_chained_invoke_failure.handler` | Verifies source and target failed workflow roots. |
 | `otel-execution-19` | `otel_19_execution_failure.handler` | Verifies a failed invocation without a completed workflow. |
 | `otel-execution-20` | `otel_20_virtual_context.handler` | Verifies a virtual child-context span under the workflow root. |
+| `otel-execution-21` | `otel_21_completed_step_replay.handler` | Verifies completed-operation spans are exported once across normal successful replay. |
+| `otel-execution-22` | `otel_22_user_function_context.handler` | Observes active execution-view callback contexts without supplying or repairing parents. |
+| `otel-execution-23` | `otel_23_callback_function_context.handler` | Verifies the same SDK-owned callback lifecycle parents in the execution view. |
+| `otel-execution-24` | `otel_24_invocation_retry_status.handler` | Verifies invocation retry status independently of step retry and recovery re-exports. |
+| `otel-execution-25` | `otel_17_wait_for_callback_failure.handler` | Targets the same errorless failed callback and its `UNSET` leaf in the execution view. |
+| `otel-execution-26` | `otel_26_external_callback_completion_replay.handler` | Requires one terminal root-callback export at first completion and no duplicate exports on two later replays. |
 | `otel-long-running-1` | `otel_long_running_1_wait.handler` | Verifies wait and resume telemetry across a long durable suspension. |
 | `otel-long-running-2` | `otel_long_running_2_retry.handler` | Verifies retry telemetry across a long durable backoff. |
 | `otel-long-running-3` | `otel_long_running_3_callback.handler` | Verifies callback telemetry when completion arrives after a long delay. |
@@ -91,6 +103,55 @@ view named by its `OtelSuite` parameter.
 The runner discovers each mapping from `TestingMetadata.TestDescription` on the
 functions in the templates.
 
+## External completion and status coverage
+
+The local testing library includes callback success, failure and timeout in the
+next invocation's `UpdatedOperationIds`. The core delivers each terminal update
+notification once per invocation, including when a resumed operation and a later
+checkpoint response carry the same completion. It tracks actual notifications,
+preserves first delivery when the update-ID metadata is absent, and clears that
+tracking at invocation boundaries. Public runner regressions cover memory and
+file stores, stored step results, failure payloads and two subsequent replays.
+An exactly empty serialized callback error is represented as absent in both
+stores, matching the SDK's service parser. Present fields remain intact,
+including empty messages, types, data and stack lists; the enclosing failed
+callback future still raises the same error as the file-store baseline.
+Detailed callback-failure history retains the service's empty error `Payload`
+object with `Truncated: false`, while SDK-facing state still has no error details.
+Metadata-only history and nonempty errors retain their existing representation.
+
+Case 24 covers invocation `RETRY` becoming `RETRYING`/`UNSET`. Case 25 targets the
+separate rule that a failed operation without error details remains `UNSET`.
+Its cloud coverage requires both views to pass the raw service-history no-error-details
+precondition and the telemetry assertions. Local file-store results establish
+SDK behavior; that AWS service precondition still needs independent validation.
+`CANCELLED`, `TIMED_OUT` and `STOPPED` without error details have explicit unit
+coverage in both views; their corresponding cloud paths are not established.
+The existing success/`OK` and detailed-error/`ERROR` controls remain in the suites.
+
+Case 26 revisits a root-level public callback on every replay. The driver waits
+for `InvocationCompleted` before completing the target and each barrier. The
+target's terminal span must precede the observed step and must not be exported
+again during the two later resumes. Invocation view also retains its initial
+pending callback segment. Cloud validation must count the raw S3 export records
+without deduplicating equal span IDs; local runner checks alone do not establish
+cloud coverage.
+
+## Callback coverage boundary
+
+Cases 22 and 23 create normal user spans from the active context, without an
+explicit parent or a copied execution ARN. They cover handler context, step and
+condition attempts, child and branch bodies, callback submission, wrapped
+`with_retry` body/strategy callbacks, and a virtual child. Case 23 places the
+retry helper and virtual child after callback completion so ordinary successful
+replay does not repeat their probes.
+
+This does not promise an operation/attempt parent for every arbitrary callback.
+General retry/wait policies, serializers, summary generators, and item naming
+have phase-specific caller scopes outside the wrapped user-function lifecycle;
+these cases do not impose a new ownership policy on them. Instrumentation
+extensions and user-created threads are outside this business-callback contract.
+
 ## How a handler maps to a requirement
 
 ```yaml

packages/aws-durable-execution-sdk-python-conformance-tests-otel/src/Makefile

@@ -5,6 +5,8 @@
 	build-Otel14MapFailure build-Otel15WaitInterrupted build-Otel16WaitForConditionFailure \
 	build-Otel17WaitForCallbackFailure build-Otel18ChainedInvokeFailure \
 	build-Otel18InvokeTarget build-Otel19ExecutionFailure build-Otel20VirtualContext \
+	build-Otel21CompletedStepReplay build-Otel22UserFunctionContext build-Otel23CallbackFunctionContext build-Otel24InvocationRetryStatus \
+	build-Otel25CallbackFailureWithoutError build-Otel26ExternalCallbackReplay \
 	build-OtelExecution1Success \
 	build-OtelExecution2WaitResume build-OtelExecution3Retry \
 	build-OtelExecution4TerminalFailure build-OtelExecution5ChildContext \
@@ -17,6 +19,9 @@
 	build-OtelExecution17WaitForCallbackFailure build-OtelExecution18ChainedInvokeFailure \
 	build-OtelExecution18InvokeTarget build-OtelExecution19ExecutionFailure \
 	build-OtelExecution20VirtualContext \
+	build-OtelExecution21CompletedStepReplay build-OtelExecution22UserFunctionContext build-OtelExecution23CallbackFunctionContext \
+	build-OtelExecution24InvocationRetryStatus \
+	build-OtelExecution25CallbackFailureWithoutError build-OtelExecution26ExternalCallbackReplay \
 	build-OtelLongRunning1Wait \
 	build-OtelLongRunning2Retry build-OtelLongRunning3Callback \
 	build-OtelLongRunning4ChainedInvoke build-OtelLongRunning4InvokeTarget
@@ -28,6 +33,8 @@ build-Otel11InvokeTarget build-Otel12ChildContextFailure build-Otel13ParallelFai
 build-Otel14MapFailure build-Otel15WaitInterrupted build-Otel16WaitForConditionFailure \
 build-Otel17WaitForCallbackFailure build-Otel18ChainedInvokeFailure \
 build-Otel18InvokeTarget build-Otel19ExecutionFailure build-Otel20VirtualContext \
+build-Otel21CompletedStepReplay build-Otel22UserFunctionContext build-Otel23CallbackFunctionContext build-Otel24InvocationRetryStatus \
+build-Otel25CallbackFailureWithoutError build-Otel26ExternalCallbackReplay \
 build-OtelExecution1Success \
 build-OtelExecution2WaitResume build-OtelExecution3Retry \
 build-OtelExecution4TerminalFailure build-OtelExecution5ChildContext \
@@ -40,6 +47,9 @@ build-OtelExecution16WaitForConditionFailure \
 build-OtelExecution17WaitForCallbackFailure build-OtelExecution18ChainedInvokeFailure \
 build-OtelExecution18InvokeTarget build-OtelExecution19ExecutionFailure \
 build-OtelExecution20VirtualContext \
+build-OtelExecution21CompletedStepReplay build-OtelExecution22UserFunctionContext build-OtelExecution23CallbackFunctionContext \
+build-OtelExecution24InvocationRetryStatus \
+build-OtelExecution25CallbackFailureWithoutError build-OtelExecution26ExternalCallbackReplay \
 build-OtelLongRunning1Wait \
 build-OtelLongRunning2Retry build-OtelLongRunning3Callback \
 build-OtelLongRunning4ChainedInvoke build-OtelLongRunning4InvokeTarget:

packages/aws-durable-execution-sdk-python-conformance-tests-otel/src/otel_21_completed_step_replay.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""Normal successful replay of a completed step for OTel case 21."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from aws_durable_execution_sdk_python import (
+    DurableContext,
+    StepContext,
+    durable_execution,
+    durable_step,
+)
+from aws_durable_execution_sdk_python.config import Duration
+from common import otel_plugin, require_scenario
+
+
+@durable_step
+def before_wait(_step_context: StepContext) -> str:
+    return "before"
+
+
+@durable_step
+def after_wait(_step_context: StepContext) -> str:
+    return "after"
+
+
+@durable_execution(plugins=[otel_plugin()])
+def handler(event: dict[str, Any], context: DurableContext) -> str:
+    require_scenario(event, "completed-step-replay")
+    before = context.step(before_wait(), name="otel-before-wait")
+    context.wait(Duration.from_seconds(1), name="otel-replay-wait")
+    after = context.step(after_wait(), name="otel-after-wait")
+    return f"{before}-{after}"

packages/aws-durable-execution-sdk-python-conformance-tests-otel/src/otel_22_user_function_context.py

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""Observe the real active SDK context inside public user callbacks."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from aws_durable_execution_sdk_python import (
+    DurableContext,
+    StepContext,
+    durable_execution,
+)
+from aws_durable_execution_sdk_python.config import (
+    Duration,
+    MapConfig,
+    ParallelBranch,
+    ParallelConfig,
+)
+from common import otel_plugin, require_scenario
+from opentelemetry import trace
+
+
+def probe(label: str) -> None:
+    if not trace.get_current_span().get_span_context().is_valid:
+        raise RuntimeError(f"No active span for conformance.{label}")
+    span = trace.get_tracer("aws-durable-execution-conformance").start_span(
+        f"conformance.{label}", attributes={"conformance.callback": label}
+    )
+    span.end()
+
+
+def step_body(_context: StepContext) -> str:
+    probe("step")
+    return "step"
+
+
+def child_step(_context: StepContext) -> str:
+    probe("child-step")
+    return "child-step"
+
+
+def child_body(context: DurableContext) -> str:
+    probe("child")
+    context.step(child_step, name="otel-context-child-step")
+    probe("child-restored")
+    return "child"
+
+
+def parallel_step_a(_context: StepContext) -> str:
+    probe("parallel-step-a")
+    return "a"
+
+
+def parallel_step_b(_context: StepContext) -> str:
+    probe("parallel-step-b")
+    return "b"
+
+
+def parallel_a(context: DurableContext) -> str:
+    probe("parallel-a")
+    return context.step(parallel_step_a, name="otel-context-branch-step-a")
+
+
+def parallel_b(context: DurableContext) -> str:
+    probe("parallel-b")
+    return context.step(parallel_step_b, name="otel-context-branch-step-b")
+
+
+def iteration_name(_item: int, index: int) -> str:
+    return ("otel-context-iteration-0", "otel-context-iteration-1")[index]
+
+
+def mapper(
+    context: DurableContext, item: int, index: int, _items: Sequence[int]
+) -> int:
+    probe(("map-0", "map-1")[index])
+
+    def map_step(_step_context: StepContext) -> int:
+        probe(("map-step-0", "map-step-1")[index])
+        return item
+
+    return context.step(
+        map_step, name=("otel-context-map-step-0", "otel-context-map-step-1")[index]
+    )
+
+
+@durable_execution(plugins=[otel_plugin()])
+def handler(event: dict[str, Any], context: DurableContext) -> str:
+    require_scenario(event, "user-function-context")
+    probe("handler")
+    context.step(step_body, name="otel-context-step")
+    context.run_in_child_context(child_body, name="otel-context-child")
+    context.parallel(
+        [
+            ParallelBranch(parallel_a, name="otel-context-branch-a"),
+            ParallelBranch(parallel_b, name="otel-context-branch-b"),
+        ],
+        name="otel-context-parallel",
+        config=ParallelConfig(max_concurrency=2),
+    )
+    context.map(
+        [0, 1],
+        mapper,
+        name="otel-context-map",
+        config=MapConfig(max_concurrency=2, item_namer=iteration_name),
+    )
+    probe("handler-restored")
+    context.wait(Duration.from_seconds(1), name="otel-context-resume")
+    probe("handler-after-resume")
+    return "context-complete"