From b0a33bc59db24e42d5dd4594f8ed967e377aff8f Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:30:42 -0700 Subject: [PATCH 1/5] fix: isolate incompatible OpenTelemetry API linkage failures --- .github/workflows/conformance-tests.yml | 1 + .github/workflows/e2e-tests.yml | 1 + .github/workflows/otel-conformance-tests.yml | 6 ++ otel-plugin/README.md | 13 +++ .../durable/otel/OtelPluginSupport.java | 32 +++++++ .../otel/GlobalProviderLinkageTest.java | 54 +++++++++++ .../durable/PluginLinkageIntegrationTest.java | 85 +++++++++++++++++ .../lambda/durable/plugin/PluginRunner.java | 7 +- .../plugin/PluginLinkageErrorTest.java | 91 +++++++++++++++++++ 9 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java diff --git a/.github/workflows/conformance-tests.yml b/.github/workflows/conformance-tests.yml index 06baa03f5..c066bf277 100644 --- a/.github/workflows/conformance-tests.yml +++ b/.github/workflows/conformance-tests.yml @@ -25,6 +25,7 @@ concurrency: # same stack -- mirrors e2e-tests.yml. group: conformance-tests cancel-in-progress: false + queue: max permissions: contents: read diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e57070144..5dd246d57 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -26,6 +26,7 @@ on: concurrency: group: e2e-tests cancel-in-progress: false + queue: max # permission can be added at job level or workflow level permissions: diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 68b3bc3f2..87077a82f 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -53,6 +53,12 @@ on: permissions: {} +# Serialize shared test resources and retain pending PR runs instead of replacing them. +concurrency: + group: otel-conformance-tests + cancel-in-progress: false + queue: max + jobs: opentelemetry: # 1. Run for non-PR events, such as scheduled runs and manual invocations diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c51032aaa..80b37c00a 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -151,6 +151,19 @@ public class MyHandler extends DurableHandler { } ``` +### OpenTelemetry version compatibility + +Keep the OpenTelemetry API, context, SDK, and Java agent versions aligned. This plugin is built and tested against +OpenTelemetry 1.66.0. Global-provider binding needs `GlobalOpenTelemetry.isSet()` and `getOrNoop()`; when the visible +API lacks either method (for example, API 1.49.0), the plugin logs a compatibility diagnostic and disables its telemetry +for that invocation. It does not install a no-op global that would prevent a provider from being registered later. + +The existing 2.x plugin constructors, registration interfaces, and instance lifetime are retained. Nonfatal linkage +errors from plugin callbacks are logged and isolated so healthy plugins and the handler can continue. Fatal JVM errors +and `ThreadDeath` retain their existing propagation behavior. Provider registration and configuration validation remain +unchanged. Align incompatible dependencies to restore instrumentation; error isolation does not make every old +agent/API combination capable of exporting telemetry. + ### 4. Grant Permissions The function's execution role needs the `AWSXRayDaemonWriteAccess` managed policy (or equivalent permissions) to write traces to X-Ray. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 63fbec908..dea21b447 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -151,6 +151,19 @@ record ProviderSetup(SdkTracerProvider sdkTracerProvider, Tracer tracer) {} * @return the resolved provider and tracer, or {@code null} when telemetry must be disabled for this invocation */ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String pluginName) { + try { + return resolveGlobalProvider(instrumentationName, pluginName); + } catch (LinkageError error) { + logger.warn( + "{} telemetry is disabled for this invocation because the visible OpenTelemetry dependencies " + + "are incompatible. Align the OpenTelemetry API, SDK, and Java agent versions.", + pluginName, + error); + return null; + } + } + + private static ProviderSetup resolveGlobalProvider(String instrumentationName, String pluginName) { if (!OtelPluginAutoConfigurationState.isInstalled()) { logger.warn( "{} telemetry is disabled for this invocation because " @@ -160,6 +173,9 @@ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String javaAgentExtensionsDiagnostic()); return null; } + if (!supportsGlobalProviderLookup(pluginName)) { + return null; + } if (!GlobalOpenTelemetry.isSet()) { logger.warn( "{} telemetry is disabled for this invocation because GlobalOpenTelemetry is not initialized yet. " @@ -186,6 +202,22 @@ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String getSdkTracerProviderForFlush(tracerProvider, pluginName), tracerProvider.get(instrumentationName)); } + private static boolean supportsGlobalProviderLookup(String pluginName) { + try { + GlobalOpenTelemetry.class.getMethod("isSet"); + GlobalOpenTelemetry.class.getMethod("getOrNoop"); + return true; + } catch (NoSuchMethodException missingApi) { + logger.warn( + "{} telemetry is disabled for this invocation because the visible OpenTelemetry API lacks {}. " + + "Global provider binding requires GlobalOpenTelemetry.isSet() and getOrNoop(); " + + "align the API, SDK, and Java agent versions.", + pluginName, + missingApi.getMessage()); + return false; + } + } + /** Returns the SdkTracerProvider for flushing, or null if the provider is wrapped by the agent classloader. */ static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvider, String pluginName) { if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java new file mode 100644 index 000000000..90e65d6b5 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java @@ -0,0 +1,54 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class GlobalProviderLinkageTest { + @BeforeEach + void markCustomizerInstalled() { + OtelPluginAutoConfigurationState.markInstalled(); + } + + @AfterEach + void resetCustomizer() { + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void unavailableGlobalMethodDisablesTelemetryLocally(boolean getterFails) { + try (var global = mockStatic(GlobalOpenTelemetry.class)) { + if (getterFails) { + global.when(GlobalOpenTelemetry::isSet).thenReturn(true); + global.when(GlobalOpenTelemetry::getOrNoop).thenThrow(new NoSuchMethodError("old API getter")); + } else { + global.when(GlobalOpenTelemetry::isSet).thenThrow(new NoSuchMethodError("old API probe")); + } + assertNull(OtelPluginSupport.tryResolveGlobalProvider("scope", "test-plugin")); + global.verify(GlobalOpenTelemetry::get, never()); + } + } + + @Test + void fatalJvmFailureStillEscapesProviderLookup() { + var fatal = new InternalError("fatal JVM failure"); + try (var global = mockStatic(GlobalOpenTelemetry.class)) { + global.when(GlobalOpenTelemetry::isSet).thenThrow(fatal); + assertSame( + fatal, + assertThrows( + InternalError.class, + () -> OtelPluginSupport.tryResolveGlobalProvider("scope", "test-plugin"))); + } + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java new file mode 100644 index 000000000..c90a6ca63 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class PluginLinkageIntegrationTest { + @Test + void incompatibleHooksLeaveHealthyPluginsAndReplayWorking() { + var starts = new AtomicInteger(); + var stepCalls = new AtomicInteger(); + var ends = Collections.synchronizedList(new ArrayList()); + var healthy = healthyPlugin(starts, ends); + var config = + DurableConfig.builder().withPlugins(brokenPlugin(), healthy).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + var saved = ctx.step("save", String.class, step -> { + stepCalls.incrementAndGet(); + return input; + }); + ctx.wait("pause", Duration.ofMinutes(1)); + return saved; + }, + config); + + assertEquals(ExecutionStatus.PENDING, runner.run("value").getStatus()); + runner.advanceTime(); + var result = runner.run("value"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult(String.class)); + assertEquals(1, stepCalls.get(), "The completed step must not be repeated on resume"); + assertEquals(2, starts.get()); + assertEquals(List.of(InvocationStatus.PENDING, InvocationStatus.SUCCEEDED), ends); + assertSame( + healthy, + config.getPluginRunner().getPlugins().get(1), + "Existing plugin instances and their lifetime are retained"); + } + + private static DurableExecutionPlugin healthyPlugin(AtomicInteger starts, List ends) { + return new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + starts.incrementAndGet(); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info.invocationStatus()); + } + }; + } + + private static DurableExecutionPlugin brokenPlugin() { + return (DurableExecutionPlugin) Proxy.newProxyInstance( + DurableExecutionPlugin.class.getClassLoader(), + new Class[] {DurableExecutionPlugin.class}, + (proxy, method, args) -> { + if (method.getDeclaringClass() == Object.class) { + return switch (method.getName()) { + case "toString" -> "incompatible plugin"; + case "hashCode" -> System.identityHashCode(proxy); + default -> proxy == args[0]; + }; + } + throw new NoSuchMethodError("incompatible optional instrumentation API"); + }); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java index e3a5707c4..bc4d8dd1d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java @@ -11,7 +11,8 @@ /** * Composes multiple {@link DurableExecutionPlugin} instances into a single dispatcher. * - *

Event hooks are fire-and-forget: each plugin is called in order, errors are swallowed. + *

Event hooks call each plugin in order. Exceptions and nonfatal linkage failures are isolated; other errors, + * including fatal JVM failures, retain their existing propagation behavior. * *

{@code onInvocationEnd} is awaited (the SDK blocks until it returns) to allow plugins to flush data before Lambda * freezes. @@ -44,13 +45,15 @@ public List getPlugins() { // ─── Event hooks ───────────────────────────────────────────────────── - /** Calls a void hook on all plugins, swallowing any errors. */ + /** Calls a void hook on all plugins, isolating exceptions and incompatible binary dependencies. */ private void run(Consumer hook) { for (var plugin : plugins) { try { hook.accept(plugin); } catch (Exception e) { logger.warn("Plugin hook threw exception", e); + } catch (LinkageError e) { + logger.warn("Plugin hook could not link a dependency; check SDK/plugin dependency compatibility", e); } } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java new file mode 100644 index 000000000..276e60738 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java @@ -0,0 +1,91 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.Proxy; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class PluginLinkageErrorTest { + @ParameterizedTest(name = "{0}: {1}") + @MethodSource("linkageFailures") + void linkageFailureDoesNotPreventTheNextPlugin( + String hook, String failureName, Consumer dispatch, Error failure) { + var healthyCalls = new AtomicInteger(); + var runner = new PluginRunner(List.of( + plugin(() -> { + throw failure; + }), + plugin(healthyCalls::incrementAndGet))); + + assertDoesNotThrow(() -> dispatch.accept(runner)); + assertEquals(1, healthyCalls.get(), "The next plugin must still receive the hook"); + } + + @ParameterizedTest(name = "{0}: {1}") + @MethodSource("otherErrors") + void otherErrorsRetainTheirExistingPropagation( + String hook, String failureName, Consumer dispatch, Error failure) { + var healthyCalls = new AtomicInteger(); + var runner = new PluginRunner(List.of( + plugin(() -> { + throw failure; + }), + plugin(healthyCalls::incrementAndGet))); + + assertSame(failure, assertThrows(Error.class, () -> dispatch.accept(runner))); + assertEquals(0, healthyCalls.get(), "Fatal and unrelated errors must not be blanket-caught"); + } + + static Stream linkageFailures() { + return hooks().flatMap(hook -> Stream.of( + new NoSuchMethodError("old API"), + new AbstractMethodError("old implementation"), + new NoClassDefFoundError("missing dependency"), + new ExceptionInInitializerError("dependency initialization")) + .map(error -> Arguments.of(hook.name(), error.getClass().getSimpleName(), hook.dispatch(), error))); + } + + static Stream otherErrors() { + return hooks().flatMap(hook -> Stream.of( + new InternalError("fatal JVM failure"), new ThreadDeath(), new AssertionError("unchanged")) + .map(error -> Arguments.of(hook.name(), error.getClass().getSimpleName(), hook.dispatch(), error))); + } + + private static Stream hooks() { + return Stream.of( + new Hook("invocation start", runner -> runner.onInvocationStart(null)), + new Hook("invocation end", runner -> runner.onInvocationEnd(null)), + new Hook("operation start", runner -> runner.onOperationStart(null)), + new Hook("operation end", runner -> runner.onOperationEnd(null)), + new Hook("operation change", runner -> runner.onOperationChange(null)), + new Hook("user function start", runner -> runner.onUserFunctionStart(null)), + new Hook("user function end", runner -> runner.onUserFunctionEnd(null))); + } + + private static DurableExecutionPlugin plugin(Runnable action) { + return (DurableExecutionPlugin) Proxy.newProxyInstance( + DurableExecutionPlugin.class.getClassLoader(), + new Class[] {DurableExecutionPlugin.class}, + (proxy, method, args) -> { + if (method.getDeclaringClass() == Object.class) { + return switch (method.getName()) { + case "toString" -> "test plugin"; + case "hashCode" -> System.identityHashCode(proxy); + default -> proxy == args[0]; + }; + } + action.run(); + return null; + }); + } + + private record Hook(String name, Consumer dispatch) {} +} From 10e8f11248dab23bac46fc7a1cece829592b6d03 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:29:49 -0700 Subject: [PATCH 2/5] test: verify installed OpenTelemetry API compatibility --- .../scripts/verify_otel_api_compatibility.py | 189 +++++++++++++++++ .github/workflows/build.yml | 4 + .../compatibility/b1/InstalledApiProbe.java | 200 ++++++++++++++++++ .../src/test/compatibility/b1/README.md | 39 ++++ otel-plugin/src/test/compatibility/b1/pom.xml | 52 +++++ 5 files changed, 484 insertions(+) create mode 100644 .github/scripts/verify_otel_api_compatibility.py create mode 100644 otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java create mode 100644 otel-plugin/src/test/compatibility/b1/README.md create mode 100644 otel-plugin/src/test/compatibility/b1/pom.xml diff --git a/.github/scripts/verify_otel_api_compatibility.py b/.github/scripts/verify_otel_api_compatibility.py new file mode 100644 index 000000000..b0e88c7aa --- /dev/null +++ b/.github/scripts/verify_otel_api_compatibility.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise real released/candidate core and plugin artifacts with two visible OTel APIs. + +Dependency resolution and every probe are required: a network, compilation, or case +failure returns nonzero. No production dependency versions are modified. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET + +API_VERSIONS = ("1.49.0", "1.66.0") +RELEASED_VERSION = "2.2.1" +CORE = "aws-durable-execution-sdk-java" +PLUGIN = "aws-durable-execution-sdk-java-plugin-otel" +PROBE = "software.amazon.lambda.durable.otel.InstalledApiProbe" + + +def execute(command: list[str], log: Path, *, env: dict[str, str] | None = None, timeout: int = 300) -> None: + with log.open("w") as output: + try: + result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, env=env, + check=False, timeout=timeout) + except subprocess.TimeoutExpired as error: + raise RuntimeError(f"Command timed out after {timeout}s; log={log}") from error + if result.returncode: + tail = "\n".join(log.read_text(errors="replace").splitlines()[-35:]) + raise RuntimeError(f"Command failed ({result.returncode}); log={log}\n{tail}") + + +def artifact(entries: list[Path], name: str, version: str) -> Path: + matches = [p for p in entries if p.name == f"{name}-{version}.jar"] + if len(matches) != 1: + raise RuntimeError(f"Expected exactly one {name}:{version}, got {matches}") + if not matches[0].is_file(): + raise RuntimeError(f"Resolved artifact is absent: {matches[0]}") + return matches[0].resolve() + + +def jar_facts(path: Path) -> dict[str, str]: + return {"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + + +def snapshot_candidate(path: Path, output: Path) -> Path: + expected = jar_facts(path)["sha256"] + directory = output / "candidate-artifacts" + directory.mkdir(exist_ok=True) + target = directory / path.name + if path.resolve() != target.resolve(): + shutil.copyfile(path, target) + if jar_facts(target)["sha256"] != expected or jar_facts(path)["sha256"] != expected: + raise RuntimeError(f"Candidate changed while being snapshotted: {path}") + return target.resolve() + + +def candidate_jar(root: Path, module: str, name: str) -> Path: + pom = ET.parse(root / "pom.xml") + version = pom.findtext("{http://maven.apache.org/POM/4.0.0}version") + if not version: + raise RuntimeError("Cannot resolve the reactor version from pom.xml") + path = root / module / "target" / f"{name}-{version}.jar" + if not path.is_file(): + raise RuntimeError(f"Build the candidate first; artifact missing: {path}") + return path.resolve() + + +def resolve_classpaths(fixture: Path, output: Path, maven: str) -> dict[str, list[Path]]: + classpaths: dict[str, list[Path]] = {} + for version in API_VERSIONS: + target = output / f"dependencies-{version}.txt" + execute([ + maven, "-B", "-f", str(fixture / "pom.xml"), + "org.apache.maven.plugins:maven-dependency-plugin:3.11.0:build-classpath", + f"-Dotel.api.version={version}", f"-Dmdep.outputFile={target}", + ], output / f"resolve-{version}.log") + classpaths[version] = [Path(p).resolve() for p in target.read_text().strip().split(os.pathsep)] + artifact(classpaths[version], "opentelemetry-api", version) + artifact(classpaths[version], "opentelemetry-context", version) + return classpaths + + +def probe_environment(view: str) -> dict[str, str]: + env = os.environ.copy() + # The fixture sets its own plugin registration/global provider. Do not inherit + # Lambda-hosted CI tracing or a developer's auto-agent/plugin configuration. + for key in ("_X_AMZN_TRACE_ID", "DURABLE_EXECUTION_PLUGINS", "JAVA_TOOL_OPTIONS", + "JDK_JAVA_OPTIONS", "OTEL_JAVAAGENT_EXTENSIONS", "AWS_LAMBDA_EXEC_WRAPPER"): + env.pop(key, None) + env["DURABLE_EXECUTION_PLUGINS"] = f"{view},compat-healthy" + return env + + +def run_matrix(args: argparse.Namespace) -> int: + root = args.root.resolve() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + fixture = root / "otel-plugin/src/test/compatibility/b1" + cp = resolve_classpaths(fixture, output, args.maven) + released_core = artifact(cp["1.66.0"], CORE, RELEASED_VERSION) + released_plugin = artifact(cp["1.66.0"], PLUGIN, RELEASED_VERSION) + new_core = args.new_core.resolve() if args.new_core else candidate_jar(root, "sdk", CORE) + new_plugin = args.new_plugin.resolve() if args.new_plugin else candidate_jar(root, "otel-plugin", PLUGIN) + for jar in (new_core, new_plugin): + if not jar.is_file(): + raise RuntimeError(f"Candidate artifact missing: {jar}") + candidate_inputs = {"core": jar_facts(new_core), "plugin": jar_facts(new_plugin)} + new_core = snapshot_candidate(new_core, output) + new_plugin = snapshot_candidate(new_plugin, output) + classes = output / "classes" + classes.mkdir(exist_ok=True) + execute([args.javac, "--release", "17", "-classpath", os.pathsep.join(map(str, cp["1.66.0"])), + "-d", str(classes), str(fixture / "InstalledApiProbe.java")], output / "compile.log") + services = classes / "META-INF/services" + services.mkdir(parents=True, exist_ok=True) + (services / "software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider").write_text( + PROBE + "$HealthyProvider\n") + report: dict[str, object] = { + "released_core": jar_facts(released_core), "released_plugin": jar_facts(released_plugin), + "new_core": jar_facts(new_core), "new_plugin": jar_facts(new_plugin), + "candidate_inputs": candidate_inputs, + "cases": [], "agent_coverage": "This matrix is visible-API skew, not a deployed Java-agent test.", + } + cases: list[dict[str, object]] = report["cases"] # type: ignore[assignment] + failures = 0 + pairs = {"old-old": (released_core, released_plugin), "new-old": (new_core, released_plugin), + "old-new": (released_core, new_plugin), "new-new": (new_core, new_plugin)} + for version in API_VERSIONS: + api = artifact(cp[version], "opentelemetry-api", version) + context = artifact(cp[version], "opentelemetry-context", version) + dependencies = [p for p in cp[version] if p.name not in + (f"{CORE}-{RELEASED_VERSION}.jar", f"{PLUGIN}-{RELEASED_VERSION}.jar")] + for label, (core, plugin) in pairs.items(): + for view in ("otel-invocation", "otel-execution"): + name = f"{label}-api{version}-{view}" + negative = label == "old-old" and version == "1.49.0" + case: dict[str, object] = {"name": name, "expected_negative_control": negative, + "api": jar_facts(api), "context": jar_facts(context)} + command = [args.java, "-cp", os.pathsep.join(map(str, [classes, core, plugin, *dependencies])), + PROBE, str(core), str(plugin), str(api), str(context), view, + str(negative).lower(), str(version == "1.66.0").lower()] + try: + log = output / f"{name}.log" + execute(command, log, env=probe_environment(view), timeout=90) + contents = log.read_text(errors="replace") + if "COMPAT_PASS " not in contents: + raise RuntimeError("Probe did not report successful completion") + if negative and "NEGATIVE_CONTROL_REPRODUCED" not in contents: + raise RuntimeError("Released negative control did not reproduce the reported failure") + case["passed"] = True + except RuntimeError as error: + failures += 1 + case.update(passed=False, error=str(error)) + cases.append(case) + (output / "results.json").write_text(json.dumps(report, indent=2) + "\n") + print(f"{'PASS' if case['passed'] else 'FAIL'} {name}", flush=True) + report["passed"] = failures == 0 + report["failure_count"] = failures + (output / "results.json").write_text(json.dumps(report, indent=2) + "\n") + print(f"Installed artifact matrix: {len(cases) - failures}/{len(cases)} passed; {output / 'results.json'}") + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--output", type=Path, default=Path("target/otel-api-compatibility")) + parser.add_argument("--new-core", type=Path) + parser.add_argument("--new-plugin", type=Path) + parser.add_argument("--maven", default=shutil.which("mvn") or "mvn") + parser.add_argument("--java", default=shutil.which("java") or "java") + parser.add_argument("--javac", default=shutil.which("javac") or "javac") + try: + return run_matrix(parser.parse_args()) + except (RuntimeError, OSError, subprocess.SubprocessError) as error: + print(f"Compatibility harness failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3e02cc11..b7bf5bef5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,6 +81,10 @@ jobs: - name: Build and test run: mvn -B install --file pom.xml + - name: Verify installed OpenTelemetry API compatibility + if: ${{ matrix.java == 17 }} + run: python3 .github/scripts/verify_otel_api_compatibility.py + - name: Setup uv for coverage badge if: ${{ matrix.java == 17 }} # cicirello/jacoco-badge-generator is a Docker-based action; the diff --git a/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java b/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java new file mode 100644 index 000000000..9d31b1d58 --- /dev/null +++ b/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java @@ -0,0 +1,200 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.ServiceLoader; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.TestResult; + +/** Compiled against the actual released 2.2.1 SPI, then run unchanged in fresh matrix JVMs. */ +public final class InstalledApiProbe { + private static final AtomicInteger HEALTHY_CREATED = new AtomicInteger(); + private static final AtomicInteger HEALTHY_STARTS = new AtomicInteger(); + private static final AtomicInteger HEALTHY_ENDS = new AtomicInteger(); + + private InstalledApiProbe() {} + + public static void main(String[] args) throws Exception { + verifyArtifacts(args); + var view = args[4]; + var root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + var logs = new ListAppender(); + logs.start(); + root.addAppender(logs); + try { + exercise(view, Boolean.parseBoolean(args[5]), Boolean.parseBoolean(args[6]), logs); + System.out.println("COMPAT_PASS " + view + " negative=" + args[5] + " api=" + args[2] + + " core=" + args[0] + " plugin=" + args[1]); + } finally { + root.detachAppender(logs); + logs.stop(); + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + } + + private static void verifyArtifacts(String[] args) throws Exception { + checkSource(DurableExecutionPlugin.class, Path.of(args[0])); + checkSource(GlobalOpenTelemetry.class, Path.of(args[2])); + checkSource(Context.class, Path.of(args[3])); + var provider = ServiceLoader.load(DurableExecutionPluginProvider.class).stream() + .map(ServiceLoader.Provider::get) + .filter(value -> value.getName().equals(args[4])) + .findFirst().orElseThrow(); + check(provider.getApiVersion() == DurableExecutionPluginProvider.API_VERSION, "released SPI version"); + checkSource(provider.getPluginType(), Path.of(args[1])); + } + + private static void exercise(String view, boolean negative, boolean compatible, ListAppender logs) { + GlobalOpenTelemetry.resetForTest(); + // Reproduce #763's documented visible-API skew, not a claim of a deployed agent test. + OtelPluginAutoConfigurationState.markInstalled(); + var handlerCalls = new AtomicInteger(); + var sideEffects = new AtomicInteger(); + var runner = createRunner(handlerCalls, sideEffects); + check(HEALTHY_CREATED.get() == 1, "existing SPI must create the healthy plugin once per configuration"); + var first = runner.run("compatibility-input"); + if (negative) { + assertNegative(first, handlerCalls, sideEffects); + return; + } + check(first.getStatus() == ExecutionStatus.PENDING, "plugin failure must preserve first invocation"); + check(HEALTHY_STARTS.get() == 1 && HEALTHY_ENDS.get() == 1 && handlerCalls.get() == 1, + "healthy plugin and handler must run"); + if (!compatible) { + synchronized (logs) { + check(hasCompatibilityDiagnostic(List.copyOf(logs.list)), "incompatible API must be diagnosed"); + } + } + resumeAndCheck(runner, view, compatible, handlerCalls, sideEffects); + } + + private static LocalDurableTestRunner createRunner( + AtomicInteger handlerCalls, AtomicInteger sideEffects) { + return LocalDurableTestRunner.create(String.class, (input, ctx) -> { + handlerCalls.incrementAndGet(); + var saved = ctx.step("saved", String.class, step -> { + sideEffects.incrementAndGet(); + return input; + }); + ctx.wait("resume", Duration.ofSeconds(1)); + return saved; + }); + } + + private static void assertNegative( + TestResult result, AtomicInteger handlerCalls, AtomicInteger sideEffects) { + check(result.getStatus() == ExecutionStatus.FAILED, "old/old older API must reproduce customer failure"); + var failure = result.getError().orElseThrow(); + check(failure.errorType().endsWith("NoSuchMethodError") + && failure.errorMessage().contains("GlobalOpenTelemetry.isSet"), + "negative control must reproduce the exact unsupported API: " + failure); + check(HEALTHY_STARTS.get() == 0 && handlerCalls.get() == 0 && sideEffects.get() == 0, + "old linkage failure must precede the healthy start hook and handler"); + System.out.println("NEGATIVE_CONTROL_REPRODUCED NoSuchMethodError GlobalOpenTelemetry.isSet"); + } + + private static void resumeAndCheck(LocalDurableTestRunner runner, String view, boolean compatible, + AtomicInteger handlerCalls, AtomicInteger sideEffects) { + var exporter = InMemorySpanExporter.create(); + var tracing = registerLateGlobal(compatible, exporter); + try { + runner.advanceTime(); + var last = runner.runUntilComplete("compatibility-input"); + check(last.getStatus() == ExecutionStatus.SUCCEEDED, "handler must complete after resume"); + check("compatibility-input".equals(last.getResult(String.class)), "handler output must be preserved"); + check(HEALTHY_STARTS.get() == 2 && HEALTHY_ENDS.get() == 2 && handlerCalls.get() == 2, + "healthy hooks and handler must remain active on resume"); + check(sideEffects.get() == 1, "completed user step must not repeat on resume"); + check(HEALTHY_CREATED.get() == 1, "resume must preserve the existing 2.x plugin instance lifetime"); + var spans = exporter.getFinishedSpanItems(); + if (compatible) check(spans.stream().anyMatch(span -> span.getName().equals("Workflow")), + "compatible global provider must export Workflow spans in " + view); + else check(spans.isEmpty(), "unsupported global API must disable the affected instrumentation"); + } finally { + if (tracing != null) tracing.close(); + } + } + + private static SdkTracerProvider registerLateGlobal(boolean compatible, InMemorySpanExporter exporter) { + // Registration must succeed: an early plugin must not freeze the global as no-op. + if (!compatible) { + GlobalOpenTelemetry.set(OpenTelemetry.noop()); + return null; + } + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + DeterministicIdGenerator.installOn(builder); + DurableSampler.installOn(builder); + var tracing = builder.build(); + OpenTelemetrySdk.builder().setTracerProvider(tracing).buildAndRegisterGlobal(); + return tracing; + } + + /** A real old-SPI service provider, discovered alongside the actual released/candidate OTel provider. */ + public static final class HealthyProvider implements DurableExecutionPluginProvider { + @Override + public String getName() { return "compat-healthy"; } + + @Override + public int getApiVersion() { return API_VERSION; } + + @Override + public Class getPluginType() { return HealthyPlugin.class; } + + @Override + public DurableExecutionPlugin createPlugin() { + HEALTHY_CREATED.incrementAndGet(); + return new HealthyPlugin(); + } + } + + public static final class HealthyPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { HEALTHY_STARTS.incrementAndGet(); } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { HEALTHY_ENDS.incrementAndGet(); } + } + + private static boolean hasCompatibilityDiagnostic(List events) { + return events.stream().anyMatch(event -> { + var text = event.getFormattedMessage(); + var error = event.getThrowableProxy(); + if (error != null) text += " " + error.getClassName() + " " + error.getMessage(); + return event.getLevel().isGreaterOrEqual(Level.WARN) + && text.contains("OpenTelemetry") + && (text.contains("API") || text.contains("isSet") || text.contains("getOrNoop") + || text.contains("NoSuchMethodError")); + }); + } + + private static void checkSource(Class type, Path expected) throws Exception { + var actual = Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toRealPath(); + check(actual.equals(expected.toRealPath()), type.getName() + " loaded from wrong artifact: " + actual); + } + + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/otel-plugin/src/test/compatibility/b1/README.md b/otel-plugin/src/test/compatibility/b1/README.md new file mode 100644 index 000000000..a0d8ab922 --- /dev/null +++ b/otel-plugin/src/test/compatibility/b1/README.md @@ -0,0 +1,39 @@ +# Installed OpenTelemetry API compatibility + +Run after building the SDK and OTel plugin: + +```sh +python3 .github/scripts/verify_otel_api_compatibility.py +``` + +The driver resolves real released core, testing, and plugin **2.2.1** artifacts +through Maven. It compiles one probe against the released SPI and runs fresh JVMs +with old/old, new/old, old/new, and new/new core/plugin pairs, in both OTel views, +using actual API/context **1.49.0** and **1.66.0** jars. Candidate jars come from +the reactor build, or explicit `--new-core` / `--new-plugin` paths. + +The sixteen required cases verify: + +- Selected core, plugin, API, and context classes load from the intended jars. +- The existing service-provider API discovers and creates a healthy plugin, with + the same instance lifetime across suspension/resume. +- Old/old plus API 1.49 reproduces the exact `GlobalOpenTelemetry.isSet` + `NoSuchMethodError` and customer failure, before healthy start hooks/user code. +- Fixed combinations isolate that mismatch, report a diagnostic, and preserve + healthy hooks, handler output, and completed-step replay behavior. +- An early unsupported/uninitialized global provider does not install a no-op + global: subsequent registration must succeed. API 1.66 then exports real + Workflow spans; unsupported API 1.49 disables the affected instrumentation. + +Resolution, compilation, timeout, and probe failures all fail the command. There +is no network-dependent skip. Logs, artifact SHA-256 hashes, negative-control +results, and the case summary are written to `target/otel-api-compatibility`. +The fixture POM is independent of the reactor and changes no production version +or dependency floor. + +This matrix reproduces visible-API/classpath skew with real artifacts. Its test +marker enables the documented plugin auto-configuration path; it does **not** +claim to deploy or validate every Java-agent version. Actual agent validation +must identify the released agent version, extension jar, visible API, runtime, +and observed behavior separately. Provider registration remains fail-fast for +invalid configuration; the fixture does not introduce invocation factories. diff --git a/otel-plugin/src/test/compatibility/b1/pom.xml b/otel-plugin/src/test/compatibility/b1/pom.xml new file mode 100644 index 000000000..bd6992fe4 --- /dev/null +++ b/otel-plugin/src/test/compatibility/b1/pom.xml @@ -0,0 +1,52 @@ + + + + + 4.0.0 + software.amazon.lambda.durable.compatibility + otel-installed-api-fixture + 1.0-SNAPSHOT + pom + + 1.66.0 + + + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-plugin-otel + 2.2.1 + + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-testing + 2.2.1 + + + io.opentelemetry + opentelemetry-api + ${otel.api.version} + + + io.opentelemetry + opentelemetry-context + ${otel.api.version} + + + io.opentelemetry + opentelemetry-sdk + 1.66.0 + + + io.opentelemetry + opentelemetry-sdk-testing + 1.66.0 + + + ch.qos.logback + logback-classic + 1.6.5 + + + From 3a49c5a96d4121899c9336ae39f38f23d2fd1219 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 01:34:41 +0000 Subject: [PATCH 3/5] ci: run OTel conformance on validated CodeBuild Java 21 --- .github/workflows/otel-conformance-tests.yml | 33 +++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 87077a82f..452db089d 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -68,8 +68,9 @@ jobs: actions: write contents: read id-token: write - uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@43872f8d5dd917fe3ee6d01a6c953c1ff67c5fe4 + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@a66037abbbfa55fde97f714e30f0bc262edefd63 with: + runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} language: java resource_prefix: j sdk_repository: aws/aws-durable-execution-sdk-java @@ -81,13 +82,37 @@ jobs: # checked out (.build/durable-sdk). examples_dir: .build/durable-sdk/conformance-tests-otel setup_command: | - if [ -z "${JAVA_HOME_21_X64:-}" ]; then - echo "The runner does not provide Java 21" + set -euo pipefail + if [ -n "${JAVA_HOME_21_X64:-}" ]; then + export JAVA_HOME="$JAVA_HOME_21_X64" + elif [ -x /usr/lib/jvm/java-21-amazon-corretto/bin/java ]; then + export JAVA_HOME=/usr/lib/jvm/java-21-amazon-corretto + elif [ -z "${JAVA_HOME:-}" ]; then + export JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")" + fi + if [ ! -x "$JAVA_HOME/bin/java" ] || [ ! -x "$JAVA_HOME/bin/javac" ]; then + echo "Selected JAVA_HOME=$JAVA_HOME is not a complete JDK" exit 1 fi - export JAVA_HOME="$JAVA_HOME_21_X64" export PATH="$JAVA_HOME/bin:$PATH" + java_spec=$(java -XshowSettings:properties -version 2>&1 | awk '$1 == "java.specification.version" {print $3}') + if [ "$java_spec" != "21" ]; then + echo "Java 21 is required; selected JAVA_HOME=$JAVA_HOME reports specification version $java_spec" + exit 1 + fi + javac_version=$(javac -version 2>&1) + if [[ ! "$javac_version" =~ ^javac[[:space:]]21([.]|[[:space:]]|$) ]]; then + echo "Java 21 javac is required; selected compiler reports $javac_version" + exit 1 + fi java -version + javac -version + if [ -n "${GITHUB_ENV:-}" ]; then + echo "JAVA_HOME=$JAVA_HOME" >> "$GITHUB_ENV" + fi + if [ -n "${GITHUB_PATH:-}" ]; then + echo "$JAVA_HOME/bin" >> "$GITHUB_PATH" + fi prepare_command: | JAVA_SDK_VERSION=$( mvn -B -q \ From 7c50701be4d4d62d77cddcab1f09301243612ade Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 02:52:45 +0000 Subject: [PATCH 4/5] ci: trigger OTel compatibility checks for module and harness changes --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7bf5bef5..61e067dba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,8 @@ on: - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'insight-plugin/**' + - 'otel-plugin/**' + - '.github/scripts/verify_otel_api_compatibility.py' - 'examples/**' - 'pom.xml' push: @@ -42,6 +44,8 @@ on: - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'insight-plugin/**' + - 'otel-plugin/**' + - '.github/scripts/verify_otel_api_compatibility.py' - 'examples/**' - 'pom.xml' From f4e4e1db81c6fb0fbbaf25815d73757abf3fda7c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 03:04:55 +0000 Subject: [PATCH 5/5] ci: use read-only resolver for review-comment intake --- .github/workflows/ai-pr-review-address.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ai-pr-review-address.yml b/.github/workflows/ai-pr-review-address.yml index 2b6e25d79..5bd13defb 100644 --- a/.github/workflows/ai-pr-review-address.yml +++ b/.github/workflows/ai-pr-review-address.yml @@ -23,7 +23,10 @@ jobs: contents: read issues: read pull-requests: read - uses: aws/aws-durable-execution-ci/.github/workflows/ai-pr-review-address.yml@d6b017da14385908951d23e26c790b28a4e5f9f0 + uses: aws/aws-durable-execution-ci/.github/workflows/ai-work-item-resolver.yml@d6b017da14385908951d23e26c790b28a4e5f9f0 + with: + work-scope: review + upload-work-items: true address: if: >-